From f683b5de47ae57a4fb6e9b80ad2f83c34c913486 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 7 May 2015 21:27:53 +0100 Subject: [PATCH 001/175] Store presence cachemap in an ordered dict, so that the newer serials will be at the end --- synapse/handlers/presence.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 9e1561040..6547e0434 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -26,6 +26,7 @@ import synapse.metrics from ._base import BaseHandler import logging +from collections import OrderedDict logger = logging.getLogger(__name__) @@ -143,7 +144,7 @@ class PresenceHandler(BaseHandler): self._remote_offline_serials = [] # map any user to a UserPresenceCache - self._user_cachemap = {} + self._user_cachemap = OrderedDict() # keep them sorted by serial self._user_cachemap_latest_serial = 0 metrics.register_callback( @@ -165,6 +166,14 @@ class PresenceHandler(BaseHandler): else: return UserPresenceCache() + def _bump_serial(self, user=None): + self._user_cachemap_latest_serial += 1 + + if user: + # Move to end + cache = self._user_cachemap.pop(user) + self._user_cachemap[user] = cache + def registered_user(self, user): return self.store.create_presence(user.localpart) @@ -301,7 +310,7 @@ class PresenceHandler(BaseHandler): def changed_presencelike_data(self, user, state): statuscache = self._get_or_make_usercache(user) - self._user_cachemap_latest_serial += 1 + self._bump_serial(user=user) statuscache.update(state, serial=self._user_cachemap_latest_serial) return self.push_presence(user, statuscache=statuscache) @@ -323,7 +332,7 @@ class PresenceHandler(BaseHandler): # No actual update but we need to bump the serial anyway for the # event source - self._user_cachemap_latest_serial += 1 + self._bump_serial() statuscache.update({}, serial=self._user_cachemap_latest_serial) self.push_update_to_local_and_remote( @@ -706,7 +715,7 @@ class PresenceHandler(BaseHandler): statuscache = self._get_or_make_usercache(user) - self._user_cachemap_latest_serial += 1 + self._bump_serial(user=user) statuscache.update(state, serial=self._user_cachemap_latest_serial) if not observers and not room_ids: From 45543028bbeb8395e8bbc5768680f6bf074d366f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 7 May 2015 22:40:10 +0100 Subject: [PATCH 002/175] Use the presence cachemap ordering to early-abort the iteration loop --- synapse/handlers/presence.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6547e0434..601a4c6db 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -875,10 +875,15 @@ class PresenceEventSource(object): updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. - for observed_user in cachemap.keys(): + for observed_user in reversed(cachemap.keys()): cached = cachemap[observed_user] - if cached.serial <= from_key or cached.serial > max_serial: + # Since this is ordered in descending order of serial, we can just + # stop once we've seen enough + if cached.serial <= from_key: + break + + if cached.serial > max_serial: continue if not (yield self.is_visible(observer_user, observed_user)): From fca28d243e520dc3f8bae919182120c4757d575c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 May 2015 16:27:36 +0100 Subject: [PATCH 003/175] Change the way we create observers to deferreds so that we don't get spammed by 'unhandled errors' --- synapse/crypto/keyring.py | 8 +++- synapse/rest/media/v1/base_resource.py | 8 +++- synapse/util/async.py | 60 ++++++++++++++++++++------ 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 8709394b9..a859872ce 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -26,7 +26,7 @@ from synapse.api.errors import SynapseError, Codes from synapse.util.retryutils import get_retry_limiter -from synapse.util.async import create_observer +from synapse.util.async import ObservableDeferred from OpenSSL import crypto @@ -111,6 +111,10 @@ class Keyring(object): if download is None: download = self._get_server_verify_key_impl(server_name, key_ids) + download = ObservableDeferred( + download, + consumeErrors=True + ) self.key_downloads[server_name] = download @download.addBoth @@ -118,7 +122,7 @@ class Keyring(object): del self.key_downloads[server_name] return ret - r = yield create_observer(download) + r = yield download.observe() defer.returnValue(r) @defer.inlineCallbacks diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 08c8d75af..4af5f7387 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -25,7 +25,7 @@ from twisted.internet import defer from twisted.web.resource import Resource from twisted.protocols.basic import FileSender -from synapse.util.async import create_observer +from synapse.util.async import ObservableDeferred import os @@ -83,13 +83,17 @@ class BaseMediaResource(Resource): download = self.downloads.get(key) if download is None: download = self._get_remote_media_impl(server_name, media_id) + download = ObservableDeferred( + download, + consumeErrors=True + ) self.downloads[key] = download @download.addBoth def callback(media_info): del self.downloads[key] return media_info - return create_observer(download) + return download.observe() @defer.inlineCallbacks def _get_remote_media_impl(self, server_name, media_id): diff --git a/synapse/util/async.py b/synapse/util/async.py index d8febdb90..34acb14a6 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -34,20 +34,56 @@ def run_on_reactor(): return sleep(0) -def create_observer(deferred): - """Creates a deferred that observes the result or failure of the given - deferred *without* affecting the given deferred. +class ObservableDeferred(object): + """Wraps a deferred object so that we can add observer deferreds. These + observer deferreds do not affect the callback chain of the original + deferred. + + If consumeErrors is true errors will be captured from the origin deferred. """ - d = defer.Deferred() - def callback(r): - d.callback(r) - return r + __slots__ = ["_deferred", "_observers", "_result"] - def errback(f): - d.errback(f) - return f + def __init__(self, deferred, consumeErrors=False): + object.__setattr__(self, "_deferred", deferred) + object.__setattr__(self, "_result", None) + object.__setattr__(self, "_observers", []) - deferred.addCallbacks(callback, errback) + def callback(r): + self._result = (True, r) + while self._observers: + try: + self._observers.pop().callback(r) + except: + pass + return r - return d + def errback(f): + self._result = (False, f) + while self._observers: + try: + self._observers.pop().errback(f) + except: + pass + + if consumeErrors: + return None + else: + return f + + deferred.addCallbacks(callback, errback) + + def observe(self): + if not self._result: + d = defer.Deferred() + self._observers.append(d) + return d + else: + success, res = self._result + return defer.succeed(res) if success else defer.fail(res) + + def __getattr__(self, name): + return getattr(self._deferred, name) + + def __setattr__(self, name, value): + setattr(self._deferred, name, value) From 476899295f5fd6cff64799bcbc84cd4bf9005e33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 May 2015 16:32:18 +0100 Subject: [PATCH 004/175] Change the way we do logging contexts so that they survive divergences --- demo/start.sh | 1 + synapse/crypto/keyclient.py | 17 +++++++---- synapse/federation/federation_server.py | 40 ++++++++++++------------- synapse/handlers/events.py | 8 ++--- synapse/handlers/presence.py | 34 ++++++++++----------- synapse/handlers/profile.py | 15 +++++----- synapse/http/client.py | 6 +++- synapse/http/matrixfederationclient.py | 32 ++++++++++---------- synapse/notifier.py | 16 +++++----- synapse/storage/_base.py | 11 +++---- synapse/util/__init__.py | 8 +++-- synapse/util/async.py | 6 ++-- synapse/util/logcontext.py | 31 +++++++++++++++++++ 13 files changed, 128 insertions(+), 97 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index 5b3daef57..b9cc14b9d 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -31,6 +31,7 @@ for port in 8080 8081 8082; do #rm $DIR/etc/$port.config python -m synapse.app.homeserver \ --generate-config \ + --enable_registration \ -H "localhost:$https_port" \ --config-path "$DIR/etc/$port.config" \ diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 4911f0896..24f15f315 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -18,7 +18,9 @@ from twisted.web.http import HTTPClient from twisted.internet.protocol import Factory from twisted.internet import defer, reactor from synapse.http.endpoint import matrix_federation_endpoint -from synapse.util.logcontext import PreserveLoggingContext +from synapse.util.logcontext import ( + preserve_context_over_fn, preserve_context_over_deferred +) import simplejson as json import logging @@ -40,11 +42,14 @@ def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1): for i in range(5): try: - with PreserveLoggingContext(): - protocol = yield endpoint.connect(factory) - server_response, server_certificate = yield protocol.remote_key - defer.returnValue((server_response, server_certificate)) - return + protocol = yield preserve_context_over_fn( + endpoint.connect, factory + ) + server_response, server_certificate = yield preserve_context_over_deferred( + protocol.remote_key + ) + defer.returnValue((server_response, server_certificate)) + return except SynapseKeyClientError as e: logger.exception("Error getting key for %r" % (server_name,)) if e.status.startswith("4"): diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 2b46188c9..cd79e23f4 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -20,7 +20,6 @@ from .federation_base import FederationBase from .units import Transaction, Edu from synapse.util.logutils import log_function -from synapse.util.logcontext import PreserveLoggingContext from synapse.events import FrozenEvent import synapse.metrics @@ -123,29 +122,28 @@ class FederationServer(FederationBase): logger.debug("[%s] Transaction is new", transaction.transaction_id) - with PreserveLoggingContext(): - results = [] + results = [] - for pdu in pdu_list: - d = self._handle_new_pdu(transaction.origin, pdu) + for pdu in pdu_list: + d = self._handle_new_pdu(transaction.origin, pdu) - try: - yield d - results.append({}) - except FederationError as e: - self.send_failure(e, transaction.origin) - results.append({"error": str(e)}) - except Exception as e: - results.append({"error": str(e)}) - logger.exception("Failed to handle PDU") + try: + yield d + results.append({}) + except FederationError as e: + self.send_failure(e, transaction.origin) + results.append({"error": str(e)}) + except Exception as e: + results.append({"error": str(e)}) + logger.exception("Failed to handle PDU") - if hasattr(transaction, "edus"): - for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu( - transaction.origin, - edu.edu_type, - edu.content - ) + if hasattr(transaction, "edus"): + for edu in [Edu(**x) for x in transaction.edus]: + self.received_edu( + transaction.origin, + edu.edu_type, + edu.content + ) for failure in getattr(transaction, "pdu_failures", []): logger.info("Got failure %r", failure) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index f9f855213..993d33ba4 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -15,7 +15,6 @@ from twisted.internet import defer -from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function from synapse.types import UserID from synapse.events.utils import serialize_event @@ -81,10 +80,9 @@ class EventStreamHandler(BaseHandler): # thundering herds on restart. timeout = random.randint(int(timeout*0.9), int(timeout*1.1)) - with PreserveLoggingContext(): - events, tokens = yield self.notifier.get_events_for( - auth_user, room_ids, pagin_config, timeout - ) + events, tokens = yield self.notifier.get_events_for( + auth_user, room_ids, pagin_config, timeout + ) time_now = self.clock.time_msec() diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 9e1561040..6ae39a1d3 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -19,7 +19,6 @@ from synapse.api.errors import SynapseError, AuthError from synapse.api.constants import PresenceState from synapse.util.logutils import log_function -from synapse.util.logcontext import PreserveLoggingContext from synapse.types import UserID import synapse.metrics @@ -278,15 +277,14 @@ class PresenceHandler(BaseHandler): now_online = state["presence"] != PresenceState.OFFLINE was_polling = target_user in self._user_cachemap - with PreserveLoggingContext(): - if now_online and not was_polling: - self.start_polling_presence(target_user, state=state) - elif not now_online and was_polling: - self.stop_polling_presence(target_user) + if now_online and not was_polling: + self.start_polling_presence(target_user, state=state) + elif not now_online and was_polling: + self.stop_polling_presence(target_user) - # TODO(paul): perform a presence push as part of start/stop poll so - # we don't have to do this all the time - self.changed_presencelike_data(target_user, state) + # TODO(paul): perform a presence push as part of start/stop poll so + # we don't have to do this all the time + self.changed_presencelike_data(target_user, state) def bump_presence_active_time(self, user, now=None): if now is None: @@ -408,10 +406,10 @@ class PresenceHandler(BaseHandler): yield self.store.set_presence_list_accepted( observer_user.localpart, observed_user.to_string() ) - with PreserveLoggingContext(): - self.start_polling_presence( - observer_user, target_user=observed_user - ) + + self.start_polling_presence( + observer_user, target_user=observed_user + ) @defer.inlineCallbacks def deny_presence(self, observed_user, observer_user): @@ -430,10 +428,9 @@ class PresenceHandler(BaseHandler): observer_user.localpart, observed_user.to_string() ) - with PreserveLoggingContext(): - self.stop_polling_presence( - observer_user, target_user=observed_user - ) + self.stop_polling_presence( + observer_user, target_user=observed_user + ) @defer.inlineCallbacks def get_presence_list(self, observer_user, accepted=None): @@ -766,8 +763,7 @@ class PresenceHandler(BaseHandler): if not self._remote_sendmap[user]: del self._remote_sendmap[user] - with PreserveLoggingContext(): - yield defer.DeferredList(deferreds, consumeErrors=True) + yield defer.DeferredList(deferreds, consumeErrors=True) @defer.inlineCallbacks def push_update_to_local_and_remote(self, observed_user, statuscache, diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index ee2732b84..a7de7a80f 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -154,14 +154,13 @@ class ProfileHandler(BaseHandler): if not self.hs.is_mine(user): defer.returnValue(None) - with PreserveLoggingContext(): - (displayname, avatar_url) = yield defer.gatherResults( - [ - self.store.get_profile_displayname(user.localpart), - self.store.get_profile_avatar_url(user.localpart), - ], - consumeErrors=True - ) + (displayname, avatar_url) = yield defer.gatherResults( + [ + self.store.get_profile_displayname(user.localpart), + self.store.get_profile_avatar_url(user.localpart), + ], + consumeErrors=True + ) state["displayname"] = displayname state["avatar_url"] = avatar_url diff --git a/synapse/http/client.py b/synapse/http/client.py index e8a5dedab..5b3cefb2d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -14,6 +14,7 @@ # limitations under the License. from synapse.api.errors import CodeMessageException +from synapse.util.logcontext import preserve_context_over_fn from syutil.jsonutil import encode_canonical_json import synapse.metrics @@ -61,7 +62,10 @@ class SimpleHttpClient(object): # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.inc(method) - d = self.agent.request(method, *args, **kwargs) + d = preserve_context_over_fn( + self.agent.request, + method, *args, **kwargs + ) def _cb(response): incoming_responses_counter.inc(method, response.code) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 7fa295cad..c99d237c7 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep -from synapse.util.logcontext import PreserveLoggingContext +from synapse.util.logcontext import preserve_context_over_fn import synapse.metrics from syutil.jsonutil import encode_canonical_json @@ -144,22 +144,22 @@ class MatrixFederationHttpClient(object): producer = body_callback(method, url_bytes, headers_dict) try: - with PreserveLoggingContext(): - request_deferred = self.agent.request( - destination, - endpoint, - method, - path_bytes, - param_bytes, - query_bytes, - Headers(headers_dict), - producer - ) + request_deferred = preserve_context_over_fn( + self.agent.request, + destination, + endpoint, + method, + path_bytes, + param_bytes, + query_bytes, + Headers(headers_dict), + producer + ) - response = yield self.clock.time_bound_deferred( - request_deferred, - time_out=60, - ) + response = yield self.clock.time_bound_deferred( + request_deferred, + time_out=60, + ) logger.debug("Got response to %s", method) break diff --git a/synapse/notifier.py b/synapse/notifier.py index 78eb28e4b..fbbccb38e 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from synapse.util.logcontext import PreserveLoggingContext +from synapse.util.logcontext import preserve_context_over_deferred from synapse.types import StreamToken import synapse.metrics @@ -223,11 +223,10 @@ class Notifier(object): def eb(failure): logger.exception("Failed to notify listener", failure) - with PreserveLoggingContext(): - yield defer.DeferredList( + yield defer.DeferredList( [notify(l).addErrback(eb) for l in listeners], consumeErrors=True, - ) + ) @defer.inlineCallbacks @log_function @@ -298,11 +297,10 @@ class Notifier(object): failure.getTracebackObject()) ) - with PreserveLoggingContext(): - yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners], - consumeErrors=True, - ) + yield defer.DeferredList( + [notify(l).addErrback(eb) for l in listeners], + consumeErrors=True, + ) @defer.inlineCallbacks def wait_for_events(self, user, rooms, filter, timeout, callback): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ee5587c72..b0020f51d 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -18,7 +18,7 @@ from synapse.api.errors import StoreError from synapse.events import FrozenEvent from synapse.events.utils import prune_event from synapse.util.logutils import log_function -from synapse.util.logcontext import PreserveLoggingContext, LoggingContext +from synapse.util.logcontext import preserve_context_over_fn, LoggingContext from synapse.util.lrucache import LruCache import synapse.metrics @@ -419,10 +419,11 @@ class SQLBaseStore(object): self._txn_perf_counters.update(desc, start, end) sql_txn_timer.inc_by(duration, desc) - with PreserveLoggingContext(): - result = yield self._db_pool.runWithConnection( - inner_func, *args, **kwargs - ) + result = yield preserve_context_over_fn( + self._db_pool.runWithConnection, + inner_func, *args, **kwargs + ) + for after_callback, after_args in after_callbacks: after_callback(*after_args) defer.returnValue(result) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 79109d0b1..364b92785 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.util.logcontext import LoggingContext +from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from twisted.internet import defer, reactor, task @@ -50,8 +50,10 @@ class Clock(object): current_context = LoggingContext.current_context() def wrapped_callback(): - LoggingContext.thread_local.current_context = current_context - callback() + with PreserveLoggingContext(): + LoggingContext.thread_local.current_context = current_context + callback() + return reactor.callLater(delay, wrapped_callback) def cancel_call_later(self, timer): diff --git a/synapse/util/async.py b/synapse/util/async.py index d8febdb90..f78395a43 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -16,15 +16,13 @@ from twisted.internet import defer, reactor -from .logcontext import PreserveLoggingContext +from .logcontext import preserve_context_over_deferred -@defer.inlineCallbacks def sleep(seconds): d = defer.Deferred() reactor.callLater(seconds, d.callback, seconds) - with PreserveLoggingContext(): - yield d + return preserve_context_over_deferred(d) def run_on_reactor(): diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index da7872e95..192e3f49f 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + import threading import logging @@ -129,3 +131,32 @@ class PreserveLoggingContext(object): def __exit__(self, type, value, traceback): """Restores the current logging context""" LoggingContext.thread_local.current_context = self.current_context + + +def preserve_context_over_fn(fn, *args, **kwargs): + with PreserveLoggingContext(): + deferred = fn(*args, **kwargs) + + return preserve_context_over_deferred(deferred) + + +def preserve_context_over_deferred(deferred): + d = defer.Deferred() + + current_context = LoggingContext.current_context() + + def cb(res): + with PreserveLoggingContext(): + LoggingContext.thread_local.current_context = current_context + res = d.callback(res) + return res + + def eb(failure): + with PreserveLoggingContext(): + LoggingContext.thread_local.current_context = current_context + res = d.errback(failure) + return res + + deferred.addCallbacks(cb, eb) + + return d From 4ac194159200e9cfb0020003409592e5324cbd18 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 May 2015 16:32:56 +0100 Subject: [PATCH 005/175] PEP8 --- synapse/handlers/profile.py | 1 - synapse/notifier.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index a7de7a80f..ffb449d45 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException from synapse.api.constants import EventTypes, Membership -from synapse.util.logcontext import PreserveLoggingContext from synapse.types import UserID from ._base import BaseHandler diff --git a/synapse/notifier.py b/synapse/notifier.py index fbbccb38e..7282dfd7f 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -16,7 +16,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from synapse.util.logcontext import preserve_context_over_deferred from synapse.types import StreamToken import synapse.metrics @@ -224,8 +223,8 @@ class Notifier(object): logger.exception("Failed to notify listener", failure) yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners], - consumeErrors=True, + [notify(l).addErrback(eb) for l in listeners], + consumeErrors=True, ) @defer.inlineCallbacks From da1aa07db5f29436aeaae8d0dcbfe9872b4a7233 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 May 2015 16:52:49 +0100 Subject: [PATCH 006/175] Add some docs --- synapse/util/logcontext.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 192e3f49f..3dce8d2bf 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -134,13 +134,25 @@ class PreserveLoggingContext(object): def preserve_context_over_fn(fn, *args, **kwargs): - with PreserveLoggingContext(): - deferred = fn(*args, **kwargs) + """Takes a function and invokes it with the given arguments, but removes + and restores the current logging context while doing so. - return preserve_context_over_deferred(deferred) + If the result is a deferred, call preserve_context_over_deferred before + returning it. + """ + with PreserveLoggingContext(): + res = fn(*args, **kwargs) + + if isinstance(res, defer.Deferred): + return preserve_context_over_deferred(res) + else: + return res def preserve_context_over_deferred(deferred): + """Given a deferred wrap it such that any callbacks added later to it will + be invoked with the current context. + """ d = defer.Deferred() current_context = LoggingContext.current_context() From 2236ef6c92b7964665f5c43b941754d70aa506d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 May 2015 19:53:34 +0100 Subject: [PATCH 007/175] Fix up leak. Add warnings. --- synapse/handlers/_base.py | 11 +++++---- synapse/handlers/federation.py | 29 +++++++++++++---------- synapse/handlers/presence.py | 10 ++++---- synapse/handlers/typing.py | 4 +++- synapse/http/server.py | 6 +++-- synapse/util/__init__.py | 3 ++- synapse/util/distributor.py | 43 ++++++++++++++++------------------ synapse/util/logcontext.py | 11 ++++++++- 8 files changed, 69 insertions(+), 48 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 4b3f4eada..ddc5c21e7 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -20,6 +20,8 @@ from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes from synapse.types import UserID +from synapse.util.logcontext import PreserveLoggingContext + import logging @@ -137,10 +139,11 @@ class BaseHandler(object): "Failed to get destination from event %s", s.event_id ) - # Don't block waiting on waking up all the listeners. - notify_d = self.notifier.on_new_room_event( - event, extra_users=extra_users - ) + with PreserveLoggingContext(): + # Don't block waiting on waking up all the listeners. + notify_d = self.notifier.on_new_room_event( + event, extra_users=extra_users + ) def log_failure(f): logger.warn( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 85e275722..77c315c47 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -21,6 +21,7 @@ from synapse.api.errors import ( AuthError, FederationError, StoreError, ) from synapse.api.constants import EventTypes, Membership, RejectedReason +from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.util.frozenutils import unfreeze @@ -197,9 +198,10 @@ class FederationHandler(BaseHandler): target_user = UserID.from_string(target_user_id) extra_users.append(target_user) - d = self.notifier.on_new_room_event( - event, extra_users=extra_users - ) + with PreserveLoggingContext(): + d = self.notifier.on_new_room_event( + event, extra_users=extra_users + ) def log_failure(f): logger.warn( @@ -431,9 +433,10 @@ class FederationHandler(BaseHandler): auth_events=auth_events, ) - d = self.notifier.on_new_room_event( - new_event, extra_users=[joinee] - ) + with PreserveLoggingContext(): + d = self.notifier.on_new_room_event( + new_event, extra_users=[joinee] + ) def log_failure(f): logger.warn( @@ -512,9 +515,10 @@ class FederationHandler(BaseHandler): target_user = UserID.from_string(target_user_id) extra_users.append(target_user) - d = self.notifier.on_new_room_event( - event, extra_users=extra_users - ) + with PreserveLoggingContext(): + d = self.notifier.on_new_room_event( + event, extra_users=extra_users + ) def log_failure(f): logger.warn( @@ -594,9 +598,10 @@ class FederationHandler(BaseHandler): ) target_user = UserID.from_string(event.state_key) - d = self.notifier.on_new_room_event( - event, extra_users=[target_user], - ) + with PreserveLoggingContext(): + d = self.notifier.on_new_room_event( + event, extra_users=[target_user], + ) def log_failure(f): logger.warn( diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6ae39a1d3..1edab0549 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError from synapse.api.constants import PresenceState +from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function from synapse.types import UserID import synapse.metrics @@ -808,10 +809,11 @@ class PresenceHandler(BaseHandler): def push_update_to_clients(self, observed_user, users_to_push=[], room_ids=[], statuscache=None): - self.notifier.on_new_user_event( - users_to_push, - room_ids, - ) + with PreserveLoggingContext(): + self.notifier.on_new_user_event( + users_to_push, + room_ids, + ) class PresenceEventSource(object): diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index c0b2bd7db..64fe51aa3 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError, AuthError +from synapse.util.logcontext import PreserveLoggingContext from synapse.types import UserID import logging @@ -216,7 +217,8 @@ class TypingNotificationHandler(BaseHandler): self._latest_room_serial += 1 self._room_serials[room_id] = self._latest_room_serial - self.notifier.on_new_user_event(rooms=[room_id]) + with PreserveLoggingContext(): + self.notifier.on_new_user_event(rooms=[room_id]) class TypingNotificationEventSource(object): diff --git a/synapse/http/server.py b/synapse/http/server.py index 93ecbd758..73efbff4f 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -17,7 +17,7 @@ from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) -from synapse.util.logcontext import LoggingContext +from synapse.util.logcontext import LoggingContext, PreserveLoggingContext import synapse.metrics from syutil.jsonutil import ( @@ -85,7 +85,9 @@ def request_handler(request_handler): "Received request: %s %s", request.method, request.path ) - yield request_handler(self, request) + d = request_handler(self, request) + with PreserveLoggingContext(): + yield d code = request.code except CodeMessageException as e: code = e.code diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 364b92785..fd3eb1f57 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -54,7 +54,8 @@ class Clock(object): LoggingContext.thread_local.current_context = current_context callback() - return reactor.callLater(delay, wrapped_callback) + with PreserveLoggingContext(): + return reactor.callLater(delay, wrapped_callback) def cancel_call_later(self, timer): timer.cancel() diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 9d9c35039..5b150cb0e 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.util.logcontext import PreserveLoggingContext - from twisted.internet import defer import logging @@ -93,7 +91,6 @@ class Signal(object): Each observer callable may return a Deferred.""" self.observers.append(observer) - @defer.inlineCallbacks def fire(self, *args, **kwargs): """Invokes every callable in the observer list, passing in the args and kwargs. Exceptions thrown by observers are logged but ignored. It is @@ -101,24 +98,24 @@ class Signal(object): Returns a Deferred that will complete when all the observers have completed.""" - with PreserveLoggingContext(): - deferreds = [] - for observer in self.observers: - d = defer.maybeDeferred(observer, *args, **kwargs) - def eb(failure): - logger.warning( - "%s signal observer %s failed: %r", - self.name, observer, failure, - exc_info=( - failure.type, - failure.value, - failure.getTracebackObject())) - if not self.suppress_failures: - failure.raiseException() - deferreds.append(d.addErrback(eb)) - results = [] - for deferred in deferreds: - result = yield deferred - results.append(result) - defer.returnValue(results) + def eb(failure): + logger.warning( + "%s signal observer %s failed: %r", + self.name, observer, failure, + exc_info=( + failure.type, + failure.value, + failure.getTracebackObject())) + if not self.suppress_failures: + failure.raiseException() + + deferreds = [ + defer.maybeDeferred(observer, *args, **kwargs) + for observer in self.observers + ] + + d = defer.gatherResults(deferreds, consumeErrors=True) + d.addErrback(eb) + + return d diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index 3dce8d2bf..a92d518b4 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -132,6 +132,13 @@ class PreserveLoggingContext(object): """Restores the current logging context""" LoggingContext.thread_local.current_context = self.current_context + if self.current_context is not LoggingContext.sentinel: + if self.current_context.parent_context is None: + logger.warn( + "Restoring dead context: %s", + self.current_context, + ) + def preserve_context_over_fn(fn, *args, **kwargs): """Takes a function and invokes it with the given arguments, but removes @@ -169,6 +176,8 @@ def preserve_context_over_deferred(deferred): res = d.errback(failure) return res - deferred.addCallbacks(cb, eb) + if deferred.called: + return deferred + deferred.addCallbacks(cb, eb) return d From 3c224f4d0ef653685cc8d20fd48d9ad62d7050e3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 11 May 2015 11:00:06 +0100 Subject: [PATCH 008/175] SYN-376: Add script for converting server keys from v1 to v2 --- scripts-dev/convert_server_keys.py | 113 +++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 scripts-dev/convert_server_keys.py diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py new file mode 100644 index 000000000..024ddcdbd --- /dev/null +++ b/scripts-dev/convert_server_keys.py @@ -0,0 +1,113 @@ +import psycopg2 +import yaml +import sys +import json +import time +import hashlib +from syutil.base64util import encode_base64 +from syutil.crypto.signing_key import read_signing_keys +from syutil.crypto.jsonsign import sign_json +from syutil.jsonutil import encode_canonical_json + + +def select_v1_keys(connection): + cursor = connection.cursor() + cursor.execute("SELECT server_name, key_id, verify_key FROM server_signature_keys") + rows = cursor.fetchall() + cursor.close() + results = {} + for server_name, key_id, verify_key in rows: + results.setdefault(server_name, {})[key_id] = encode_base64(verify_key) + return results + + +def select_v1_certs(connection): + cursor = connection.cursor() + cursor.execute("SELECT server_name, tls_certificate FROM server_tls_certificates") + rows = cursor.fetchall() + cursor.close() + results = {} + for server_name, tls_certificate in rows: + results[server_name] = tls_certificate + return results + + +def select_v2_json(connection): + cursor = connection.cursor() + cursor.execute("SELECT server_name, key_id, key_json FROM server_keys_json") + rows = cursor.fetchall() + cursor.close() + results = {} + for server_name, key_id, key_json in rows: + results.setdefault(server_name, {})[key_id] = json.loads(str(key_json).decode("utf-8")) + return results + + +def convert_v1_to_v2(server_name, valid_until, keys, certificate): + return { + "old_verify_keys": {}, + "server_name": server_name, + "verify_keys": keys, + "valid_until_ts": valid_until, + "tls_fingerprints": [fingerprint(certificate)], + } + + +def fingerprint(certificate): + finger = hashlib.sha256(certificate) + return {"sha256": encode_base64(finger.digest())} + + +def rows_v2(server, json): + valid_until = json["valid_until_ts"] + key_json = encode_canonical_json(json) + for key_id in json["verify_keys"]: + yield (server, key_id, "-", valid_until, valid_until, buffer(key_json)) + + +def main(): + config = yaml.load(open(sys.argv[1])) + valid_until = int(time.time() / (3600 * 24)) * 1000 * 3600 * 24 + + server_name = config["server_name"] + signing_key = read_signing_keys(open(config["signing_key_path"]))[0] + + database = config["database"] + assert database["name"] == "psycopg2", "Can only convert for postgresql" + args = database["args"] + args.pop("cp_max") + args.pop("cp_min") + connection = psycopg2.connect(**args) + keys = select_v1_keys(connection) + certificates = select_v1_certs(connection) + json = select_v2_json(connection) + + result = {} + for server in keys: + if not server in json: + v2_json = convert_v1_to_v2( + server, valid_until, keys[server], certificates[server] + ) + v2_json = sign_json(v2_json, server_name, signing_key) + result[server] = v2_json + + yaml.safe_dump(result, sys.stdout, default_flow_style=False) + + rows = list( + row for server, json in result.items() + for row in rows_v2(server, json) + ) + + cursor = connection.cursor() + cursor.executemany( + "INSERT INTO server_keys_json (" + " server_name, key_id, from_server," + " ts_added_ms, ts_valid_until_ms, key_json" + ") VALUES (%s, %s, %s, %s, %s, %s)", + rows + ) + connection.commit() + + +if __name__ == '__main__': + main() From 5e3b254dc810c5f17f635005253a977af65e3a53 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 11 May 2015 14:37:33 +0100 Subject: [PATCH 009/175] Use wait_for_events to implement 'get_events' --- synapse/notifier.py | 115 +++++++++++++------------------------------- 1 file changed, 33 insertions(+), 82 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 78eb28e4b..e16a4608e 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -305,14 +305,16 @@ class Notifier(object): ) @defer.inlineCallbacks - def wait_for_events(self, user, rooms, filter, timeout, callback): + def wait_for_events(self, user, rooms, timeout, callback, + from_token=StreamToken("s0", "0", "0")): """Wait until the callback returns a non empty response or the timeout fires. """ deferred = defer.Deferred() - - from_token = StreamToken("s0", "0", "0") + appservice = yield self.hs.get_datastore().get_app_service_by_user_id( + user.to_string() + ) listener = [_NotificationListener( user=user, @@ -321,6 +323,7 @@ class Notifier(object): limit=1, timeout=timeout, deferred=deferred, + appservice=appservice, )] if timeout: @@ -363,65 +366,43 @@ class Notifier(object): defer.returnValue(result) + @defer.inlineCallbacks def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any new events to happen before returning. """ - deferred = defer.Deferred() - - self._get_events( - deferred, user, rooms, pagination_config.from_token, - pagination_config.limit, timeout - ).addErrback(deferred.errback) - - return deferred - - @defer.inlineCallbacks - def _get_events(self, deferred, user, rooms, from_token, limit, timeout): + from_token = pagination_config.from_token if not from_token: from_token = yield self.event_sources.get_current_token() - appservice = yield self.hs.get_datastore().get_app_service_by_user_id( - user.to_string() - ) + limit = pagination_config.limit - listener = _NotificationListener( - user, - rooms, - from_token, - limit, - timeout, - deferred, - appservice=appservice - ) - - def _timeout_listener(): - # TODO (erikj): We should probably set to_token to the current - # max rather than reusing from_token. - # Remove the timer from the listener so we don't try to cancel it. - listener.timer = None - listener.notify( - self, - [], - listener.from_token, - listener.from_token, - ) - - if timeout: - self._register_with_keys(listener) - - yield self._check_for_updates(listener) - - if not timeout: - _timeout_listener() - else: - # Only add the timer if the listener hasn't been notified - if not listener.notified(): - listener.timer = self.clock.call_later( - timeout/1000.0, _timeout_listener + @defer.inlineCallbacks + def check_for_updates(): + events = [] + end_token = from_token + for name, source in self.event_sources.sources.items(): + keyname = "%s_key" % name + stuff, new_key = yield source.get_new_events_for_user( + user, getattr(from_token, keyname), limit, ) - return + events.extend(stuff) + end_token = from_token.copy_and_replace(keyname, new_key) + + if events: + defer.returnValue((events, (from_token, end_token))) + else: + defer.returnValue(None) + + result = yield self.wait_for_events( + user, rooms, timeout, check_for_updates, from_token=from_token + ) + + if result is None: + result = ([], (from_token, from_token)) + + defer.returnValue(result) @log_function def _register_with_keys(self, listener): @@ -436,36 +417,6 @@ class Notifier(object): listener.appservice, set() ).add(listener) - @defer.inlineCallbacks - @log_function - def _check_for_updates(self, listener): - # TODO (erikj): We need to think about limits across multiple sources - events = [] - - from_token = listener.from_token - limit = listener.limit - - # TODO (erikj): DeferredList? - for name, source in self.event_sources.sources.items(): - keyname = "%s_key" % name - - stuff, new_key = yield source.get_new_events_for_user( - listener.user, - getattr(from_token, keyname), - limit, - ) - - events.extend(stuff) - - from_token = from_token.copy_and_replace(keyname, new_key) - - end_token = from_token - - if events: - listener.notify(self, events, listener.from_token, end_token) - - defer.returnValue(listener) - def _user_joined_room(self, user, room_id): new_listeners = self.user_to_listeners.get(user, set()) From e269c511f61100bfd96bb0201db320aa6d59925c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 11 May 2015 15:01:51 +0100 Subject: [PATCH 010/175] Don't bother passing the events to the notifier since it isn't using them --- synapse/notifier.py | 113 +++++++------------------------------------- 1 file changed, 18 insertions(+), 95 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index e16a4608e..abe12b143 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext +from synapse.util.async import run_on_reactor from synapse.types import StreamToken import synapse.metrics @@ -50,13 +51,9 @@ class _NotificationListener(object): so that it can remove itself from the indexes in the Notifier class. """ - def __init__(self, user, rooms, from_token, limit, timeout, deferred, - appservice=None): + def __init__(self, user, rooms, deferred, appservice=None): self.user = user self.appservice = appservice - self.from_token = from_token - self.limit = limit - self.timeout = timeout self.deferred = deferred self.rooms = rooms self.timer = None @@ -64,17 +61,14 @@ class _NotificationListener(object): def notified(self): return self.deferred.called - def notify(self, notifier, events, start_token, end_token): + def notify(self, notifier): """ Inform whoever is listening about the new events. This will also remove this listener from all the indexes in the Notifier it knows about. """ - result = (events, (start_token, end_token)) - try: - self.deferred.callback(result) - notified_events_counter.inc_by(len(events)) + self.deferred.callback(None) except defer.AlreadyCalledError: pass @@ -161,6 +155,7 @@ class Notifier(object): listening to the room, and any listeners for the users in the `extra_users` param. """ + yield run_on_reactor() # poke any interested application service. self.hs.get_handlers().appservice_handler.notify_interested_services( event @@ -168,8 +163,6 @@ class Notifier(object): room_id = event.room_id - room_source = self.event_sources.sources["room"] - room_listeners = self.room_to_listeners.get(room_id, set()) _discard_if_notified(room_listeners) @@ -200,34 +193,12 @@ class Notifier(object): logger.debug("on_new_room_event listeners %s", listeners) - # TODO (erikj): Can we make this more efficient by hitting the - # db once? - - @defer.inlineCallbacks - def notify(listener): - events, end_key = yield room_source.get_new_events_for_user( - listener.user, - listener.from_token.room_key, - listener.limit, - ) - - if events: - end_token = listener.from_token.copy_and_replace( - "room_key", end_key - ) - - listener.notify( - self, events, listener.from_token, end_token - ) - - def eb(failure): - logger.exception("Failed to notify listener", failure) - with PreserveLoggingContext(): - yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners], - consumeErrors=True, - ) + for listener in listeners: + try: + listener.notify(self) + except: + logger.exception("Failed to notify listener") @defer.inlineCallbacks @log_function @@ -237,11 +208,7 @@ class Notifier(object): Will wake up all listeners for the given users and rooms. """ - # TODO(paul): This is horrible, having to manually list every event - # source here individually - presence_source = self.event_sources.sources["presence"] - typing_source = self.event_sources.sources["typing"] - + yield run_on_reactor() listeners = set() for user in users: @@ -258,51 +225,12 @@ class Notifier(object): listeners |= room_listeners - @defer.inlineCallbacks - def notify(listener): - presence_events, presence_end_key = ( - yield presence_source.get_new_events_for_user( - listener.user, - listener.from_token.presence_key, - listener.limit, - ) - ) - typing_events, typing_end_key = ( - yield typing_source.get_new_events_for_user( - listener.user, - listener.from_token.typing_key, - listener.limit, - ) - ) - - if presence_events or typing_events: - end_token = listener.from_token.copy_and_replace( - "presence_key", presence_end_key - ).copy_and_replace( - "typing_key", typing_end_key - ) - - listener.notify( - self, - presence_events + typing_events, - listener.from_token, - end_token - ) - - def eb(failure): - logger.error( - "Failed to notify listener", - exc_info=( - failure.type, - failure.value, - failure.getTracebackObject()) - ) - with PreserveLoggingContext(): - yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners], - consumeErrors=True, - ) + for listener in listeners: + try: + listener.notify(self) + except: + logger.exception("Failed to notify listener") @defer.inlineCallbacks def wait_for_events(self, user, rooms, timeout, callback, @@ -319,9 +247,6 @@ class Notifier(object): listener = [_NotificationListener( user=user, rooms=rooms, - from_token=from_token, - limit=1, - timeout=timeout, deferred=deferred, appservice=appservice, )] @@ -338,7 +263,7 @@ class Notifier(object): def _timeout_listener(): timed_out[0] = True timer[0] = None - listener[0].notify(self, [], from_token, from_token) + listener[0].notify(self) # We create multiple notification listeners so we have to manage # canceling the timeout ourselves. @@ -350,10 +275,8 @@ class Notifier(object): listener[0] = _NotificationListener( user=user, rooms=rooms, - from_token=from_token, - limit=1, - timeout=timeout, deferred=deferred, + appservice=appservice, ) self._register_with_keys(listener[0]) result = yield callback() From 17653a5dfeae55b1f721336fc23058e6e67a3f9a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 11 May 2015 18:00:33 +0100 Subject: [PATCH 011/175] Move storage.stream._StreamToken to types.RoomStreamToken --- synapse/storage/stream.py | 118 +++++++++++--------------------------- synapse/types.py | 52 +++++++++++++++++ 2 files changed, 85 insertions(+), 85 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 280d4ad60..b03fc67f7 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -37,11 +37,9 @@ from twisted.internet import defer from ._base import SQLBaseStore from synapse.api.constants import EventTypes -from synapse.api.errors import SynapseError +from synapse.types import RoomStreamToken from synapse.util.logutils import log_function -from collections import namedtuple - import logging @@ -55,76 +53,26 @@ _STREAM_TOKEN = "stream" _TOPOLOGICAL_TOKEN = "topological" -class _StreamToken(namedtuple("_StreamToken", "topological stream")): - """Tokens are positions between events. The token "s1" comes after event 1. +def lower_bound(token): + if token.topological is None: + return "(%d < %s)" % (token.stream, "stream_ordering") + else: + return "(%d < %s OR (%d = %s AND %d < %s))" % ( + token.topological, "topological_ordering", + token.topological, "topological_ordering", + token.stream, "stream_ordering", + ) - s0 s1 - | | - [0] V [1] V [2] - Tokens can either be a point in the live event stream or a cursor going - through historic events. - - When traversing the live event stream events are ordered by when they - arrived at the homeserver. - - When traversing historic events the events are ordered by their depth in - the event graph "topological_ordering" and then by when they arrived at the - homeserver "stream_ordering". - - Live tokens start with an "s" followed by the "stream_ordering" id of the - event it comes after. Historic tokens start with a "t" followed by the - "topological_ordering" id of the event it comes after, follewed by "-", - followed by the "stream_ordering" id of the event it comes after. - """ - __slots__ = [] - - @classmethod - def parse(cls, string): - try: - if string[0] == 's': - return cls(topological=None, stream=int(string[1:])) - if string[0] == 't': - parts = string[1:].split('-', 1) - return cls(topological=int(parts[0]), stream=int(parts[1])) - except: - pass - raise SynapseError(400, "Invalid token %r" % (string,)) - - @classmethod - def parse_stream_token(cls, string): - try: - if string[0] == 's': - return cls(topological=None, stream=int(string[1:])) - except: - pass - raise SynapseError(400, "Invalid token %r" % (string,)) - - def __str__(self): - if self.topological is not None: - return "t%d-%d" % (self.topological, self.stream) - else: - return "s%d" % (self.stream,) - - def lower_bound(self): - if self.topological is None: - return "(%d < %s)" % (self.stream, "stream_ordering") - else: - return "(%d < %s OR (%d = %s AND %d < %s))" % ( - self.topological, "topological_ordering", - self.topological, "topological_ordering", - self.stream, "stream_ordering", - ) - - def upper_bound(self): - if self.topological is None: - return "(%d >= %s)" % (self.stream, "stream_ordering") - else: - return "(%d > %s OR (%d = %s AND %d >= %s))" % ( - self.topological, "topological_ordering", - self.topological, "topological_ordering", - self.stream, "stream_ordering", - ) +def upper_bound(token): + if token.topological is None: + return "(%d >= %s)" % (token.stream, "stream_ordering") + else: + return "(%d > %s OR (%d = %s AND %d >= %s))" % ( + token.topological, "topological_ordering", + token.topological, "topological_ordering", + token.stream, "stream_ordering", + ) class StreamStore(SQLBaseStore): @@ -139,8 +87,8 @@ class StreamStore(SQLBaseStore): limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. - from_id = _StreamToken.parse_stream_token(from_key) - to_id = _StreamToken.parse_stream_token(to_key) + from_id = RoomStreamToken.parse_stream_token(from_key) + to_id = RoomStreamToken.parse_stream_token(to_key) if from_key == to_key: defer.returnValue(([], to_key)) @@ -234,8 +182,8 @@ class StreamStore(SQLBaseStore): limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. - from_id = _StreamToken.parse_stream_token(from_key) - to_id = _StreamToken.parse_stream_token(to_key) + from_id = RoomStreamToken.parse_stream_token(from_key) + to_id = RoomStreamToken.parse_stream_token(to_key) if from_key == to_key: return defer.succeed(([], to_key)) @@ -288,17 +236,17 @@ class StreamStore(SQLBaseStore): args = [False, room_id] if direction == 'b': order = "DESC" - bounds = _StreamToken.parse(from_key).upper_bound() + bounds = upper_bound(RoomStreamToken.parse(from_key)) if to_key: bounds = "%s AND %s" % ( - bounds, _StreamToken.parse(to_key).lower_bound() + bounds, lower_bound(RoomStreamToken.parse(to_key)) ) else: order = "ASC" - bounds = _StreamToken.parse(from_key).lower_bound() + bounds = lower_bound(RoomStreamToken.parse(from_key)) if to_key: bounds = "%s AND %s" % ( - bounds, _StreamToken.parse(to_key).upper_bound() + bounds, upper_bound(RoomStreamToken.parse(to_key)) ) if int(limit) > 0: @@ -333,7 +281,7 @@ class StreamStore(SQLBaseStore): # when we are going backwards so we subtract one from the # stream part. toke -= 1 - next_token = str(_StreamToken(topo, toke)) + next_token = str(RoomStreamToken(topo, toke)) else: # TODO (erikj): We should work out what to do here instead. next_token = to_key if to_key else from_key @@ -354,7 +302,7 @@ class StreamStore(SQLBaseStore): with_feedback=False, from_token=None): # TODO (erikj): Handle compressed feedback - end_token = _StreamToken.parse_stream_token(end_token) + end_token = RoomStreamToken.parse_stream_token(end_token) if from_token is None: sql = ( @@ -365,7 +313,7 @@ class StreamStore(SQLBaseStore): " LIMIT ?" ) else: - from_token = _StreamToken.parse_stream_token(from_token) + from_token = RoomStreamToken.parse_stream_token(from_token) sql = ( "SELECT stream_ordering, topological_ordering, event_id" " FROM events" @@ -395,7 +343,7 @@ class StreamStore(SQLBaseStore): # stream part. topo = rows[0]["topological_ordering"] toke = rows[0]["stream_ordering"] - 1 - start_token = str(_StreamToken(topo, toke)) + start_token = str(RoomStreamToken(topo, toke)) token = (start_token, str(end_token)) else: @@ -439,5 +387,5 @@ class StreamStore(SQLBaseStore): stream = row["stream_ordering"] topo = event.depth internal = event.internal_metadata - internal.before = str(_StreamToken(topo, stream - 1)) - internal.after = str(_StreamToken(topo, stream)) + internal.before = str(RoomStreamToken(topo, stream - 1)) + internal.after = str(RoomStreamToken(topo, stream)) diff --git a/synapse/types.py b/synapse/types.py index f6a1b0bbc..0f16867d7 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -121,4 +121,56 @@ class StreamToken( return StreamToken(**d) +class RoomStreamToken(namedtuple("_StreamToken", "topological stream")): + """Tokens are positions between events. The token "s1" comes after event 1. + + s0 s1 + | | + [0] V [1] V [2] + + Tokens can either be a point in the live event stream or a cursor going + through historic events. + + When traversing the live event stream events are ordered by when they + arrived at the homeserver. + + When traversing historic events the events are ordered by their depth in + the event graph "topological_ordering" and then by when they arrived at the + homeserver "stream_ordering". + + Live tokens start with an "s" followed by the "stream_ordering" id of the + event it comes after. Historic tokens start with a "t" followed by the + "topological_ordering" id of the event it comes after, follewed by "-", + followed by the "stream_ordering" id of the event it comes after. + """ + __slots__ = [] + + @classmethod + def parse(cls, string): + try: + if string[0] == 's': + return cls(topological=None, stream=int(string[1:])) + if string[0] == 't': + parts = string[1:].split('-', 1) + return cls(topological=int(parts[0]), stream=int(parts[1])) + except: + pass + raise SynapseError(400, "Invalid token %r" % (string,)) + + @classmethod + def parse_stream_token(cls, string): + try: + if string[0] == 's': + return cls(topological=None, stream=int(string[1:])) + except: + pass + raise SynapseError(400, "Invalid token %r" % (string,)) + + def __str__(self): + if self.topological is not None: + return "t%d-%d" % (self.topological, self.stream) + else: + return "s%d" % (self.stream,) + + ClientInfo = namedtuple("ClientInfo", ("device_id", "token_id")) From 84e6b4001f22b0e8c2f806053189fcdb1e85205b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 11 May 2015 18:01:31 +0100 Subject: [PATCH 012/175] Initial hack at wiring together pagination and backfill --- synapse/handlers/federation.py | 108 +++++++++++++++++++++++++++- synapse/handlers/message.py | 10 ++- synapse/storage/event_federation.py | 28 +++++++- 3 files changed, 141 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 85e275722..4d39cd4b3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -218,10 +218,11 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def backfill(self, dest, room_id, limit): + def backfill(self, dest, room_id, limit, extremities=[]): """ Trigger a backfill request to `dest` for the given `room_id` """ - extremities = yield self.store.get_oldest_events_in_room(room_id) + if not extremities: + extremities = yield self.store.get_oldest_events_in_room(room_id) pdus = yield self.replication_layer.backfill( dest, @@ -248,6 +249,109 @@ class FederationHandler(BaseHandler): defer.returnValue(events) + @defer.inlineCallbacks + def maybe_backfill(self, room_id, current_depth): + """Checks the database to see if we should backfill before paginating + """ + extremities = yield self.store.get_oldest_events_with_depth_in_room( + room_id + ) + + logger.debug("Got extremeties: %r", extremities) + + if not extremities: + return + + # Check if we reached a point where we should start backfilling. + sorted_extremeties_tuple = sorted( + extremities.items(), + key=lambda e: -int(e[1]) + ) + max_depth = sorted_extremeties_tuple[0][1] + + logger.debug("max_depth: %r", max_depth) + if current_depth > max_depth: + return + + # Now we need to decide which hosts to hit first. + + # First we try hosts that are already in the room, that were around + # at the time. TODO: HEURISTIC ALERT. + + curr_state = yield self.state_handler.get_current_state(room_id) + + def get_domains_from_state(state): + joined_users = [ + (state_key, int(event.depth)) + for (e_type, state_key), event in state.items() + if e_type == EventTypes.Member + and event.membership == Membership.JOIN + ] + + joined_domains = {} + for u, d in joined_users: + try: + dom = UserID.from_string(u).domain + old_d = joined_domains.get(dom) + if old_d: + joined_domains[dom] = min(d, old_d) + else: + joined_domains[dom] = d + except: + pass + + return sorted(joined_domains.items(), key=lambda d: d[1]) + + curr_domains = get_domains_from_state(curr_state) + + logger.debug("curr_domains: %r", curr_domains) + + likely_domains = [ + domain for domain, depth in curr_domains + ] + + @defer.inlineCallbacks + def try_backfill(domains): + # TODO: Should we try multiple of these at a time? + for dom in domains: + events = yield self.backfill( + dom, room_id, + limit=100, + extremities=[e for e in extremities.keys()] + ) + + if events: + defer.returnValue(True) + defer.returnValue(False) + + success = yield try_backfill(likely_domains) + if success: + defer.returnValue(True) + + # Huh, well *those* domains didn't work out. Lets try some domains + # from the time. + + tried_domains = set(likely_domains) + + states = yield defer.gatherResults({ + e: self.state_handler.resolve_state_groups([e])[1] + for e in extremities.keys() + }) + + for e_id, _ in sorted_extremeties_tuple: + likely_domains = get_domains_from_state(states[e_id])[0] + + success = yield try_backfill([ + dom for dom in likely_domains + if dom not in tried_domains + ]) + if success: + defer.returnValue(True) + + tried_domains.update(likely_domains) + + defer.returnValue(False) + @defer.inlineCallbacks def send_invite(self, target_host, event): """ Sends the invite to the remote server for signing. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 22e19af17..38e375f86 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -21,7 +21,7 @@ from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator from synapse.util.logcontext import PreserveLoggingContext -from synapse.types import UserID +from synapse.types import UserID, RoomStreamToken from ._base import BaseHandler @@ -92,6 +92,14 @@ class MessageHandler(BaseHandler): yield self.hs.get_event_sources().get_current_token() ) + room_token = RoomStreamToken.parse(pagin_config.from_token.room_key) + if room_token.topological is None: + raise SynapseError(400, "Invalid token") + + yield self.hs.get_handlers().federation_handler.maybe_backfill( + room_id, room_token.topological + ) + user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 74b4e2359..2b5424ced 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -79,6 +79,28 @@ class EventFederationStore(SQLBaseStore): room_id, ) + def get_oldest_events_with_depth_in_room(self, room_id): + return self.runInteraction( + "get_oldest_events_with_depth_in_room", + self.get_oldest_events_with_depth_in_room_txn, + room_id, + ) + + def get_oldest_events_with_depth_in_room_txn(self, txn, room_id): + sql = ( + "SELECT b.event_id, MAX(e.depth) FROM events as e" + " INNER JOIN event_edges as g" + " ON g.event_id = e.event_id AND g.room_id = e.room_id" + " INNER JOIN event_backward_extremities as b" + " ON g.prev_event_id = b.event_id AND g.room_id = b.room_id" + " WHERE b.room_id = ? AND g.is_state is ?" + " GROUP BY b.event_id" + ) + + txn.execute(sql, (room_id, False,)) + + return dict(txn.fetchall()) + def _get_oldest_events_in_room_txn(self, txn, room_id): return self._simple_select_onecol_txn( txn, @@ -247,11 +269,13 @@ class EventFederationStore(SQLBaseStore): do_insert = depth < min_depth if min_depth else True if do_insert: - self._simple_insert_txn( + self._simple_upsert_txn( txn, table="room_depth", - values={ + keyvalues={ "room_id": room_id, + }, + values={ "min_depth": depth, }, ) From 4df11b503957a74a948150950da49574c21887bf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 10:28:10 +0100 Subject: [PATCH 013/175] Make get_current_token accept a direction parameter, which tells whether the source whether we want a token for going 'forwards' or 'backwards' --- synapse/handlers/message.py | 4 +++- synapse/handlers/room.py | 4 ++-- synapse/storage/stream.py | 20 ++++++++++++++++++-- synapse/streams/events.py | 6 +++--- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 38e375f86..1809a44a9 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -89,7 +89,9 @@ class MessageHandler(BaseHandler): if not pagin_config.from_token: pagin_config.from_token = ( - yield self.hs.get_event_sources().get_current_token() + yield self.hs.get_event_sources().get_current_token( + direction='b' + ) ) room_token = RoomStreamToken.parse(pagin_config.from_token.room_key) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index cfa2e38ed..29b6d5275 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -577,8 +577,8 @@ class RoomEventSource(object): defer.returnValue((events, end_key)) - def get_current_key(self): - return self.store.get_room_events_max_id() + def get_current_key(self, direction='f'): + return self.store.get_room_events_max_id(direction) @defer.inlineCallbacks def get_pagination_rows(self, user, config, key): diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index b03fc67f7..8045e17fd 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -364,9 +364,25 @@ class StreamStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_room_events_max_id(self): + def get_room_events_max_id(self, direction='f'): token = yield self._stream_id_gen.get_max_token(self) - defer.returnValue("s%d" % (token,)) + if direction != 'b': + defer.returnValue("s%d" % (token,)) + else: + topo = yield self.runInteraction( + "_get_max_topological_txn", self._get_max_topological_txn + ) + defer.returnValue("t%d-%d" % (topo, token)) + + def _get_max_topological_txn(self, txn): + txn.execute( + "SELECT MAX(topological_ordering) FROM events" + " WHERE outlier = ?", + (False,) + ) + + rows = txn.fetchall() + return rows[0][0] if rows else 0 @defer.inlineCallbacks def _get_min_token(self): diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 5c8e54b78..dff7970be 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -31,7 +31,7 @@ class NullSource(object): def get_new_events_for_user(self, user, from_key, limit): return defer.succeed(([], from_key)) - def get_current_key(self): + def get_current_key(self, direction='f'): return defer.succeed(0) def get_pagination_rows(self, user, pagination_config, key): @@ -52,10 +52,10 @@ class EventSources(object): } @defer.inlineCallbacks - def get_current_token(self): + def get_current_token(self, direction='f'): token = StreamToken( room_key=( - yield self.sources["room"].get_current_key() + yield self.sources["room"].get_current_key(direction) ), presence_key=( yield self.sources["presence"].get_current_key() From 367382b575a61f780f3e70a62cc01a790dcc9375 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 10:35:45 +0100 Subject: [PATCH 014/175] Handle the case where the other side is unreachable when backfilling --- synapse/handlers/federation.py | 56 +++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4d39cd4b3..8b5ac5d6c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -18,7 +18,7 @@ from ._base import BaseHandler from synapse.api.errors import ( - AuthError, FederationError, StoreError, + AuthError, FederationError, StoreError, CodeMessageException, SynapseError, ) from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.util.logutils import log_function @@ -29,6 +29,8 @@ from synapse.crypto.event_signing import ( ) from synapse.types import UserID +from synapse.util.retryutils import NotRetryingDestination + from twisted.internet import defer import itertools @@ -251,15 +253,15 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def maybe_backfill(self, room_id, current_depth): - """Checks the database to see if we should backfill before paginating + """Checks the database to see if we should backfill before paginating, + and if so do. """ extremities = yield self.store.get_oldest_events_with_depth_in_room( room_id ) - logger.debug("Got extremeties: %r", extremities) - if not extremities: + logger.debug("Not backfilling as no extremeties found.") return # Check if we reached a point where we should start backfilling. @@ -269,14 +271,17 @@ class FederationHandler(BaseHandler): ) max_depth = sorted_extremeties_tuple[0][1] - logger.debug("max_depth: %r", max_depth) if current_depth > max_depth: + logger.debug( + "Not backfilling as we don't need to. %d < %d", + current_depth, max_depth, + ) return # Now we need to decide which hosts to hit first. - # First we try hosts that are already in the room, that were around - # at the time. TODO: HEURISTIC ALERT. + # First we try hosts that are already in the room + # TODO: HEURISTIC ALERT. curr_state = yield self.state_handler.get_current_state(room_id) @@ -304,8 +309,6 @@ class FederationHandler(BaseHandler): curr_domains = get_domains_from_state(curr_state) - logger.debug("curr_domains: %r", curr_domains) - likely_domains = [ domain for domain, depth in curr_domains ] @@ -314,11 +317,36 @@ class FederationHandler(BaseHandler): def try_backfill(domains): # TODO: Should we try multiple of these at a time? for dom in domains: - events = yield self.backfill( - dom, room_id, - limit=100, - extremities=[e for e in extremities.keys()] - ) + try: + events = yield self.backfill( + dom, room_id, + limit=100, + extremities=[e for e in extremities.keys()] + ) + except SynapseError: + logger.info( + "Failed to backfil from %s because %s", + dom, e, + ) + continue + except CodeMessageException as e: + if 400 <= e.code < 500: + raise + + logger.info( + "Failed to backfil from %s because %s", + dom, e, + ) + continue + except NotRetryingDestination as e: + logger.info(e.message) + continue + except Exception as e: + logger.info( + "Failed to backfil from %s because %s", + dom, e, + ) + continue if events: defer.returnValue(True) From 5c75adff951b27744528d9c095f4ff1f8df1f77a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 12 May 2015 11:00:37 +0100 Subject: [PATCH 015/175] Add a NotifierUserStream to hold all the notification listeners for a user --- synapse/notifier.py | 228 ++++++++++++++++++++++---------------------- 1 file changed, 115 insertions(+), 113 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index abe12b143..0b50898f3 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -43,28 +43,18 @@ def count(func, l): class _NotificationListener(object): """ This represents a single client connection to the events stream. - The events stream handler will have yielded to the deferred, so to notify the handler it is sufficient to resolve the deferred. - - This listener will also keep track of which rooms it is listening in - so that it can remove itself from the indexes in the Notifier class. """ - def __init__(self, user, rooms, deferred, appservice=None): - self.user = user - self.appservice = appservice + def __init__(self, deferred): self.deferred = deferred - self.rooms = rooms - self.timer = None def notified(self): return self.deferred.called - def notify(self, notifier): - """ Inform whoever is listening about the new events. This will - also remove this listener from all the indexes in the Notifier - it knows about. + def notify(self): + """ Inform whoever is listening about the new events. """ try: @@ -72,27 +62,45 @@ class _NotificationListener(object): except defer.AlreadyCalledError: pass - # Should the following be done be using intrusively linked lists? - # -- erikj + +class _NotifierUserStream(object): + """This represents a user connected to the event stream. + It tracks the most recent stream token for that user. + At a given point a user may have a number of streams listening for + events. + + This listener will also keep track of which rooms it is listening in + so that it can remove itself from the indexes in the Notifier class. + """ + + def __init__(self, user, rooms, current_token, appservice=None): + self.user = user + self.appservice = appservice + self.listeners = set() + self.rooms = rooms + self.current_token = current_token + + def notify(self, new_token): + for listener in self.listeners: + listener.notify(new_token) + self.listeners.clear() + + def remove(self, notifier): + """ Remove this listener from all the indexes in the Notifier + it knows about. + """ for room in self.rooms: - lst = notifier.room_to_listeners.get(room, set()) + lst = notifier.room_to_user_streams.get(room, set()) lst.discard(self) - notifier.user_to_listeners.get(self.user, set()).discard(self) + notifier.user_to_user_streams.get(self.user, set()).discard(self) if self.appservice: - notifier.appservice_to_listeners.get( + notifier.appservice_to_user_streams.get( self.appservice, set() ).discard(self) - # Cancel the timeout for this notifer if one exists. - if self.timer is not None: - try: - notifier.clock.cancel_call_later(self.timer) - except: - logger.warn("Failed to cancel notifier timer") - class Notifier(object): """ This class is responsible for notifying any listeners when there are @@ -104,11 +112,12 @@ class Notifier(object): def __init__(self, hs): self.hs = hs - self.room_to_listeners = {} - self.user_to_listeners = {} - self.appservice_to_listeners = {} + self.user_to_user_stream = {} + self.room_to_user_streams = {} + self.appservice_to_user_streams = {} self.event_sources = hs.get_event_sources() + self.store = hs.get_datastore() self.clock = hs.get_clock() @@ -120,34 +129,34 @@ class Notifier(object): # when rendering the metrics page, which is likely once per minute at # most when scraping it. def count_listeners(): - all_listeners = set() + all_user_streams = set() - for x in self.room_to_listeners.values(): - all_listeners |= x - for x in self.user_to_listeners.values(): - all_listeners |= x - for x in self.appservice_to_listeners.values(): - all_listeners |= x + for x in self.room_to_user_streams.values(): + all_user_streams |= x + for x in self.user_to_user_streams.values(): + all_user_streams |= x + for x in self.appservice_to_user_streams.values(): + all_user_streams |= x - return len(all_listeners) + return sum(len(stream.listeners) for stream in all_user_streams) metrics.register_callback("listeners", count_listeners) metrics.register_callback( "rooms", - lambda: count(bool, self.room_to_listeners.values()), + lambda: count(bool, self.room_to_user_streams.values()), ) metrics.register_callback( "users", - lambda: count(bool, self.user_to_listeners.values()), + lambda: len(self.user_to_user_stream), ) metrics.register_callback( "appservices", - lambda: count(bool, self.appservice_to_listeners.values()), + lambda: count(bool, self.appservice_to_user_streams.values()), ) @log_function @defer.inlineCallbacks - def on_new_room_event(self, event, extra_users=[]): + def on_new_room_event(self, event, new_token, extra_users=[]): """ Used by handlers to inform the notifier something has happened in the room, room event wise. @@ -155,6 +164,7 @@ class Notifier(object): listening to the room, and any listeners for the users in the `extra_users` param. """ + assert isinstance(new_token, StreamToken) yield run_on_reactor() # poke any interested application service. self.hs.get_handlers().appservice_handler.notify_interested_services( @@ -163,72 +173,60 @@ class Notifier(object): room_id = event.room_id - room_listeners = self.room_to_listeners.get(room_id, set()) + room_user_streams = self.room_to_user_streams.get(room_id, set()) - _discard_if_notified(room_listeners) - - listeners = room_listeners.copy() + user_streams = room_user_streams.copy() for user in extra_users: - user_listeners = self.user_to_listeners.get(user, set()) + user_stream = self.user_to_user_stream.get(user) + if user_stream is not None: + user_streams.add(user_stream) - _discard_if_notified(user_listeners) - - listeners |= user_listeners - - for appservice in self.appservice_to_listeners: + for appservice in self.appservice_to_user_streams: # TODO (kegan): Redundant appservice listener checks? - # App services will already be in the room_to_listeners set, but + # App services will already be in the room_to_user_streams set, but # that isn't enough. They need to be checked here in order to # receive *invites* for users they are interested in. Does this - # make the room_to_listeners check somewhat obselete? + # make the room_to_user_streams check somewhat obselete? if appservice.is_interested(event): - app_listeners = self.appservice_to_listeners.get( + app_user_streams = self.appservice_to_user_streams.get( appservice, set() ) + user_streams |= app_user_streams - _discard_if_notified(app_listeners) - - listeners |= app_listeners - - logger.debug("on_new_room_event listeners %s", listeners) + logger.debug("on_new_room_event listeners %s", user_streams) with PreserveLoggingContext(): - for listener in listeners: + for user_stream in user_streams: try: - listener.notify(self) + user_stream.notify(new_token) except: logger.exception("Failed to notify listener") @defer.inlineCallbacks @log_function - def on_new_user_event(self, users=[], rooms=[]): + def on_new_user_event(self, new_token, users=[], rooms=[]): """ Used to inform listeners that something has happend presence/user event wise. Will wake up all listeners for the given users and rooms. """ + assert isinstance(new_token, StreamToken) yield run_on_reactor() - listeners = set() + user_streams = set() for user in users: - user_listeners = self.user_to_listeners.get(user, set()) - - _discard_if_notified(user_listeners) - - listeners |= user_listeners + user_stream = self.user_to_user_stream.get(user) + if user_stream: + user_stream.add(user_stream) for room in rooms: - room_listeners = self.room_to_listeners.get(room, set()) - - _discard_if_notified(room_listeners) - - listeners |= room_listeners + user_streams |= self.room_to_user_streams.get(room, set()) with PreserveLoggingContext(): - for listener in listeners: + for user_stream in user_streams: try: - listener.notify(self) + user_streams.notify(new_token) except: logger.exception("Failed to notify listener") @@ -240,21 +238,32 @@ class Notifier(object): """ deferred = defer.Deferred() - appservice = yield self.hs.get_datastore().get_app_service_by_user_id( - user.to_string() - ) - listener = [_NotificationListener( - user=user, - rooms=rooms, - deferred=deferred, - appservice=appservice, - )] + user_stream = self.user_to_user_streams.get(user) + if user_stream is None: + appservice = yield self.store.get_app_service_by_user_id( + user.to_string() + ) + current_token = yield self.event_sources.get_current_token() + user_stream = _NotifierUserStream( + user=user, + rooms=rooms, + appservice=appservice, + current_token=current_token, + ) + self._register_with_keys(user_stream) + else: + current_token = user_stream.current_token - if timeout: - self._register_with_keys(listener[0]) + if timeout and not current_token.is_after(from_token): + listener = [_NotificationListener(deferred)] + user_stream.listeners.add(listener[0]) + + if current_token.is_after(from_token): + result = yield callback(from_token, current_token) + else: + result = None - result = yield callback() timer = [None] if timeout: @@ -263,23 +272,19 @@ class Notifier(object): def _timeout_listener(): timed_out[0] = True timer[0] = None - listener[0].notify(self) + listener[0].notify(user_stream) # We create multiple notification listeners so we have to manage # canceling the timeout ourselves. timer[0] = self.clock.call_later(timeout/1000., _timeout_listener) while not result and not timed_out[0]: - yield deferred + new_token = yield deferred deferred = defer.Deferred() - listener[0] = _NotificationListener( - user=user, - rooms=rooms, - deferred=deferred, - appservice=appservice, - ) - self._register_with_keys(listener[0]) - result = yield callback() + listener[0] = _NotificationListener(deferred) + user_stream.listeners.add(listener[0]) + result = yield callback(current_token, new_token) + current_token = new_token if timer[0] is not None: try: @@ -302,7 +307,7 @@ class Notifier(object): limit = pagination_config.limit @defer.inlineCallbacks - def check_for_updates(): + def check_for_updates(start_token, end_token): events = [] end_token = from_token for name, source in self.event_sources.sources.items(): @@ -328,26 +333,23 @@ class Notifier(object): defer.returnValue(result) @log_function - def _register_with_keys(self, listener): - for room in listener.rooms: - s = self.room_to_listeners.setdefault(room, set()) - s.add(listener) + def _register_with_keys(self, user_stream): + self.user_to_user_stream[user_stream.user] = user_stream - self.user_to_listeners.setdefault(listener.user, set()).add(listener) + for room in user_stream.rooms: + s = self.room_to_user_stream.setdefault(room, set()) + s.add(user_stream) - if listener.appservice: - self.appservice_to_listeners.setdefault( - listener.appservice, set() - ).add(listener) + if user_stream.appservice: + self.appservice_to_user_stream.setdefault( + user_stream.appservice, set() + ).add(user_stream) def _user_joined_room(self, user, room_id): - new_listeners = self.user_to_listeners.get(user, set()) - - listeners = self.room_to_listeners.setdefault(room_id, set()) - listeners |= new_listeners - - for l in new_listeners: - l.rooms.add(room_id) + new_user_stream = self.user_to_user_stream.get(user) + room_streams = self.room_to_user_streams.setdefault(room_id, set()) + room_streams.add(new_user_stream) + new_user_stream.rooms.add(room_id) def _discard_if_notified(listener_set): From 5002056b16549ba2a9175ab62897014ee58e73ff Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 12 May 2015 11:20:40 +0100 Subject: [PATCH 016/175] SYN-377: Make sure that the StreamIdGenerator.get_next.__exit__ is called from the main thread after the transaction completes, not from database thread before the transaction completes. --- synapse/storage/events.py | 38 +++++++++++++-------------- synapse/storage/util/id_generators.py | 12 ++++++--- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 38395c66a..626a5eaf6 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -23,6 +23,7 @@ from synapse.crypto.event_signing import compute_event_reference_hash from syutil.base64util import decode_base64 from syutil.jsonutil import encode_canonical_json +from contextlib import contextmanager import logging @@ -41,17 +42,25 @@ class EventsStore(SQLBaseStore): self.min_token -= 1 stream_ordering = self.min_token + if stream_ordering is None: + stream_ordering_manager = yield self._stream_id_gen.get_next(self) + else: + @contextmanager + def stream_ordering_manager(): + yield stream_ordering + try: - yield self.runInteraction( - "persist_event", - self._persist_event_txn, - event=event, - context=context, - backfilled=backfilled, - stream_ordering=stream_ordering, - is_new_state=is_new_state, - current_state=current_state, - ) + with stream_ordering_manager as stream_ordering: + yield self.runInteraction( + "persist_event", + self._persist_event_txn, + event=event, + context=context, + backfilled=backfilled, + stream_ordering=stream_ordering, + is_new_state=is_new_state, + current_state=current_state, + ) except _RollbackButIsFineException: pass @@ -95,15 +104,6 @@ class EventsStore(SQLBaseStore): # Remove the any existing cache entries for the event_id txn.call_after(self._invalidate_get_event_cache, event.event_id) - if stream_ordering is None: - with self._stream_id_gen.get_next_txn(txn) as stream_ordering: - return self._persist_event_txn( - txn, event, context, backfilled, - stream_ordering=stream_ordering, - is_new_state=is_new_state, - current_state=current_state, - ) - # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table if current_state: diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index e40eb8a8c..89d1643f1 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -78,14 +78,18 @@ class StreamIdGenerator(object): self._current_max = None self._unfinished_ids = deque() - def get_next_txn(self, txn): + @defer.inlineCallbacks + def get_next(self, store): """ Usage: - with stream_id_gen.get_next_txn(txn) as stream_id: + with yield stream_id_gen.get_next as stream_id: # ... persist event ... """ if not self._current_max: - self._get_or_compute_current_max(txn) + yield store.runInteraction( + "_compute_current_max", + self._get_or_compute_current_max, + ) with self._lock: self._current_max += 1 @@ -101,7 +105,7 @@ class StreamIdGenerator(object): with self._lock: self._unfinished_ids.remove(next_id) - return manager() + defer.returnValue(manager()) @defer.inlineCallbacks def get_max_token(self, store): From 2551b6645d5d0855f72638d718ceaf365bbb5938 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 12 May 2015 11:54:18 +0100 Subject: [PATCH 017/175] Update the end_token correctly, otherwise the token doesn't advance and the client gets duplicate events --- synapse/notifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index abe12b143..ef7d15671 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -311,7 +311,7 @@ class Notifier(object): user, getattr(from_token, keyname), limit, ) events.extend(stuff) - end_token = from_token.copy_and_replace(keyname, new_key) + end_token = end_token.copy_and_replace(keyname, new_key) if events: defer.returnValue((events, (from_token, end_token))) From 95dedb866f04ee4ae034c35130f2a8dc86243fbb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 13:14:29 +0100 Subject: [PATCH 018/175] Unwrap defer.gatherResults failures --- synapse/federation/federation_base.py | 4 +++- synapse/handlers/federation.py | 3 ++- synapse/handlers/message.py | 5 +++-- synapse/handlers/profile.py | 3 ++- synapse/handlers/room.py | 4 ++-- synapse/util/__init__.py | 6 ++++++ 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 21a763214..5217d91aa 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -24,6 +24,8 @@ from synapse.crypto.event_signing import check_event_content_hash from synapse.api.errors import SynapseError +from synapse.util import unwrapFirstError + import logging @@ -94,7 +96,7 @@ class FederationBase(object): yield defer.gatherResults( [do(pdu) for pdu in pdus], consumeErrors=True - ) + ).addErrback(unwrapFirstError) defer.returnValue(signed_pdus) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 77c315c47..cd8500157 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -21,6 +21,7 @@ from synapse.api.errors import ( AuthError, FederationError, StoreError, ) from synapse.api.constants import EventTypes, Membership, RejectedReason +from synapse.util import unwrapFirstError from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor @@ -926,7 +927,7 @@ class FederationHandler(BaseHandler): if d in have_events and not have_events[d] ], consumeErrors=True - ) + ).addErrback(unwrapFirstError) if different_events: local_view = dict(auth_events) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 22e19af17..b7d52647d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -20,6 +20,7 @@ from synapse.api.errors import RoomError, SynapseError from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator +from synapse.util import unwrapFirstError from synapse.util.logcontext import PreserveLoggingContext from synapse.types import UserID @@ -303,7 +304,7 @@ class MessageHandler(BaseHandler): event.room_id ), ] - ) + ).addErrback(unwrapFirstError) start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) @@ -328,7 +329,7 @@ class MessageHandler(BaseHandler): yield defer.gatherResults( [handle_room(e) for e in room_list], consumeErrors=True - ) + ).addErrback(unwrapFirstError) ret = { "rooms": rooms_ret, diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index ffb449d45..71ff78ab2 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException from synapse.api.constants import EventTypes, Membership from synapse.types import UserID +from synapse.util import unwrapFirstError from ._base import BaseHandler @@ -159,7 +160,7 @@ class ProfileHandler(BaseHandler): self.store.get_profile_avatar_url(user.localpart), ], consumeErrors=True - ) + ).addErrback(unwrapFirstError) state["displayname"] = displayname state["avatar_url"] = avatar_url diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index cfa2e38ed..ea5abba6a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -21,7 +21,7 @@ from ._base import BaseHandler from synapse.types import UserID, RoomAlias, RoomID from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import StoreError, SynapseError -from synapse.util import stringutils +from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor from synapse.events.utils import serialize_event @@ -537,7 +537,7 @@ class RoomListHandler(BaseHandler): for room in chunk ], consumeErrors=True, - ) + ).addErrback(unwrapFirstError) for i, room in enumerate(chunk): room["num_joined_members"] = len(results[i]) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index fd3eb1f57..c1a16b639 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -23,6 +23,12 @@ import logging logger = logging.getLogger(__name__) +def unwrapFirstError(failure): + # defer.gatherResults and DeferredLists wrap failures. + failure.trap(defer.FirstError) + return failure.value.subFailure + + class Clock(object): """A small utility that obtains current time-of-day so that time may be mocked during unit-tests. From 8022b27fc26bd2127019f5179c8956ea475dd284 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 13:14:48 +0100 Subject: [PATCH 019/175] Make distributer.fire work as it did --- synapse/util/distributor.py | 44 +++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 5b150cb0e..064c4a7a1 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -15,6 +15,12 @@ from twisted.internet import defer +from synapse.util.logcontext import ( + PreserveLoggingContext, preserve_context_over_deferred, +) + +from synapse.util import unwrapFirstError + import logging @@ -99,23 +105,27 @@ class Signal(object): Returns a Deferred that will complete when all the observers have completed.""" - def eb(failure): - logger.warning( - "%s signal observer %s failed: %r", - self.name, observer, failure, - exc_info=( - failure.type, - failure.value, - failure.getTracebackObject())) - if not self.suppress_failures: - failure.raiseException() + def do(observer): + def eb(failure): + logger.warning( + "%s signal observer %s failed: %r", + self.name, observer, failure, + exc_info=( + failure.type, + failure.value, + failure.getTracebackObject())) + if not self.suppress_failures: + return failure + return defer.maybeDeferred(observer, *args, **kwargs).addErrback(eb) - deferreds = [ - defer.maybeDeferred(observer, *args, **kwargs) - for observer in self.observers - ] + with PreserveLoggingContext(): + deferreds = [ + do(observer) + for observer in self.observers + ] - d = defer.gatherResults(deferreds, consumeErrors=True) - d.addErrback(eb) + d = defer.gatherResults(deferreds, consumeErrors=True) - return d + d.addErrback(unwrapFirstError) + + return preserve_context_over_deferred(d) From 6e5ac4a28fe79162e62b68cc62aa4e37badcc8b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 13:58:14 +0100 Subject: [PATCH 020/175] Err, gatherResults doesn't take a dict... --- synapse/handlers/federation.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8b5ac5d6c..31c09365e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -361,10 +361,13 @@ class FederationHandler(BaseHandler): tried_domains = set(likely_domains) - states = yield defer.gatherResults({ - e: self.state_handler.resolve_state_groups([e])[1] - for e in extremities.keys() - }) + event_ids = list(extremities.keys()) + + states = yield defer.gatherResults([ + self.state_handler.resolve_state_groups([e])[1] + for e in event_ids + ]) + states = dict(zip(event_ids, states)) for e_id, _ in sorted_extremeties_tuple: likely_domains = get_domains_from_state(states[e_id])[0] From a0dfffb33cf8ca721526be0c6a1e05199f2b6258 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:00:31 +0100 Subject: [PATCH 021/175] And another typo. --- synapse/handlers/federation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 31c09365e..6f97127ae 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -274,7 +274,7 @@ class FederationHandler(BaseHandler): if current_depth > max_depth: logger.debug( "Not backfilling as we don't need to. %d < %d", - current_depth, max_depth, + max_depth, current_depth, ) return @@ -364,10 +364,10 @@ class FederationHandler(BaseHandler): event_ids = list(extremities.keys()) states = yield defer.gatherResults([ - self.state_handler.resolve_state_groups([e])[1] + self.state_handler.resolve_state_groups([e]) for e in event_ids ]) - states = dict(zip(event_ids, states)) + states = dict(zip(event_ids, [s[1] for s in states])) for e_id, _ in sorted_extremeties_tuple: likely_domains = get_domains_from_state(states[e_id])[0] From 0d31ad5101546380308e7735d4543102b7e60bca Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:02:01 +0100 Subject: [PATCH 022/175] Typos everywhere --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6f97127ae..7b7b998f0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -370,7 +370,7 @@ class FederationHandler(BaseHandler): states = dict(zip(event_ids, [s[1] for s in states])) for e_id, _ in sorted_extremeties_tuple: - likely_domains = get_domains_from_state(states[e_id])[0] + likely_domains = get_domains_from_state(states[e_id]) success = yield try_backfill([ dom for dom in likely_domains From 07a12231569189be1699f50d71b38414ba822bdc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:09:54 +0100 Subject: [PATCH 023/175] s/backfil/backfill/ --- synapse/handlers/federation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7b7b998f0..109311258 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -325,7 +325,7 @@ class FederationHandler(BaseHandler): ) except SynapseError: logger.info( - "Failed to backfil from %s because %s", + "Failed to backfill from %s because %s", dom, e, ) continue @@ -334,7 +334,7 @@ class FederationHandler(BaseHandler): raise logger.info( - "Failed to backfil from %s because %s", + "Failed to backfill from %s because %s", dom, e, ) continue @@ -342,8 +342,8 @@ class FederationHandler(BaseHandler): logger.info(e.message) continue except Exception as e: - logger.info( - "Failed to backfil from %s because %s", + logger.warn( + "Failed to backfill from %s because %s", dom, e, ) continue From 74850d7f75f64e537c3db36103107aece0fdf47f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:14:58 +0100 Subject: [PATCH 024/175] Do state groups persistence /after/ checking if we have already persisted the event --- synapse/storage/events.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 38395c66a..a66e84b34 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -135,19 +135,17 @@ class EventsStore(SQLBaseStore): outlier = event.internal_metadata.is_outlier() if not outlier: - self._store_state_groups_txn(txn, event, context) - self._update_min_depth_for_room_txn( txn, event.room_id, event.depth ) - have_persisted = self._simple_select_one_onecol_txn( + have_persisted = self._simple_select_one_txn( txn, - table="event_json", + table="events", keyvalues={"event_id": event.event_id}, - retcol="event_id", + retcols=["event_id", "outlier"], allow_none=True, ) @@ -162,7 +160,9 @@ class EventsStore(SQLBaseStore): # if we are persisting an event that we had persisted as an outlier, # but is no longer one. if have_persisted: - if not outlier: + if not outlier and have_persisted["outlier"]: + self._store_state_groups_txn(txn, event, context) + sql = ( "UPDATE event_json SET internal_metadata = ?" " WHERE event_id = ?" @@ -182,6 +182,9 @@ class EventsStore(SQLBaseStore): ) return + if not outlier: + self._store_state_groups_txn(txn, event, context) + self._handle_prev_events( txn, outlier=outlier, From c1779a79bc6da69621d0034e582008e95db02dad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:41:50 +0100 Subject: [PATCH 025/175] Fix up _handle_prev_events to not try to insert duplicate rows --- synapse/storage/event_federation.py | 36 +++++++++++------------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 2b5424ced..6773e4468 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -330,31 +330,21 @@ class EventFederationStore(SQLBaseStore): txn.execute(query, (event_id, room_id)) - # Insert all the prev_events as a backwards thing, they'll get - # deleted in a second if they're incorrect anyway. - self._simple_insert_many_txn( - txn, - table="event_backward_extremities", - values=[ - { - "event_id": e_id, - "room_id": room_id, - } - for e_id, _ in prev_events - ], + query = ( + "INSERT INTO event_backward_extremities (event_id, room_id)" + " SELECT ?, ? WHERE NOT EXISTS (" + " SELECT 1 FROM event_backward_extremities" + " WHERE event_id = ? AND room_id = ?" + " )" + " AND NOT EXISTS (" + " SELECT 1 FROM events WHERE event_id = ? AND room_id = ?" + " )" ) - # Also delete from the backwards extremities table all ones that - # reference events that we have already seen - query = ( - "DELETE FROM event_backward_extremities WHERE EXISTS (" - "SELECT 1 FROM events " - "WHERE " - "event_backward_extremities.event_id = events.event_id " - "AND not events.outlier " - ")" - ) - txn.execute(query) + txn.executemany(query, [ + (e_id, room_id, e_id, room_id, e_id, room_id,) + for e_id, _ in prev_events + ]) txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id From e4eddf9b367bdd0384f9b834cb8ba75db4804ae1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:47:23 +0100 Subject: [PATCH 026/175] We do actually want to delete rows out of event_backward_extremities --- synapse/storage/event_federation.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 6773e4468..f807236eb 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -336,16 +336,23 @@ class EventFederationStore(SQLBaseStore): " SELECT 1 FROM event_backward_extremities" " WHERE event_id = ? AND room_id = ?" " )" - " AND NOT EXISTS (" - " SELECT 1 FROM events WHERE event_id = ? AND room_id = ?" - " )" ) txn.executemany(query, [ - (e_id, room_id, e_id, room_id, e_id, room_id,) + (e_id, room_id, e_id, room_id, ) for e_id, _ in prev_events ]) + + # Also delete from the backwards extremities table all ones that + # reference events that we have already seen + query = ( + "DELETE FROM event_backward_extremities" + " WHERE event_id = ? AND room_id = ?" + ) + txn.executemany(query, [(e_id, room_id) for e_id, _ in prev_events]) + + txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id ) From 30c72d377ef4047f93a6210e25a92dc5272ea0e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:47:40 +0100 Subject: [PATCH 027/175] Newlines --- synapse/storage/event_federation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index f807236eb..7ea0ee232 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -343,7 +343,6 @@ class EventFederationStore(SQLBaseStore): for e_id, _ in prev_events ]) - # Also delete from the backwards extremities table all ones that # reference events that we have already seen query = ( @@ -352,7 +351,6 @@ class EventFederationStore(SQLBaseStore): ) txn.executemany(query, [(e_id, room_id) for e_id, _ in prev_events]) - txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id ) From 8b28209c6050656b998f7eb7fab8ac55ae9b019b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 15:02:53 +0100 Subject: [PATCH 028/175] Err, delete the right stuff --- synapse/storage/event_federation.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7ea0ee232..a1982dfbb 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -336,20 +336,21 @@ class EventFederationStore(SQLBaseStore): " SELECT 1 FROM event_backward_extremities" " WHERE event_id = ? AND room_id = ?" " )" + " AND NOT EXISTS (" + " SELECT 1 FROM events WHERE event_id = ? AND room_id = ?" + " )" ) txn.executemany(query, [ - (e_id, room_id, e_id, room_id, ) + (e_id, room_id, e_id, room_id, e_id, room_id, ) for e_id, _ in prev_events ]) - # Also delete from the backwards extremities table all ones that - # reference events that we have already seen query = ( "DELETE FROM event_backward_extremities" " WHERE event_id = ? AND room_id = ?" ) - txn.executemany(query, [(e_id, room_id) for e_id, _ in prev_events]) + txn.execute(query, (event_id, room_id)) txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id From d7b3ac46f8ac32048559e06770e4fc2d57caeaf7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 12 May 2015 15:44:21 +0100 Subject: [PATCH 029/175] Revert "Improvement to performance of presence event stream handling" --- synapse/handlers/presence.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 28688d532..1edab0549 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -26,7 +26,6 @@ import synapse.metrics from ._base import BaseHandler import logging -from collections import OrderedDict logger = logging.getLogger(__name__) @@ -144,7 +143,7 @@ class PresenceHandler(BaseHandler): self._remote_offline_serials = [] # map any user to a UserPresenceCache - self._user_cachemap = OrderedDict() # keep them sorted by serial + self._user_cachemap = {} self._user_cachemap_latest_serial = 0 metrics.register_callback( @@ -166,14 +165,6 @@ class PresenceHandler(BaseHandler): else: return UserPresenceCache() - def _bump_serial(self, user=None): - self._user_cachemap_latest_serial += 1 - - if user: - # Move to end - cache = self._user_cachemap.pop(user) - self._user_cachemap[user] = cache - def registered_user(self, user): return self.store.create_presence(user.localpart) @@ -309,7 +300,7 @@ class PresenceHandler(BaseHandler): def changed_presencelike_data(self, user, state): statuscache = self._get_or_make_usercache(user) - self._bump_serial(user=user) + self._user_cachemap_latest_serial += 1 statuscache.update(state, serial=self._user_cachemap_latest_serial) return self.push_presence(user, statuscache=statuscache) @@ -331,7 +322,7 @@ class PresenceHandler(BaseHandler): # No actual update but we need to bump the serial anyway for the # event source - self._bump_serial() + self._user_cachemap_latest_serial += 1 statuscache.update({}, serial=self._user_cachemap_latest_serial) self.push_update_to_local_and_remote( @@ -713,7 +704,7 @@ class PresenceHandler(BaseHandler): statuscache = self._get_or_make_usercache(user) - self._bump_serial(user=user) + self._user_cachemap_latest_serial += 1 statuscache.update(state, serial=self._user_cachemap_latest_serial) if not observers and not room_ids: @@ -873,15 +864,10 @@ class PresenceEventSource(object): updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. - for observed_user in reversed(cachemap.keys()): + for observed_user in cachemap.keys(): cached = cachemap[observed_user] - # Since this is ordered in descending order of serial, we can just - # stop once we've seen enough - if cached.serial <= from_key: - break - - if cached.serial > max_serial: + if cached.serial <= from_key or cached.serial > max_serial: continue if not (yield self.is_visible(observer_user, observed_user)): From e12268597804d759f82ccd827359059ba7cb05ef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 16:12:37 +0100 Subject: [PATCH 030/175] You need to call contextmanager --- synapse/storage/events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a5a686907..9242b0a84 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -48,6 +48,7 @@ class EventsStore(SQLBaseStore): @contextmanager def stream_ordering_manager(): yield stream_ordering + stream_ordering_manager = stream_ordering_manager() try: with stream_ordering_manager as stream_ordering: From 80fd2b574c9e56637e67b996289607185c590109 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 16:19:42 +0100 Subject: [PATCH 031/175] Don't talk to yourself when backfilling --- synapse/handlers/federation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7d9906039..880cbd77e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -314,6 +314,7 @@ class FederationHandler(BaseHandler): likely_domains = [ domain for domain, depth in curr_domains + if domain is not self.server_name ] @defer.inlineCallbacks @@ -363,6 +364,7 @@ class FederationHandler(BaseHandler): # from the time. tried_domains = set(likely_domains) + tried_domains.add(self.server_name) event_ids = list(extremities.keys()) From 409bcc76bdbdb5410d755b1eded370491641976f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 11:13:31 +0100 Subject: [PATCH 032/175] Load events for state group seperately --- synapse/storage/_base.py | 4 ++-- synapse/storage/state.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c9fe5a355..0279400a8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -867,9 +867,9 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) def _get_events(self, event_ids, check_redacted=True, - get_prev_content=False): + get_prev_content=False, desc="_get_events"): return self.runInteraction( - "_get_events", self._get_events_txn, event_ids, + desc, self._get_events_txn, event_ids, check_redacted=check_redacted, get_prev_content=get_prev_content, ) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index dbc0e49c1..9ed541299 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -43,6 +43,7 @@ class StateStore(SQLBaseStore): * `state_groups_state`: Maps state group to state events. """ + @defer.inlineCallbacks def get_state_groups(self, event_ids): """ Get the state groups for the given list of event_ids @@ -71,17 +72,22 @@ class StateStore(SQLBaseStore): retcol="event_id", ) - state = self._get_events_txn(txn, state_ids) + # state = self._get_events_txn(txn, state_ids) - res[group] = state + res[group] = state_ids return res - return self.runInteraction( + states = yield self.runInteraction( "get_state_groups", f, ) + for vals in states.values(): + vals[:] = yield self._get_events(vals, desc="_get_state_groups_ev") + + defer.returnValue(states) + def _store_state_groups_txn(self, txn, event, context): if context.current_state is None: return From fec4485e28569718b9a0c341be4aaead8533c280 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 11:22:42 +0100 Subject: [PATCH 033/175] Batch fetching of events for state groups --- synapse/storage/state.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 9ed541299..c300c6e29 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -83,8 +83,31 @@ class StateStore(SQLBaseStore): f, ) + def fetch_events(txn, events): + sql = ( + "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " + " FROM event_json as e" + " LEFT JOIN redactions as r ON e.event_id = r.redacts" + " LEFT JOIN rejections as rej on rej.event_id = e.event_id" + " WHERE e.event_id IN (%s)" + ) % (",".join(["?"]*len(events)),) + + txn.execute(sql, events) + rows = txn.fetchall() + + return [ + self._get_event_from_row_txn( + txn, row[0], row[1], row[2], + rejected_reason=row[3], + ) + for row in rows + ] + for vals in states.values(): - vals[:] = yield self._get_events(vals, desc="_get_state_groups_ev") + vals[:] = yield self.runInteraction( + "_get_state_groups_ev", + fetch_events, vals + ) defer.returnValue(states) From 619a21812be7872832865372587e98ed9e690184 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 11:29:03 +0100 Subject: [PATCH 034/175] defer.gatherResults loop --- synapse/storage/state.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index c300c6e29..2b5c2d999 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -103,12 +103,18 @@ class StateStore(SQLBaseStore): for row in rows ] - for vals in states.values(): + @defer.inlineCallbacks + def c(vals): vals[:] = yield self.runInteraction( "_get_state_groups_ev", fetch_events, vals ) + yield defer.gatherResults( + [c(vals) for vals in states.values()], + consumeErrors=True, + ) + defer.returnValue(states) def _store_state_groups_txn(self, txn, event, context): From 02590c3e1db79c2c8f158a73562d139ce411d5d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 11:31:28 +0100 Subject: [PATCH 035/175] Temp turn off checking for rejections and redactions --- synapse/storage/state.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 2b5c2d999..6d7d576cb 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -85,10 +85,8 @@ class StateStore(SQLBaseStore): def fetch_events(txn, events): sql = ( - "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " + "SELECT e.internal_metadata, e.json " " FROM event_json as e" - " LEFT JOIN redactions as r ON e.event_id = r.redacts" - " LEFT JOIN rejections as rej on rej.event_id = e.event_id" " WHERE e.event_id IN (%s)" ) % (",".join(["?"]*len(events)),) @@ -97,8 +95,7 @@ class StateStore(SQLBaseStore): return [ self._get_event_from_row_txn( - txn, row[0], row[1], row[2], - rejected_reason=row[3], + txn, row[0], row[1], None ) for row in rows ] From 63878c03794d33a8767425e114845159e5c1cb9a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 13 May 2015 13:42:21 +0100 Subject: [PATCH 036/175] Don't bother checking for updates if the stream token hasn't advanced for a user --- synapse/handlers/_base.py | 7 ++- synapse/handlers/federation.py | 25 +++++---- synapse/handlers/presence.py | 4 ++ synapse/handlers/typing.py | 4 +- synapse/notifier.py | 75 ++++++++++++++++++--------- synapse/storage/events.py | 3 ++ synapse/types.py | 19 ++++++- tests/handlers/test_federation.py | 4 +- tests/handlers/test_room.py | 8 +-- tests/handlers/test_typing.py | 12 ++--- tests/rest/client/v1/test_presence.py | 15 ++++-- tests/utils.py | 2 +- 12 files changed, 123 insertions(+), 55 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index ddc5c21e7..833ff4137 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -105,7 +105,9 @@ class BaseHandler(object): if not suppress_auth: self.auth.check(event, auth_events=context.current_state) - yield self.store.persist_event(event, context=context) + (event_stream_id, max_stream_id) = yield self.store.persist_event( + event, context=context + ) federation_handler = self.hs.get_handlers().federation_handler @@ -142,7 +144,8 @@ class BaseHandler(object): with PreserveLoggingContext(): # Don't block waiting on waking up all the listeners. notify_d = self.notifier.on_new_room_event( - event, extra_users=extra_users + event, event_stream_id, max_stream_id, + extra_users=extra_users ) def log_failure(f): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7d9906039..bc0f7b0ee 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -160,7 +160,7 @@ class FederationHandler(BaseHandler): ) try: - yield self._handle_new_event( + _, event_stream_id, max_stream_id = yield self._handle_new_event( origin, event, state=state, @@ -203,7 +203,8 @@ class FederationHandler(BaseHandler): with PreserveLoggingContext(): d = self.notifier.on_new_room_event( - event, extra_users=extra_users + event, event_stream_id, max_stream_id, + extra_users=extra_users ) def log_failure(f): @@ -561,7 +562,7 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } - yield self._handle_new_event( + _, event_stream_id, max_stream_id = yield self._handle_new_event( origin, new_event, state=state, @@ -571,7 +572,8 @@ class FederationHandler(BaseHandler): with PreserveLoggingContext(): d = self.notifier.on_new_room_event( - new_event, extra_users=[joinee] + new_event, event_stream_id, max_stream_id, + extra_users=[joinee] ) def log_failure(f): @@ -637,7 +639,9 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - context = yield self._handle_new_event(origin, event) + context, event_stream_id, max_stream_id = yield self._handle_new_event( + origin, event + ) logger.debug( "on_send_join_request: After _handle_new_event: %s, sigs: %s", @@ -653,7 +657,7 @@ class FederationHandler(BaseHandler): with PreserveLoggingContext(): d = self.notifier.on_new_room_event( - event, extra_users=extra_users + event, event_stream_id, max_stream_id, extra_users=extra_users ) def log_failure(f): @@ -727,7 +731,7 @@ class FederationHandler(BaseHandler): context = yield self.state_handler.compute_event_context(event) - yield self.store.persist_event( + event_stream_id, max_stream_id = yield self.store.persist_event( event, context=context, backfilled=False, @@ -736,7 +740,8 @@ class FederationHandler(BaseHandler): target_user = UserID.from_string(event.state_key) with PreserveLoggingContext(): d = self.notifier.on_new_room_event( - event, extra_users=[target_user], + event, event_stream_id, max_stream_id, + extra_users=[target_user], ) def log_failure(f): @@ -914,7 +919,7 @@ class FederationHandler(BaseHandler): ) raise - yield self.store.persist_event( + event_stream_id, max_stream_id = yield self.store.persist_event( event, context=context, backfilled=backfilled, @@ -922,7 +927,7 @@ class FederationHandler(BaseHandler): current_state=current_state, ) - defer.returnValue(context) + defer.returnValue((context, event_stream_id, max_stream_id)) @defer.inlineCallbacks def on_query_auth(self, origin, event_id, remote_auth_chain, rejects, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 28688d532..7db4b062d 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -345,6 +345,8 @@ class PresenceHandler(BaseHandler): curr_users = yield rm_handler.get_room_members(room_id) for local_user in [c for c in curr_users if self.hs.is_mine(c)]: + statuscache = self._get_or_offline_usercache(local_user) + statuscache.update({}, serial=self._user_cachemap_latest_serial) self.push_update_to_local_and_remote( observed_user=local_user, users_to_push=[user], @@ -820,6 +822,8 @@ class PresenceHandler(BaseHandler): room_ids=[], statuscache=None): with PreserveLoggingContext(): self.notifier.on_new_user_event( + "presence_key", + self._user_cachemap_latest_serial, users_to_push, room_ids, ) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 64fe51aa3..a9895292c 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -218,7 +218,9 @@ class TypingNotificationHandler(BaseHandler): self._room_serials[room_id] = self._latest_room_serial with PreserveLoggingContext(): - self.notifier.on_new_user_event(rooms=[room_id]) + self.notifier.on_new_user_event( + "typing_key", self._latest_room_serial, rooms=[room_id] + ) class TypingNotificationEventSource(object): diff --git a/synapse/notifier.py b/synapse/notifier.py index 214a2b28c..4d10c0503 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -52,12 +52,11 @@ class _NotificationListener(object): def notified(self): return self.deferred.called - def notify(self): + def notify(self, token): """ Inform whoever is listening about the new events. """ - try: - self.deferred.callback(None) + self.deferred.callback(token) except defer.AlreadyCalledError: pass @@ -73,15 +72,18 @@ class _NotifierUserStream(object): """ def __init__(self, user, rooms, current_token, appservice=None): - self.user = user + self.user = str(user) self.appservice = appservice self.listeners = set() - self.rooms = rooms + self.rooms = set(rooms) self.current_token = current_token - def notify(self, new_token): + def notify(self, stream_key, stream_id): + self.current_token = self.current_token.copy_and_replace( + stream_key, stream_id + ) for listener in self.listeners: - listener.notify(new_token) + listener.notify(self.current_token) self.listeners.clear() def remove(self, notifier): @@ -117,6 +119,7 @@ class Notifier(object): self.event_sources = hs.get_event_sources() self.store = hs.get_datastore() + self.pending_new_room_events = [] self.clock = hs.get_clock() @@ -153,9 +156,21 @@ class Notifier(object): lambda: count(bool, self.appservice_to_user_streams.values()), ) + def notify_pending_new_room_events(self, max_room_stream_id): + pending = sorted(self.pending_new_room_events) + self.pending_new_room_events = [] + for event, room_stream_id, extra_users in pending: + if room_stream_id > max_room_stream_id: + self.pending_new_room_events.append(( + event, room_stream_id, extra_users + )) + else: + self._on_new_room_event(event, room_stream_id, extra_users) + @log_function @defer.inlineCallbacks - def on_new_room_event(self, event, new_token, extra_users=[]): + def on_new_room_event(self, event, room_stream_id, max_room_stream_id, + extra_users=[]): """ Used by handlers to inform the notifier something has happened in the room, room event wise. @@ -163,8 +178,18 @@ class Notifier(object): listening to the room, and any listeners for the users in the `extra_users` param. """ - assert isinstance(new_token, StreamToken) yield run_on_reactor() + + self.notify_pending_new_room_events(max_room_stream_id) + + if room_stream_id > max_room_stream_id: + self.pending_new_room_events.append(( + event, room_stream_id, extra_users + )) + else: + self._on_new_room_event(event, room_stream_id, extra_users) + + def _on_new_room_event(self, event, room_stream_id, extra_users=[]): # poke any interested application service. self.hs.get_handlers().appservice_handler.notify_interested_services( event @@ -197,33 +222,32 @@ class Notifier(object): for user_stream in user_streams: try: - user_stream.notify(new_token) + user_stream.notify("room_key", "s%d" % (room_stream_id,)) except: logger.exception("Failed to notify listener") @defer.inlineCallbacks @log_function - def on_new_user_event(self, new_token, users=[], rooms=[]): + def on_new_user_event(self, stream_key, new_token, users=[], rooms=[]): """ Used to inform listeners that something has happend presence/user event wise. Will wake up all listeners for the given users and rooms. """ - assert isinstance(new_token, StreamToken) yield run_on_reactor() user_streams = set() for user in users: user_stream = self.user_to_user_stream.get(user) - if user_stream: - user_stream.add(user_stream) + if user_stream is not None: + user_streams.add(user_stream) for room in rooms: user_streams |= self.room_to_user_streams.get(room, set()) for user_stream in user_streams: try: - user_streams.notify(new_token) + user_stream.notify(stream_key, new_token) except: logger.exception("Failed to notify listener") @@ -236,12 +260,12 @@ class Notifier(object): deferred = defer.Deferred() - user_stream = self.user_to_user_streams.get(user) + user = str(user) + user_stream = self.user_to_user_stream.get(user) if user_stream is None: - appservice = yield self.store.get_app_service_by_user_id( - user.to_string() - ) + appservice = yield self.store.get_app_service_by_user_id(user) current_token = yield self.event_sources.get_current_token() + rooms = yield self.store.get_rooms_for_user(user) user_stream = _NotifierUserStream( user=user, rooms=rooms, @@ -252,8 +276,9 @@ class Notifier(object): else: current_token = user_stream.current_token + listener = [_NotificationListener(deferred)] + if timeout and not current_token.is_after(from_token): - listener = [_NotificationListener(deferred)] user_stream.listeners.add(listener[0]) if current_token.is_after(from_token): @@ -334,7 +359,7 @@ class Notifier(object): self.user_to_user_stream[user_stream.user] = user_stream for room in user_stream.rooms: - s = self.room_to_user_stream.setdefault(room, set()) + s = self.room_to_user_streams.setdefault(room, set()) s.add(user_stream) if user_stream.appservice: @@ -343,10 +368,12 @@ class Notifier(object): ).add(user_stream) def _user_joined_room(self, user, room_id): + user = str(user) new_user_stream = self.user_to_user_stream.get(user) - room_streams = self.room_to_user_streams.setdefault(room_id, set()) - room_streams.add(new_user_stream) - new_user_stream.rooms.add(room_id) + if new_user_stream is not None: + room_streams = self.room_to_user_streams.setdefault(room_id, set()) + room_streams.add(new_user_stream) + new_user_stream.rooms.add(room_id) def _discard_if_notified(listener_set): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a5a686907..7d6df5f4c 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -64,6 +64,9 @@ class EventsStore(SQLBaseStore): except _RollbackButIsFineException: pass + max_persisted_id = yield self._stream_id_gen.get_max_token(self) + defer.returnValue((stream_ordering, max_persisted_id)) + @defer.inlineCallbacks def get_event(self, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False, diff --git a/synapse/types.py b/synapse/types.py index 0f16867d7..d89a04f7c 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -70,6 +70,8 @@ class DomainSpecificString( """Return a string encoding the fields of the structure object.""" return "%s%s:%s" % (self.SIGIL, self.localpart, self.domain) + __str__ = to_string + @classmethod def create(cls, localpart, domain,): return cls(localpart=localpart, domain=domain) @@ -107,7 +109,6 @@ class StreamToken( def from_string(cls, string): try: keys = string.split(cls._SEPARATOR) - return cls(*keys) except: raise SynapseError(400, "Invalid Token") @@ -115,6 +116,22 @@ class StreamToken( def to_string(self): return self._SEPARATOR.join([str(k) for k in self]) + @property + def room_stream_id(self): + # TODO(markjh): Awful hack to work around hacks in the presence tests + if type(self.room_key) is int: + return self.room_key + else: + return int(self.room_key[1:].split("-")[-1]) + + def is_after(self, other_token): + """Does this token contain events that the other doesn't?""" + return ( + (other_token.room_stream_id < self.room_stream_id) + or (int(other_token.presence_key) < int(self.presence_key)) + or (int(other_token.typing_key) < int(self.typing_key)) + ) + def copy_and_replace(self, key, new_value): d = self._asdict() d[key] = new_value diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 08d2404b6..f3821242b 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -83,7 +83,7 @@ class FederationTestCase(unittest.TestCase): "hashes": {"sha256":"AcLrgtUIqqwaGoHhrEvYG1YLDIsVPYJdSRGhkp3jJp8"}, }) - self.datastore.persist_event.return_value = defer.succeed(None) + self.datastore.persist_event.return_value = defer.succeed((1,1)) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) @@ -126,5 +126,5 @@ class FederationTestCase(unittest.TestCase): self.auth.check.assert_called_once_with(ANY, auth_events={}) self.notifier.on_new_room_event.assert_called_once_with( - ANY, extra_users=[] + ANY, 1, 1, extra_users=[] ) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 6417f7330..a2d763599 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -87,6 +87,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) + self.datastore.persist_event.return_value = (1,1) + @defer.inlineCallbacks def test_invite(self): room_id = "!foo:red" @@ -160,7 +162,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event, context=context, ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[UserID.from_string(target_user_id)] + event, 1, 1, extra_users=[UserID.from_string(target_user_id)] ) self.assertFalse(self.datastore.get_room.called) self.assertFalse(self.datastore.store_room.called) @@ -226,7 +228,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event, context=context ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[user] + event, 1, 1, extra_users=[user] ) join_signal_observer.assert_called_with( @@ -304,7 +306,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event, context=context ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[user] + event, 1, 1, extra_users=[user] ) leave_signal_observer.assert_called_with( diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index b318d4944..7ccbe2ea9 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -183,7 +183,7 @@ class TypingNotificationsTestCase(unittest.TestCase): ) self.on_new_user_event.assert_has_calls([ - call(rooms=[self.room_id]), + call('typing_key', 1, rooms=[self.room_id]), ]) self.assertEquals(self.event_source.get_current_key(), 1) @@ -246,7 +246,7 @@ class TypingNotificationsTestCase(unittest.TestCase): ) self.on_new_user_event.assert_has_calls([ - call(rooms=[self.room_id]), + call('typing_key', 1, rooms=[self.room_id]), ]) self.assertEquals(self.event_source.get_current_key(), 1) @@ -300,7 +300,7 @@ class TypingNotificationsTestCase(unittest.TestCase): ) self.on_new_user_event.assert_has_calls([ - call(rooms=[self.room_id]), + call('typing_key', 1, rooms=[self.room_id]), ]) yield put_json.await_calls() @@ -332,7 +332,7 @@ class TypingNotificationsTestCase(unittest.TestCase): ) self.on_new_user_event.assert_has_calls([ - call(rooms=[self.room_id]), + call('typing_key', 1, rooms=[self.room_id]), ]) self.on_new_user_event.reset_mock() @@ -352,7 +352,7 @@ class TypingNotificationsTestCase(unittest.TestCase): self.clock.advance_time(11) self.on_new_user_event.assert_has_calls([ - call(rooms=[self.room_id]), + call('typing_key', 2, rooms=[self.room_id]), ]) self.assertEquals(self.event_source.get_current_key(), 2) @@ -378,7 +378,7 @@ class TypingNotificationsTestCase(unittest.TestCase): ) self.on_new_user_event.assert_has_calls([ - call(rooms=[self.room_id]), + call('typing_key', 3, rooms=[self.room_id]), ]) self.on_new_user_event.reset_mock() diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 8e0c5fa63..c0c52796a 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -27,6 +27,7 @@ from synapse.handlers.presence import PresenceHandler from synapse.rest.client.v1 import presence from synapse.rest.client.v1 import events from synapse.types import UserID +from synapse.util.async import run_on_reactor OFFLINE = PresenceState.OFFLINE @@ -264,6 +265,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): datastore=Mock(spec=[ "set_presence_state", "get_presence_list", + "get_rooms_for_user", ]), clock=Mock(spec=[ "call_later", @@ -298,6 +300,9 @@ class PresenceEventStreamTestCase(unittest.TestCase): self.mock_datastore.get_app_service_by_user_id = Mock( return_value=defer.succeed(None) ) + self.mock_datastore.get_rooms_for_user = ( + lambda u: get_rooms_for_user(UserID.from_string(u)) + ) def get_profile_displayname(user_id): return defer.succeed("Frank") @@ -350,19 +355,19 @@ class PresenceEventStreamTestCase(unittest.TestCase): self.mock_datastore.set_presence_state.return_value = defer.succeed( {"state": ONLINE} ) - self.mock_datastore.get_presence_list.return_value = defer.succeed( - [] - ) + self.mock_datastore.get_presence_list.return_value = defer.succeed([]) yield self.presence.set_state(self.u_banana, self.u_banana, state={"presence": ONLINE} ) + yield run_on_reactor() + (code, response) = yield self.mock_resource.trigger("GET", - "/events?from=0_1_0&timeout=0", None) + "/events?from=s0_1_0&timeout=0", None) self.assertEquals(200, code) - self.assertEquals({"start": "0_1_0", "end": "0_2_0", "chunk": [ + self.assertEquals({"start": "s0_1_0", "end": "s0_2_0", "chunk": [ {"type": "m.presence", "content": { "user_id": "@banana:test", diff --git a/tests/utils.py b/tests/utils.py index cc038fecf..a67530bd6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -355,7 +355,7 @@ class MemoryDataStore(object): return [] def get_room_events_max_id(self): - return 0 # TODO (erikj) + return "s0" # TODO (erikj) def get_send_event_level(self, room_id): return defer.succeed(0) From 6edff11a888d2c3f7a6749599fcb9d4974a76bbb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 14:39:05 +0100 Subject: [PATCH 037/175] Don't fetch redaction and rejection stuff for each event, so we can use index only scan --- synapse/storage/_base.py | 23 +++++++++++++++++++---- synapse/storage/state.py | 7 +++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 0279400a8..a6fc4d6ea 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -918,10 +918,10 @@ class SQLBaseStore(object): start_time = update_counter("event_cache", start_time) sql = ( - "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " + "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " "FROM event_json as e " + "LEFT JOIN rejections as rej USING (event_id) " "LEFT JOIN redactions as r ON e.event_id = r.redacts " - "LEFT JOIN rejections as rej on rej.event_id = e.event_id " "WHERE e.event_id = ? " "LIMIT 1 " ) @@ -967,6 +967,14 @@ class SQLBaseStore(object): internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) + if rejected_reason: + rejected_reason = self._simple_select_one_onecol_txn( + txn, + table="rejections", + keyvalues={"event_id": rejected_reason}, + retcol="reason", + ) + ev = FrozenEvent( d, internal_metadata_dict=internal_metadata, @@ -977,12 +985,19 @@ class SQLBaseStore(object): if check_redacted and redacted: ev = prune_event(ev) - ev.unsigned["redacted_by"] = redacted + redaction_id = self._simple_select_one_onecol_txn( + txn, + table="redactions", + keyvalues={"redacts": ev.event_id}, + retcol="event_id", + ) + + ev.unsigned["redacted_by"] = redaction_id # Get the redaction event. because = self._get_event_txn( txn, - redacted, + redaction_id, check_redacted=False ) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 6d7d576cb..6d0ecf8dd 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -85,8 +85,10 @@ class StateStore(SQLBaseStore): def fetch_events(txn, events): sql = ( - "SELECT e.internal_metadata, e.json " + "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " " FROM event_json as e" + " LEFT JOIN rejections as rej USING (event_id)" + " LEFT JOIN redactions as r ON e.event_id = r.redacts" " WHERE e.event_id IN (%s)" ) % (",".join(["?"]*len(events)),) @@ -95,7 +97,8 @@ class StateStore(SQLBaseStore): return [ self._get_event_from_row_txn( - txn, row[0], row[1], None + txn, row[0], row[1], row[2], + rejected_reason=row[3], ) for row in rows ] From df6db5c8025e67c410a60994ac87eb39d842af30 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 13 May 2015 15:08:24 +0100 Subject: [PATCH 038/175] Don't bother checking for new events from a source if the stream token hasn't advanced for that source --- synapse/notifier.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 4d10c0503..d2fefea75 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -294,7 +294,7 @@ class Notifier(object): def _timeout_listener(): timed_out[0] = True timer[0] = None - listener[0].notify(user_stream) + listener[0].notify(from_token) # We create multiple notification listeners so we have to manage # canceling the timeout ourselves. @@ -329,11 +329,15 @@ class Notifier(object): limit = pagination_config.limit @defer.inlineCallbacks - def check_for_updates(start_token, end_token): + def check_for_updates(before_token, after_token): events = [] end_token = from_token for name, source in self.event_sources.sources.items(): keyname = "%s_key" % name + before_id = getattr(before_token, keyname) + after_id = getattr(after_token, keyname) + if before_id == after_id: + continue stuff, new_key = yield source.get_new_events_for_user( user, getattr(from_token, keyname), limit, ) From ca4f458787d4fcccf4d6b240f7497cac4e174bcc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 15:13:42 +0100 Subject: [PATCH 039/175] Fetch events in bulk --- synapse/storage/_base.py | 75 +++++++++++++++++++++++++++++++++------- synapse/storage/state.py | 22 +----------- 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a6fc4d6ea..f7b4def9e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -875,19 +875,11 @@ class SQLBaseStore(object): def _get_events_txn(self, txn, event_ids, check_redacted=True, get_prev_content=False): - if not event_ids: - return [] - - events = [ - self._get_event_txn( - txn, event_id, - check_redacted=check_redacted, - get_prev_content=get_prev_content - ) - for event_id in event_ids - ] - - return [e for e in events if e] + return self._fetch_events_txn( + txn, event_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) def _invalidate_get_event_cache(self, event_id): for check_redacted in (False, True): @@ -950,6 +942,63 @@ class SQLBaseStore(object): else: return None + def _fetch_events_txn(self, txn, events, check_redacted=True, + get_prev_content=False, allow_rejected=False): + if not events: + return [] + + event_map = {} + + for event_id in events: + try: + ret = self._get_event_cache.get(event_id, check_redacted, get_prev_content) + + if allow_rejected or not ret.rejected_reason: + event_map[event_id] = ret + else: + return None + except KeyError: + pass + + missing_events = [ + e for e in events + if e not in event_map + ] + + if missing_events: + sql = ( + "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " + " FROM event_json as e" + " LEFT JOIN rejections as rej USING (event_id)" + " LEFT JOIN redactions as r ON e.event_id = r.redacts" + " WHERE e.event_id IN (%s)" + ) % (",".join(["?"]*len(missing_events)),) + + txn.execute(sql, missing_events) + rows = txn.fetchall() + + res = [ + self._get_event_from_row_txn( + txn, row[0], row[1], row[2], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=row[3], + ) + for row in rows + ] + + event_map.update({ + e.event_id: e + for e in res if e + }) + + for e in res: + self._get_event_cache.prefill( + e.event_id, check_redacted, get_prev_content, e + ) + + return [event_map[e_id] for e_id in events if e_id in event_map] + def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False, rejected_reason=None): diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 6d0ecf8dd..a80a94743 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -83,31 +83,11 @@ class StateStore(SQLBaseStore): f, ) - def fetch_events(txn, events): - sql = ( - "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " - " FROM event_json as e" - " LEFT JOIN rejections as rej USING (event_id)" - " LEFT JOIN redactions as r ON e.event_id = r.redacts" - " WHERE e.event_id IN (%s)" - ) % (",".join(["?"]*len(events)),) - - txn.execute(sql, events) - rows = txn.fetchall() - - return [ - self._get_event_from_row_txn( - txn, row[0], row[1], row[2], - rejected_reason=row[3], - ) - for row in rows - ] - @defer.inlineCallbacks def c(vals): vals[:] = yield self.runInteraction( "_get_state_groups_ev", - fetch_events, vals + self._fetch_events_txn, vals ) yield defer.gatherResults( From 5971d240d458dbea23abed803aa3d7bf31c2efce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 15:26:49 +0100 Subject: [PATCH 040/175] Limit batch size --- synapse/storage/_base.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f7b4def9e..f169884d8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -28,6 +28,7 @@ from twisted.internet import defer from collections import namedtuple, OrderedDict import functools +import itertools import simplejson as json import sys import time @@ -875,11 +876,15 @@ class SQLBaseStore(object): def _get_events_txn(self, txn, event_ids, check_redacted=True, get_prev_content=False): - return self._fetch_events_txn( - txn, event_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - ) + N = 50 # Only fetch 100 events at a time. + return list(itertools.chain(*[ + self._fetch_events_txn( + txn, event_ids[i*N:(i+1)*N], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + for i in range(1 + len(event_ids) / N) + ])) def _invalidate_get_event_cache(self, event_id): for check_redacted in (False, True): From cf706cc6ef1ef864400fc265b1c7014e2feeabb0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 15:31:25 +0100 Subject: [PATCH 041/175] Don't return None --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f169884d8..f990b8149 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -961,7 +961,7 @@ class SQLBaseStore(object): if allow_rejected or not ret.rejected_reason: event_map[event_id] = ret else: - return None + event_map[event_id] = None except KeyError: pass From 9af432257da39385ce03c846b3c76f7fc7778ff0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 13 May 2015 15:42:13 +0100 Subject: [PATCH 042/175] Don't set a timer if there's already a result to return --- synapse/notifier.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index d2fefea75..6fcb7767a 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -82,9 +82,10 @@ class _NotifierUserStream(object): self.current_token = self.current_token.copy_and_replace( stream_key, stream_id ) - for listener in self.listeners: + listeners = self.listeners + self.listeners = set() + for listener in listeners: listener.notify(self.current_token) - self.listeners.clear() def remove(self, notifier): """ Remove this listener from all the indexes in the Notifier @@ -202,7 +203,7 @@ class Notifier(object): user_streams = room_user_streams.copy() for user in extra_users: - user_stream = self.user_to_user_stream.get(user) + user_stream = self.user_to_user_stream.get(str(user)) if user_stream is not None: user_streams.add(user_stream) @@ -288,12 +289,18 @@ class Notifier(object): timer = [None] + if result: + user_stream.listeners.discard(listener[0]) + defer.returnValue(result) + return + if timeout: timed_out = [False] def _timeout_listener(): timed_out[0] = True timer[0] = None + user_stream.listeners.discard(listener[0]) listener[0].notify(from_token) # We create multiple notification listeners so we have to manage From 8888982db3dbf839003fb544072348f73609724f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 15:43:32 +0100 Subject: [PATCH 043/175] Don't insert None --- synapse/storage/_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f990b8149..bb80108a3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1002,7 +1002,10 @@ class SQLBaseStore(object): e.event_id, check_redacted, get_prev_content, e ) - return [event_map[e_id] for e_id in events if e_id in event_map] + return [ + event_map[e_id] for e_id in events + if e_id in event_map and event_id[e_id] + ] def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False, From a988361aea45e77ed94952e16a2c96fd0cb1d362 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 15:44:15 +0100 Subject: [PATCH 044/175] Typo --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index bb80108a3..b21056f61 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -1004,7 +1004,7 @@ class SQLBaseStore(object): return [ event_map[e_id] for e_id in events - if e_id in event_map and event_id[e_id] + if e_id in event_map and event_map[e_id] ] def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, From f1b83d88a3d3ad596631e51852a9802d0a7270a0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 13 May 2015 16:54:02 +0100 Subject: [PATCH 045/175] Discard unused NotifierUserStreams --- synapse/notifier.py | 50 ++++++++++++++++++--------- tests/rest/client/v1/test_presence.py | 1 + tests/utils.py | 3 ++ 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 6fcb7767a..344dd0317 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -71,14 +71,17 @@ class _NotifierUserStream(object): so that it can remove itself from the indexes in the Notifier class. """ - def __init__(self, user, rooms, current_token, appservice=None): + def __init__(self, user, rooms, current_token, time_now_ms, + appservice=None): self.user = str(user) self.appservice = appservice self.listeners = set() self.rooms = set(rooms) self.current_token = current_token + self.last_notified_ms = time_now_ms - def notify(self, stream_key, stream_id): + def notify(self, stream_key, stream_id, time_now_ms): + self.last_notified_ms = time_now_ms self.current_token = self.current_token.copy_and_replace( stream_key, stream_id ) @@ -96,7 +99,7 @@ class _NotifierUserStream(object): lst = notifier.room_to_user_streams.get(room, set()) lst.discard(self) - notifier.user_to_user_streams.get(self.user, set()).discard(self) + notifier.user_to_user_stream.pop(self.user) if self.appservice: notifier.appservice_to_user_streams.get( @@ -111,6 +114,8 @@ class Notifier(object): Primarily used from the /events stream. """ + UNUSED_STREAM_EXPIRY_MS = 10 * 60 * 1000 + def __init__(self, hs): self.hs = hs @@ -128,6 +133,10 @@ class Notifier(object): "user_joined_room", self._user_joined_room ) + self.clock.looping_call( + self.remove_expired_streams, self.UNUSED_STREAM_EXPIRY_MS + ) + # This is not a very cheap test to perform, but it's only executed # when rendering the metrics page, which is likely once per minute at # most when scraping it. @@ -221,9 +230,12 @@ class Notifier(object): logger.debug("on_new_room_event listeners %s", user_streams) + time_now_ms = self.clock.time_msec() for user_stream in user_streams: try: - user_stream.notify("room_key", "s%d" % (room_stream_id,)) + user_stream.notify( + "room_key", "s%d" % (room_stream_id,), time_now_ms + ) except: logger.exception("Failed to notify listener") @@ -246,9 +258,10 @@ class Notifier(object): for room in rooms: user_streams |= self.room_to_user_streams.get(room, set()) + time_now_ms = self.clock.time_msec() for user_stream in user_streams: try: - user_stream.notify(stream_key, new_token) + user_stream.notify(stream_key, new_token, time_now_ms) except: logger.exception("Failed to notify listener") @@ -260,6 +273,7 @@ class Notifier(object): """ deferred = defer.Deferred() + time_now_ms = self.clock.time_msec() user = str(user) user_stream = self.user_to_user_stream.get(user) @@ -272,6 +286,7 @@ class Notifier(object): rooms=rooms, appservice=appservice, current_token=current_token, + time_now_ms=time_now_ms, ) self._register_with_keys(user_stream) else: @@ -365,6 +380,20 @@ class Notifier(object): defer.returnValue(result) + @log_function + def remove_expired_streams(self): + time_now_ms = self.clock.time_msec() + expired_streams = [] + expire_before_ts = time_now_ms - self.UNUSED_STREAM_EXPIRY_MS + for stream in self.user_to_user_stream.values(): + if stream.listeners: + continue + if stream.last_notified_ms < expire_before_ts: + expired_streams.append(stream) + + for expired_stream in expired_streams: + expired_stream.remove(self) + @log_function def _register_with_keys(self, user_stream): self.user_to_user_stream[user_stream.user] = user_stream @@ -385,14 +414,3 @@ class Notifier(object): room_streams = self.room_to_user_streams.setdefault(room_id, set()) room_streams.add(new_user_stream) new_user_stream.rooms.add(room_id) - - -def _discard_if_notified(listener_set): - """Remove any 'stale' listeners from the given set. - """ - to_discard = set() - for l in listener_set: - if l.notified(): - to_discard.add(l) - - listener_set -= to_discard diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index c0c52796a..29c0038f0 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -271,6 +271,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): "call_later", "cancel_call_later", "time_msec", + "looping_call", ]), ) diff --git a/tests/utils.py b/tests/utils.py index a67530bd6..3b5c33591 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -197,6 +197,9 @@ class MockClock(object): return t + def looping_call(self, function, interval): + pass + def cancel_call_later(self, timer): if timer[2]: raise Exception("Cannot cancel an expired timer") From 4071f2965320950c7f1bbdd39105f8c34ca95034 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 16:59:41 +0100 Subject: [PATCH 046/175] Fetch events from events_id in their own transactions --- synapse/storage/_base.py | 154 +++++++++++++++++++++++++++++++++++++- synapse/storage/state.py | 10 +-- synapse/storage/stream.py | 22 +++--- 3 files changed, 168 insertions(+), 18 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b21056f61..f6c1ec424 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -867,11 +867,26 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) + @defer.inlineCallbacks def _get_events(self, event_ids, check_redacted=True, get_prev_content=False, desc="_get_events"): - return self.runInteraction( - desc, self._get_events_txn, event_ids, - check_redacted=check_redacted, get_prev_content=get_prev_content, + N = 50 # Only fetch 100 events at a time. + + ds = [ + self.runInteraction( + desc, + self._fetch_events_txn, + event_ids[i*N:(i+1)*N], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + for i in range(1 + len(event_ids) / N) + ] + + res = yield defer.gatherResults(ds, consumeErrors=True) + + defer.returnValue( + list(itertools.chain(*res)) ) def _get_events_txn(self, txn, event_ids, check_redacted=True, @@ -1007,6 +1022,139 @@ class SQLBaseStore(object): if e_id in event_map and event_map[e_id] ] + @defer.inlineCallbacks + def _fetch_events(self, events, check_redacted=True, + get_prev_content=False, allow_rejected=False): + if not events: + defer.returnValue([]) + + event_map = {} + + for event_id in events: + try: + ret = self._get_event_cache.get(event_id, check_redacted, get_prev_content) + + if allow_rejected or not ret.rejected_reason: + event_map[event_id] = ret + else: + event_map[event_id] = None + except KeyError: + pass + + missing_events = [ + e for e in events + if e not in event_map + ] + + if missing_events: + sql = ( + "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " + " FROM event_json as e" + " LEFT JOIN rejections as rej USING (event_id)" + " LEFT JOIN redactions as r ON e.event_id = r.redacts" + " WHERE e.event_id IN (%s)" + ) % (",".join(["?"]*len(missing_events)),) + + rows = yield self._execute( + "_fetch_events", + None, + sql, + *missing_events + ) + + res_ds = [ + self._get_event_from_row( + row[0], row[1], row[2], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=row[3], + ) + for row in rows + ] + + res = yield defer.gatherResults(res_ds, consumeErrors=True) + + event_map.update({ + e.event_id: e + for e in res if e + }) + + for e in res: + self._get_event_cache.prefill( + e.event_id, check_redacted, get_prev_content, e + ) + + defer.returnValue([ + event_map[e_id] for e_id in events + if e_id in event_map and event_map[e_id] + ]) + + @defer.inlineCallbacks + def _get_event_from_row(self, internal_metadata, js, redacted, + check_redacted=True, get_prev_content=False, + rejected_reason=None): + + start_time = time.time() * 1000 + + def update_counter(desc, last_time): + curr_time = self._get_event_counters.update(desc, last_time) + sql_getevents_timer.inc_by(curr_time - last_time, desc) + return curr_time + + d = json.loads(js) + start_time = update_counter("decode_json", start_time) + + internal_metadata = json.loads(internal_metadata) + start_time = update_counter("decode_internal", start_time) + + if rejected_reason: + rejected_reason = yield self._simple_select_one_onecol( + desc="_get_event_from_row", + table="rejections", + keyvalues={"event_id": rejected_reason}, + retcol="reason", + ) + + ev = FrozenEvent( + d, + internal_metadata_dict=internal_metadata, + rejected_reason=rejected_reason, + ) + start_time = update_counter("build_frozen_event", start_time) + + if check_redacted and redacted: + ev = prune_event(ev) + + redaction_id = yield self._simple_select_one_onecol( + desc="_get_event_from_row", + table="redactions", + keyvalues={"redacts": ev.event_id}, + retcol="event_id", + ) + + ev.unsigned["redacted_by"] = redaction_id + # Get the redaction event. + + because = yield self.get_event_txn( + redaction_id, + check_redacted=False + ) + + if because: + ev.unsigned["redacted_because"] = because + start_time = update_counter("redact_event", start_time) + + if get_prev_content and "replaces_state" in ev.unsigned: + prev = yield self.get_event( + ev.unsigned["replaces_state"], + get_prev_content=False, + ) + if prev: + ev.unsigned["prev_content"] = prev.get_dict()["content"] + start_time = update_counter("get_prev_content", start_time) + + defer.returnValue(ev) + def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False, rejected_reason=None): diff --git a/synapse/storage/state.py b/synapse/storage/state.py index a80a94743..483b316e9 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -85,13 +85,13 @@ class StateStore(SQLBaseStore): @defer.inlineCallbacks def c(vals): - vals[:] = yield self.runInteraction( - "_get_state_groups_ev", - self._fetch_events_txn, vals - ) + vals[:] = yield self._fetch_events(vals, get_prev_content=False) yield defer.gatherResults( - [c(vals) for vals in states.values()], + [ + c(vals) + for vals in states.values() + ], consumeErrors=True, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8045e17fd..db9c2f038 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -298,6 +298,7 @@ class StreamStore(SQLBaseStore): return self.runInteraction("paginate_room_events", f) + @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, end_token, with_feedback=False, from_token=None): # TODO (erikj): Handle compressed feedback @@ -349,20 +350,21 @@ class StreamStore(SQLBaseStore): else: token = (str(end_token), str(end_token)) - events = self._get_events_txn( - txn, - [r["event_id"] for r in rows], - get_prev_content=True - ) + return rows, token - self._set_before_and_after(events, rows) - - return events, token - - return self.runInteraction( + rows, token = yield self.runInteraction( "get_recent_events_for_room", get_recent_events_for_room_txn ) + events = yield self._get_events( + [r["event_id"] for r in rows], + get_prev_content=True + ) + + self._set_before_and_after(events, rows) + + defer.returnValue((events, token)) + @defer.inlineCallbacks def get_room_events_max_id(self, direction='f'): token = yield self._stream_id_gen.get_max_token(self) From 968b01a91a759e4137dbbd5e2537abefe3b1ccbe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 17:02:46 +0100 Subject: [PATCH 047/175] Actually use async method --- synapse/storage/_base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f6c1ec424..0ba12d48a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -873,9 +873,7 @@ class SQLBaseStore(object): N = 50 # Only fetch 100 events at a time. ds = [ - self.runInteraction( - desc, - self._fetch_events_txn, + self._fetch_events( event_ids[i*N:(i+1)*N], check_redacted=check_redacted, get_prev_content=get_prev_content, From 5e0c533672f63990f7fdcf0b2c3cce15c3682d78 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 13 May 2015 17:20:28 +0100 Subject: [PATCH 048/175] Fix metric counter --- synapse/notifier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 344dd0317..1f7f0a143 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -145,8 +145,8 @@ class Notifier(object): for x in self.room_to_user_streams.values(): all_user_streams |= x - for x in self.user_to_user_streams.values(): - all_user_streams |= x + for x in self.user_to_user_stream: + all_user_streams.add(x) for x in self.appservice_to_user_streams.values(): all_user_streams |= x From 4f1d984e56e795a35a9bdb691e58f2b6f90e25f9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 May 2015 17:22:26 +0100 Subject: [PATCH 049/175] Add index on events --- synapse/storage/__init__.py | 2 +- .../storage/schema/delta/19/event_index.sql | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/19/event_index.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7cb91a0be..75af44d78 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 18 +SCHEMA_VERSION = 19 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/19/event_index.sql b/synapse/storage/schema/delta/19/event_index.sql new file mode 100644 index 000000000..f3792817b --- /dev/null +++ b/synapse/storage/schema/delta/19/event_index.sql @@ -0,0 +1,19 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE INDEX events_order_topo_stream_room ON events( + topological_ordering, stream_ordering, room_id +); \ No newline at end of file From 47fb089eb534ded5e4d72d725dc5dd436ed80c00 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 May 2015 10:22:18 +0100 Subject: [PATCH 050/175] Specify python 2.7 in the virtualenv setup (SYN-319) #resolved --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5a6e10563..259fbaf45 100644 --- a/README.rst +++ b/README.rst @@ -117,7 +117,7 @@ Installing prerequisites on Mac OS X:: To install the synapse homeserver run:: - $ virtualenv ~/.synapse + $ virtualenv -p python2.7 ~/.synapse $ source ~/.synapse/bin/activate $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master From 3edd2d5c93ccbec46f101e65c6c7874a90bf0018 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 14 May 2015 11:25:30 +0100 Subject: [PATCH 051/175] Fix v2 sync, update the last_notified_ms only if there was an active listener --- synapse/handlers/sync.py | 2 +- synapse/notifier.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 35a62fda4..bd8c60368 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -92,7 +92,7 @@ class SyncHandler(BaseHandler): result = yield self.current_sync_for_user(sync_config, since_token) defer.returnValue(result) else: - def current_sync_callback(): + def current_sync_callback(before_token, after_token): return self.current_sync_for_user(sync_config, since_token) rm_handler = self.hs.get_handlers().room_member_handler diff --git a/synapse/notifier.py b/synapse/notifier.py index 1f7f0a143..2de7dca8a 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -81,14 +81,15 @@ class _NotifierUserStream(object): self.last_notified_ms = time_now_ms def notify(self, stream_key, stream_id, time_now_ms): - self.last_notified_ms = time_now_ms self.current_token = self.current_token.copy_and_replace( stream_key, stream_id ) - listeners = self.listeners - self.listeners = set() - for listener in listeners: - listener.notify(self.current_token) + if self.listeners: + self.last_notified_ms = time_now_ms + listeners = self.listeners + self.listeners = set() + for listener in listeners: + listener.notify(self.current_token) def remove(self, notifier): """ Remove this listener from all the indexes in the Notifier From 7c549dd5572b2881ee61cb11c572a82662414cdb Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 May 2015 11:44:03 +0100 Subject: [PATCH 052/175] Add ID generator for push_rules_enable to #resolve SYN-378 --- synapse/storage/_base.py | 1 + synapse/storage/push_rule.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c9fe5a355..81052409b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -309,6 +309,7 @@ class SQLBaseStore(object): self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) self._pushers_id_gen = IdGenerator("pushers", "id", self) self._push_rule_id_gen = IdGenerator("push_rules", "id", self) + self._push_rules_enable_id_gen = IdGenerator("push_rules_enable", "id", self) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 34805e276..e7988676c 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -204,11 +204,21 @@ class PushRuleStore(SQLBaseStore): @defer.inlineCallbacks def set_push_rule_enabled(self, user_name, rule_id, enabled): - yield self._simple_upsert( + ret = yield self.runInteraction( + "_set_push_rule_enabled_txn", + self._set_push_rule_enabled_txn, + user_name, rule_id, enabled + ) + defer.returnValue(ret) + + def _set_push_rule_enabled_txn(self, txn, user_name, rule_id, enabled): + new_id = self._push_rules_enable_id_gen.get_next_txn(txn) + self._simple_upsert_txn( + txn, PushRuleEnableTable.table_name, {'user_name': user_name, 'rule_id': rule_id}, {'enabled': 1 if enabled else 0}, - desc="set_push_rule_enabled", + {'id': new_id}, ) From c37a6e151ff551bb4ec243d5d237761c2ed6b914 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 May 2015 12:03:13 +0100 Subject: [PATCH 053/175] Make shared secret registration work again --- synapse/rest/client/v2_alpha/register.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 3640fb4a2..72dfb876c 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -82,8 +82,10 @@ class RegisterRestServlet(RestServlet): [LoginType.EMAIL_IDENTITY] ] + result = None if service: is_application_server = True + params = body elif 'mac' in body: # Check registration-specific shared secret auth if 'username' not in body: @@ -92,6 +94,7 @@ class RegisterRestServlet(RestServlet): body['username'], body['mac'] ) is_using_shared_secret = True + params = body else: authed, result, params = yield self.auth_handler.check_auth( flows, body, self.hs.get_ip_from_request(request) @@ -118,7 +121,7 @@ class RegisterRestServlet(RestServlet): password=new_password ) - if LoginType.EMAIL_IDENTITY in result: + if result and LoginType.EMAIL_IDENTITY in result: threepid = result[LoginType.EMAIL_IDENTITY] for reqd in ['medium', 'address', 'validated_at']: From 084c365c3ad7f3a30ecfccca6cc79fb2e0c652ad Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 14 May 2015 12:03:26 +0100 Subject: [PATCH 054/175] Use the current token when timing out a notifier, make sure the user_id is a string in on_new_user_event --- synapse/notifier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 2de7dca8a..3dbd6f984 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -252,7 +252,7 @@ class Notifier(object): user_streams = set() for user in users: - user_stream = self.user_to_user_stream.get(user) + user_stream = self.user_to_user_stream.get(str(user)) if user_stream is not None: user_streams.add(user_stream) @@ -317,7 +317,7 @@ class Notifier(object): timed_out[0] = True timer[0] = None user_stream.listeners.discard(listener[0]) - listener[0].notify(from_token) + listener[0].notify(current_token) # We create multiple notification listeners so we have to manage # canceling the timeout ourselves. From 0c894e1ebdb204cf0a0dce16ed819b9e5d9f3fc0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 May 2015 13:11:28 +0100 Subject: [PATCH 055/175] Throw error when creating room if alias contains whitespace #SYN-335 --- synapse/handlers/room.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index dac683616..401cc677d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -26,6 +26,7 @@ from synapse.util.async import run_on_reactor from synapse.events.utils import serialize_event import logging +import string logger = logging.getLogger(__name__) @@ -50,6 +51,10 @@ class RoomCreationHandler(BaseHandler): self.ratelimit(user_id) if "room_alias_name" in config: + for wchar in string.whitespace: + if wchar in config["room_alias_name"]: + raise SynapseError(400, "Invalid characters in room alias") + room_alias = RoomAlias.create( config["room_alias_name"], self.hs.hostname, From 92e1c8983dcbc1b9e75ff71b06928fd51627f61a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 May 2015 13:21:55 +0100 Subject: [PATCH 056/175] Disallow whitespace in aliases here too --- synapse/handlers/directory.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index f76febee8..e41a68883 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -22,6 +22,7 @@ from synapse.api.constants import EventTypes from synapse.types import RoomAlias import logging +import string logger = logging.getLogger(__name__) @@ -40,6 +41,10 @@ class DirectoryHandler(BaseHandler): def _create_association(self, room_alias, room_id, servers=None): # general association creation for both human users and app services + for wchar in string.whitespace: + if wchar in room_alias.localpart: + raise SynapseError(400, "Invalid characters in room alias") + if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") # TODO(erikj): Change this. From cdb3757942fefdcdc3d33b9c6d7c9e44decefd6f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 13:31:55 +0100 Subject: [PATCH 057/175] Refactor _get_events --- synapse/storage/_base.py | 362 +++++++++++---------------------------- synapse/storage/state.py | 2 +- synapse/util/__init__.py | 28 +++ 3 files changed, 131 insertions(+), 261 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 46a1c0746..a6c6676f8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -17,6 +17,7 @@ import logging from synapse.api.errors import StoreError from synapse.events import FrozenEvent from synapse.events.utils import prune_event +from synapse.util import unwrap_deferred from synapse.util.logutils import log_function from synapse.util.logcontext import preserve_context_over_fn, LoggingContext from synapse.util.lrucache import LruCache @@ -28,7 +29,6 @@ from twisted.internet import defer from collections import namedtuple, OrderedDict import functools -import itertools import simplejson as json import sys import time @@ -870,35 +870,43 @@ class SQLBaseStore(object): @defer.inlineCallbacks def _get_events(self, event_ids, check_redacted=True, - get_prev_content=False, desc="_get_events"): - N = 50 # Only fetch 100 events at a time. + get_prev_content=False, allow_rejected=False, txn=None): + if not event_ids: + defer.returnValue([]) - ds = [ - self._fetch_events( - event_ids[i*N:(i+1)*N], - check_redacted=check_redacted, - get_prev_content=get_prev_content, - ) - for i in range(1 + len(event_ids) / N) - ] - - res = yield defer.gatherResults(ds, consumeErrors=True) - - defer.returnValue( - list(itertools.chain(*res)) + event_map = self._get_events_from_cache( + event_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, ) + missing_events = [e for e in event_ids if e not in event_map] + + missing_events = yield self._fetch_events( + txn, + missing_events, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + event_map.update(missing_events) + + defer.returnValue([ + event_map[e_id] for e_id in event_ids + if e_id in event_map and event_map[e_id] + ]) + def _get_events_txn(self, txn, event_ids, check_redacted=True, - get_prev_content=False): - N = 50 # Only fetch 100 events at a time. - return list(itertools.chain(*[ - self._fetch_events_txn( - txn, event_ids[i*N:(i+1)*N], - check_redacted=check_redacted, - get_prev_content=get_prev_content, - ) - for i in range(1 + len(event_ids) / N) - ])) + get_prev_content=False, allow_rejected=False): + return unwrap_deferred(self._get_events( + event_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + txn=txn, + )) def _invalidate_get_event_cache(self, event_id): for check_redacted in (False, True): @@ -909,68 +917,24 @@ class SQLBaseStore(object): def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): - start_time = time.time() * 1000 - - def update_counter(desc, last_time): - curr_time = self._get_event_counters.update(desc, last_time) - sql_getevents_timer.inc_by(curr_time - last_time, desc) - return curr_time - - try: - ret = self._get_event_cache.get(event_id, check_redacted, get_prev_content) - - if allow_rejected or not ret.rejected_reason: - return ret - else: - return None - except KeyError: - pass - finally: - start_time = update_counter("event_cache", start_time) - - sql = ( - "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " - "FROM event_json as e " - "LEFT JOIN rejections as rej USING (event_id) " - "LEFT JOIN redactions as r ON e.event_id = r.redacts " - "WHERE e.event_id = ? " - "LIMIT 1 " - ) - - txn.execute(sql, (event_id,)) - - res = txn.fetchone() - - if not res: - return None - - internal_metadata, js, redacted, rejected_reason = res - - start_time = update_counter("select_event", start_time) - - result = self._get_event_from_row_txn( - txn, internal_metadata, js, redacted, + events = self._get_events_txn( + txn, [event_id], check_redacted=check_redacted, get_prev_content=get_prev_content, - rejected_reason=rejected_reason, + allow_rejected=allow_rejected, ) - self._get_event_cache.prefill(event_id, check_redacted, get_prev_content, result) - if allow_rejected or not rejected_reason: - return result - else: - return None - - def _fetch_events_txn(self, txn, events, check_redacted=True, - get_prev_content=False, allow_rejected=False): - if not events: - return [] + return events[0] if events else None + def _get_events_from_cache(self, events, check_redacted, get_prev_content, + allow_rejected): event_map = {} for event_id in events: try: - ret = self._get_event_cache.get(event_id, check_redacted, get_prev_content) + ret = self._get_event_cache.get( + event_id, check_redacted, get_prev_content + ) if allow_rejected or not ret.rejected_reason: event_map[event_id] = ret @@ -979,136 +943,82 @@ class SQLBaseStore(object): except KeyError: pass - missing_events = [ - e for e in events - if e not in event_map - ] - - if missing_events: - sql = ( - "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " - " FROM event_json as e" - " LEFT JOIN rejections as rej USING (event_id)" - " LEFT JOIN redactions as r ON e.event_id = r.redacts" - " WHERE e.event_id IN (%s)" - ) % (",".join(["?"]*len(missing_events)),) - - txn.execute(sql, missing_events) - rows = txn.fetchall() - - res = [ - self._get_event_from_row_txn( - txn, row[0], row[1], row[2], - check_redacted=check_redacted, - get_prev_content=get_prev_content, - rejected_reason=row[3], - ) - for row in rows - ] - - event_map.update({ - e.event_id: e - for e in res if e - }) - - for e in res: - self._get_event_cache.prefill( - e.event_id, check_redacted, get_prev_content, e - ) - - return [ - event_map[e_id] for e_id in events - if e_id in event_map and event_map[e_id] - ] + return event_map @defer.inlineCallbacks - def _fetch_events(self, events, check_redacted=True, + def _fetch_events(self, txn, events, check_redacted=True, get_prev_content=False, allow_rejected=False): if not events: - defer.returnValue([]) + defer.returnValue({}) - event_map = {} + rows = [] + N = 2 + for i in range(1 + len(events) / N): + evs = events[i*N:(i + 1)*N] + if not evs: + break - for event_id in events: - try: - ret = self._get_event_cache.get(event_id, check_redacted, get_prev_content) - - if allow_rejected or not ret.rejected_reason: - event_map[event_id] = ret - else: - event_map[event_id] = None - except KeyError: - pass - - missing_events = [ - e for e in events - if e not in event_map - ] - - if missing_events: sql = ( "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " " FROM event_json as e" " LEFT JOIN rejections as rej USING (event_id)" " LEFT JOIN redactions as r ON e.event_id = r.redacts" " WHERE e.event_id IN (%s)" - ) % (",".join(["?"]*len(missing_events)),) + ) % (",".join(["?"]*len(evs)),) - rows = yield self._execute( - "_fetch_events", - None, - sql, - *missing_events + if txn: + txn.execute(sql, evs) + rows.extend(txn.fetchall()) + else: + res = yield self._execute("_fetch_events", None, sql, *evs) + rows.extend(res) + + res = [] + for row in rows: + e = yield self._get_event_from_row( + txn, + row[0], row[1], row[2], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=row[3], + ) + res.append(e) + + for e in res: + self._get_event_cache.prefill( + e.event_id, check_redacted, get_prev_content, e ) - res_ds = [ - self._get_event_from_row( - row[0], row[1], row[2], - check_redacted=check_redacted, - get_prev_content=get_prev_content, - rejected_reason=row[3], - ) - for row in rows - ] - - res = yield defer.gatherResults(res_ds, consumeErrors=True) - - event_map.update({ - e.event_id: e - for e in res if e - }) - - for e in res: - self._get_event_cache.prefill( - e.event_id, check_redacted, get_prev_content, e - ) - - defer.returnValue([ - event_map[e_id] for e_id in events - if e_id in event_map and event_map[e_id] - ]) + defer.returnValue({ + e.event_id: e + for e in res if e + }) @defer.inlineCallbacks - def _get_event_from_row(self, internal_metadata, js, redacted, + def _get_event_from_row(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False, rejected_reason=None): - - start_time = time.time() * 1000 - - def update_counter(desc, last_time): - curr_time = self._get_event_counters.update(desc, last_time) - sql_getevents_timer.inc_by(curr_time - last_time, desc) - return curr_time - d = json.loads(js) - start_time = update_counter("decode_json", start_time) - internal_metadata = json.loads(internal_metadata) - start_time = update_counter("decode_internal", start_time) + + def select(txn, *args, **kwargs): + if txn: + return self._simple_select_one_onecol_txn(txn, *args, **kwargs) + else: + return self._simple_select_one_onecol( + *args, + desc="_get_event_from_row", **kwargs + ) + + def get_event(txn, *args, **kwargs): + if txn: + return self._get_event_txn(txn, *args, **kwargs) + else: + return self.get_event(*args, **kwargs) if rejected_reason: - rejected_reason = yield self._simple_select_one_onecol( - desc="_get_event_from_row", + rejected_reason = yield select( + txn, table="rejections", keyvalues={"event_id": rejected_reason}, retcol="reason", @@ -1119,13 +1029,12 @@ class SQLBaseStore(object): internal_metadata_dict=internal_metadata, rejected_reason=rejected_reason, ) - start_time = update_counter("build_frozen_event", start_time) if check_redacted and redacted: ev = prune_event(ev) - redaction_id = yield self._simple_select_one_onecol( - desc="_get_event_from_row", + redaction_id = yield select( + txn, table="redactions", keyvalues={"redacts": ev.event_id}, retcol="event_id", @@ -1134,93 +1043,26 @@ class SQLBaseStore(object): ev.unsigned["redacted_by"] = redaction_id # Get the redaction event. - because = yield self.get_event_txn( + because = yield get_event( + txn, redaction_id, check_redacted=False ) if because: ev.unsigned["redacted_because"] = because - start_time = update_counter("redact_event", start_time) if get_prev_content and "replaces_state" in ev.unsigned: - prev = yield self.get_event( + prev = yield get_event( + txn, ev.unsigned["replaces_state"], get_prev_content=False, ) if prev: ev.unsigned["prev_content"] = prev.get_dict()["content"] - start_time = update_counter("get_prev_content", start_time) defer.returnValue(ev) - def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, - check_redacted=True, get_prev_content=False, - rejected_reason=None): - - start_time = time.time() * 1000 - - def update_counter(desc, last_time): - curr_time = self._get_event_counters.update(desc, last_time) - sql_getevents_timer.inc_by(curr_time - last_time, desc) - return curr_time - - d = json.loads(js) - start_time = update_counter("decode_json", start_time) - - internal_metadata = json.loads(internal_metadata) - start_time = update_counter("decode_internal", start_time) - - if rejected_reason: - rejected_reason = self._simple_select_one_onecol_txn( - txn, - table="rejections", - keyvalues={"event_id": rejected_reason}, - retcol="reason", - ) - - ev = FrozenEvent( - d, - internal_metadata_dict=internal_metadata, - rejected_reason=rejected_reason, - ) - start_time = update_counter("build_frozen_event", start_time) - - if check_redacted and redacted: - ev = prune_event(ev) - - redaction_id = self._simple_select_one_onecol_txn( - txn, - table="redactions", - keyvalues={"redacts": ev.event_id}, - retcol="event_id", - ) - - ev.unsigned["redacted_by"] = redaction_id - # Get the redaction event. - - because = self._get_event_txn( - txn, - redaction_id, - check_redacted=False - ) - - if because: - ev.unsigned["redacted_because"] = because - start_time = update_counter("redact_event", start_time) - - if get_prev_content and "replaces_state" in ev.unsigned: - prev = self._get_event_txn( - txn, - ev.unsigned["replaces_state"], - get_prev_content=False, - ) - if prev: - ev.unsigned["prev_content"] = prev.get_dict()["content"] - start_time = update_counter("get_prev_content", start_time) - - return ev - def _parse_events(self, rows): return self.runInteraction( "_parse_events", self._parse_events_txn, rows diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 483b316e9..26fd3b3e6 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -85,7 +85,7 @@ class StateStore(SQLBaseStore): @defer.inlineCallbacks def c(vals): - vals[:] = yield self._fetch_events(vals, get_prev_content=False) + vals[:] = yield self._get_events(vals, get_prev_content=False) yield defer.gatherResults( [ diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index c1a16b639..b9afb3364 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -29,6 +29,34 @@ def unwrapFirstError(failure): return failure.value.subFailure +def unwrap_deferred(d): + """Given a deferred that we know has completed, return its value or raise + the failure as an exception + """ + if not d.called: + raise RuntimeError("deferred has not finished") + + res = [] + + def f(r): + res.append(r) + return r + d.addCallback(f) + + if res: + return res[0] + + def f(r): + res.append(r) + return r + d.addErrback(f) + + if res: + res[0].raiseException() + else: + raise RuntimeError("deferred did not call callbacks") + + class Clock(object): """A small utility that obtains current time-of-day so that time may be mocked during unit-tests. From f6f902d459c0f888b70742b8f7cca640e544adf6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 13:45:48 +0100 Subject: [PATCH 058/175] Move fetching of events into their own transactions --- synapse/storage/event_federation.py | 38 +++++++++++++--------------- synapse/storage/roommember.py | 39 +++++++++++++---------------- synapse/storage/state.py | 2 -- synapse/storage/stream.py | 19 +++++++------- 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index a1982dfbb..5d4b7843f 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + from ._base import SQLBaseStore, cached from syutil.base64util import encode_base64 @@ -33,16 +35,7 @@ class EventFederationStore(SQLBaseStore): """ def get_auth_chain(self, event_ids): - return self.runInteraction( - "get_auth_chain", - self._get_auth_chain_txn, - event_ids - ) - - def _get_auth_chain_txn(self, txn, event_ids): - results = self._get_auth_chain_ids_txn(txn, event_ids) - - return self._get_events_txn(txn, results) + return self.get_auth_chain_ids(event_ids).addCallback(self._get_events) def get_auth_chain_ids(self, event_ids): return self.runInteraction( @@ -369,7 +362,7 @@ class EventFederationStore(SQLBaseStore): return self.runInteraction( "get_backfill_events", self._get_backfill_events, room_id, event_list, limit - ) + ).addCallback(self._get_events) def _get_backfill_events(self, txn, room_id, event_list, limit): logger.debug( @@ -415,16 +408,26 @@ class EventFederationStore(SQLBaseStore): front = new_front event_results += new_front - return self._get_events_txn(txn, event_results) + return event_results + @defer.inlineCallbacks def get_missing_events(self, room_id, earliest_events, latest_events, limit, min_depth): - return self.runInteraction( + ids = yield self.runInteraction( "get_missing_events", self._get_missing_events, room_id, earliest_events, latest_events, limit, min_depth ) + events = yield self._get_events(ids) + + events = sorted( + [ev for ev in events if ev.depth >= min_depth], + key=lambda e: e.depth, + ) + + defer.returnValue(events[:limit]) + def _get_missing_events(self, txn, room_id, earliest_events, latest_events, limit, min_depth): @@ -456,14 +459,7 @@ class EventFederationStore(SQLBaseStore): front = new_front event_results |= new_front - events = self._get_events_txn(txn, event_results) - - events = sorted( - [ev for ev in events if ev.depth >= min_depth], - key=lambda e: e.depth, - ) - - return events[:limit] + return event_results def clean_room_for_join(self, room_id): return self.runInteraction( diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 839c74f63..80717f6cd 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -76,16 +76,16 @@ class RoomMemberStore(SQLBaseStore): Returns: Deferred: Results in a MembershipEvent or None. """ - def f(txn): - events = self._get_members_events_txn( - txn, - room_id, - user_id=user_id, - ) - - return events[0] if events else None - - return self.runInteraction("get_room_member", f) + return self.runInteraction( + "get_room_member", + self._get_members_events_txn, + room_id, + user_id=user_id, + ).addCallback( + self._get_events + ).addCallback( + lambda events: events[0] if events else None + ) def get_users_in_room(self, room_id): def f(txn): @@ -110,15 +110,12 @@ class RoomMemberStore(SQLBaseStore): Returns: list of namedtuples representing the members in this room. """ - - def f(txn): - return self._get_members_events_txn( - txn, - room_id, - membership=membership, - ) - - return self.runInteraction("get_room_members", f) + return self.runInteraction( + "get_room_members", + self._get_members_events_txn, + room_id, + membership=membership, + ).addCallback(self._get_events) def get_rooms_for_user_where_membership_is(self, user_id, membership_list): """ Get all the rooms for this user where the membership for this user @@ -190,14 +187,14 @@ class RoomMemberStore(SQLBaseStore): return self.runInteraction( "get_members_query", self._get_members_events_txn, where_clause, where_values - ) + ).addCallbacks(self._get_events) def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None): rows = self._get_members_rows_txn( txn, room_id, membership, user_id, ) - return self._get_events_txn(txn, [r["event_id"] for r in rows]) + return [r["event_id"] for r in rows] def _get_members_rows_txn(self, txn, room_id, membership=None, user_id=None): where_clause = "c.room_id = ?" diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 26fd3b3e6..3f5642642 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -72,8 +72,6 @@ class StateStore(SQLBaseStore): retcol="event_id", ) - # state = self._get_events_txn(txn, state_ids) - res[group] = state_ids return res diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index db9c2f038..d16b57c51 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -224,7 +224,7 @@ class StreamStore(SQLBaseStore): return self.runInteraction("get_room_events_stream", f) - @log_function + @defer.inlineCallbacks def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, with_feedback=False): @@ -286,17 +286,18 @@ class StreamStore(SQLBaseStore): # TODO (erikj): We should work out what to do here instead. next_token = to_key if to_key else from_key - events = self._get_events_txn( - txn, - [r["event_id"] for r in rows], - get_prev_content=True - ) + return rows, next_token, - self._set_before_and_after(events, rows) + rows, token = yield self.runInteraction("paginate_room_events", f) - return events, next_token, + events = yield self._get_events( + [r["event_id"] for r in rows], + get_prev_content=True + ) - return self.runInteraction("paginate_room_events", f) + self._set_before_and_after(events, rows) + + defer.returnValue((events, token)) @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, end_token, From ab78a8926e508a56ea8a4719f80e6402493ff9e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 13:47:16 +0100 Subject: [PATCH 059/175] Err, we probably want a bigger limit --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a6c6676f8..5c33bd81a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -952,7 +952,7 @@ class SQLBaseStore(object): defer.returnValue({}) rows = [] - N = 2 + N = 200 for i in range(1 + len(events) / N): evs = events[i*N:(i + 1)*N] if not evs: From e1e9f0c5b23e1404ba885b501b6a347346a119ea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 13:58:49 +0100 Subject: [PATCH 060/175] loop -> gatherResults --- synapse/storage/_base.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5c33bd81a..015e04a8c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -973,16 +973,20 @@ class SQLBaseStore(object): res = yield self._execute("_fetch_events", None, sql, *evs) rows.extend(res) - res = [] - for row in rows: - e = yield self._get_event_from_row( - txn, - row[0], row[1], row[2], - check_redacted=check_redacted, - get_prev_content=get_prev_content, - rejected_reason=row[3], - ) - res.append(e) + res = yield defer.gatherResults( + [ + defer.maybeDeferred( + self._get_event_from_row, + txn, + row[0], row[1], row[2], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=row[3], + ) + for row in rows + ], + consumeErrors=True, + ) for e in res: self._get_event_cache.prefill( From 2f7f8e1c2b10b9444a3de7777d3bcc9d19f9d6c8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 14:17:36 +0100 Subject: [PATCH 061/175] Preemptively jump into a transaction if we ask for get_prev_content --- synapse/storage/_base.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 015e04a8c..d896f5f91 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -883,20 +883,30 @@ class SQLBaseStore(object): missing_events = [e for e in event_ids if e not in event_map] - missing_events = yield self._fetch_events( - txn, - missing_events, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) + def get_missing(txn=None): + missing_events = yield self._fetch_events( + txn, + missing_events, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) - event_map.update(missing_events) + event_map.update(missing_events) + + defer.returnValue([ + event_map[e_id] for e_id in event_ids + if e_id in event_map and event_map[e_id] + ]) + + if missing_events and get_prev_content and not txn: + if get_prev_content and not txn: + # If we want prev_content then lets just jump into a txn. + res = yield self.runInteraction("_get_events", get_missing) + defer.returnValue(res) + + defer.returnValue(get_missing()) - defer.returnValue([ - event_map[e_id] for e_id in event_ids - if e_id in event_map and event_map[e_id] - ]) def _get_events_txn(self, txn, event_ids, check_redacted=True, get_prev_content=False, allow_rejected=False): From 67800f7626d07202b19da9090432aab4b0bc1aef Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 May 2015 14:19:10 +0100 Subject: [PATCH 062/175] Treat setting your display name to the empty string as removing it (SYN-186). --- synapse/handlers/profile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 71ff78ab2..799faffe5 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -88,6 +88,9 @@ class ProfileHandler(BaseHandler): if target_user != auth_user: raise AuthError(400, "Cannot set another user's displayname") + if new_displayname == '': + new_displayname = None + yield self.store.set_profile_displayname( target_user.localpart, new_displayname ) From 656223fbd3a87b278b8101175e0ac117a059a812 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 14:26:35 +0100 Subject: [PATCH 063/175] Actually, we probably want to run this in a transaction --- synapse/storage/_base.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d896f5f91..e44821b5e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -881,32 +881,29 @@ class SQLBaseStore(object): allow_rejected=allow_rejected, ) - missing_events = [e for e in event_ids if e not in event_map] + missing_events_ids = [e for e in event_ids if e not in event_map] - def get_missing(txn=None): - missing_events = yield self._fetch_events( + def get_missing(txn): + missing_events = unwrap_deferred(self._fetch_events( txn, - missing_events, + missing_events_ids, check_redacted=check_redacted, get_prev_content=get_prev_content, allow_rejected=allow_rejected, - ) + )) event_map.update(missing_events) - defer.returnValue([ + return [ event_map[e_id] for e_id in event_ids if e_id in event_map and event_map[e_id] - ]) - - if missing_events and get_prev_content and not txn: - if get_prev_content and not txn: - # If we want prev_content then lets just jump into a txn. - res = yield self.runInteraction("_get_events", get_missing) - defer.returnValue(res) - - defer.returnValue(get_missing()) + ] + if not txn: + res = yield self.runInteraction("_get_events", get_missing) + defer.returnValue(res) + else: + defer.returnValue(get_missing(txn)) def _get_events_txn(self, txn, event_ids, check_redacted=True, get_prev_content=False, allow_rejected=False): From 7d6a1dae31c96379f29bc9e4191d2c02f1dad640 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 14:27:58 +0100 Subject: [PATCH 064/175] Jump out early --- synapse/storage/_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e44821b5e..1a76b22e4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -883,6 +883,12 @@ class SQLBaseStore(object): missing_events_ids = [e for e in event_ids if e not in event_map] + if not missing_events_ids: + defer.returnValue([ + event_map[e_id] for e_id in event_ids + if e_id in event_map and event_map[e_id] + ]) + def get_missing(txn): missing_events = unwrap_deferred(self._fetch_events( txn, From 0ad1c672344e09a2a09690a1ab1bb61fe0397293 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 14 May 2015 14:35:07 +0100 Subject: [PATCH 065/175] Add some doc-strings to notifier --- synapse/notifier.py | 50 +++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 3dbd6f984..862b42cfc 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -81,6 +81,13 @@ class _NotifierUserStream(object): self.last_notified_ms = time_now_ms def notify(self, stream_key, stream_id, time_now_ms): + """Notify any listeners for this user of a new event from an + event source. + Args: + stream_key(str): The stream the event came from. + stream_id(str): The new id for the stream the event came from. + time_now_ms(int): The current time in milliseconds. + """ self.current_token = self.current_token.copy_and_replace( stream_key, stream_id ) @@ -167,17 +174,6 @@ class Notifier(object): lambda: count(bool, self.appservice_to_user_streams.values()), ) - def notify_pending_new_room_events(self, max_room_stream_id): - pending = sorted(self.pending_new_room_events) - self.pending_new_room_events = [] - for event, room_stream_id, extra_users in pending: - if room_stream_id > max_room_stream_id: - self.pending_new_room_events.append(( - event, room_stream_id, extra_users - )) - else: - self._on_new_room_event(event, room_stream_id, extra_users) - @log_function @defer.inlineCallbacks def on_new_room_event(self, event, room_stream_id, max_room_stream_id, @@ -188,19 +184,37 @@ class Notifier(object): This triggers the notifier to wake up any listeners that are listening to the room, and any listeners for the users in the `extra_users` param. + + The events can be peristed out of order. The notifier will wait + until all previous events have been persisted before notifying + the client streams. """ yield run_on_reactor() - self.notify_pending_new_room_events(max_room_stream_id) + self.pending_new_room_events.append(( + event, room_stream_id, extra_users + )) + self._notify_pending_new_room_events(max_room_stream_id) - if room_stream_id > max_room_stream_id: - self.pending_new_room_events.append(( - event, room_stream_id, extra_users - )) - else: - self._on_new_room_event(event, room_stream_id, extra_users) + def _notify_pending_new_room_events(self, max_room_stream_id): + """Notify for the room events that were queued waiting for a previous + event to be persisted. + Args: + max_room_stream_id(int): The highest stream_id below which all + events have been persisted. + """ + pending = sorted(self.pending_new_room_events) + self.pending_new_room_events = [] + for event, room_stream_id, extra_users in pending: + if room_stream_id > max_room_stream_id: + self.pending_new_room_events.append(( + event, room_stream_id, extra_users + )) + else: + self._on_new_room_event(event, room_stream_id, extra_users) def _on_new_room_event(self, event, room_stream_id, extra_users=[]): + """Notify any user streams that are interested in this room event""" # poke any interested application service. self.hs.get_handlers().appservice_handler.notify_interested_services( event From 386b7330d2902fe8acac0efadb095be389700764 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 14:45:22 +0100 Subject: [PATCH 066/175] Move from _base to events --- synapse/storage/_base.py | 233 +----------------------------------- synapse/storage/events.py | 246 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+), 232 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1a76b22e4..a8f8989e3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -15,9 +15,7 @@ import logging from synapse.api.errors import StoreError -from synapse.events import FrozenEvent -from synapse.events.utils import prune_event -from synapse.util import unwrap_deferred + from synapse.util.logutils import log_function from synapse.util.logcontext import preserve_context_over_fn, LoggingContext from synapse.util.lrucache import LruCache @@ -29,7 +27,6 @@ from twisted.internet import defer from collections import namedtuple, OrderedDict import functools -import simplejson as json import sys import time import threading @@ -868,234 +865,6 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) - @defer.inlineCallbacks - def _get_events(self, event_ids, check_redacted=True, - get_prev_content=False, allow_rejected=False, txn=None): - if not event_ids: - defer.returnValue([]) - - event_map = self._get_events_from_cache( - event_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) - - missing_events_ids = [e for e in event_ids if e not in event_map] - - if not missing_events_ids: - defer.returnValue([ - event_map[e_id] for e_id in event_ids - if e_id in event_map and event_map[e_id] - ]) - - def get_missing(txn): - missing_events = unwrap_deferred(self._fetch_events( - txn, - missing_events_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - )) - - event_map.update(missing_events) - - return [ - event_map[e_id] for e_id in event_ids - if e_id in event_map and event_map[e_id] - ] - - if not txn: - res = yield self.runInteraction("_get_events", get_missing) - defer.returnValue(res) - else: - defer.returnValue(get_missing(txn)) - - def _get_events_txn(self, txn, event_ids, check_redacted=True, - get_prev_content=False, allow_rejected=False): - return unwrap_deferred(self._get_events( - event_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - txn=txn, - )) - - def _invalidate_get_event_cache(self, event_id): - for check_redacted in (False, True): - for get_prev_content in (False, True): - self._get_event_cache.invalidate(event_id, check_redacted, - get_prev_content) - - def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=False, allow_rejected=False): - - events = self._get_events_txn( - txn, [event_id], - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) - - return events[0] if events else None - - def _get_events_from_cache(self, events, check_redacted, get_prev_content, - allow_rejected): - event_map = {} - - for event_id in events: - try: - ret = self._get_event_cache.get( - event_id, check_redacted, get_prev_content - ) - - if allow_rejected or not ret.rejected_reason: - event_map[event_id] = ret - else: - event_map[event_id] = None - except KeyError: - pass - - return event_map - - @defer.inlineCallbacks - def _fetch_events(self, txn, events, check_redacted=True, - get_prev_content=False, allow_rejected=False): - if not events: - defer.returnValue({}) - - rows = [] - N = 200 - for i in range(1 + len(events) / N): - evs = events[i*N:(i + 1)*N] - if not evs: - break - - sql = ( - "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " - " FROM event_json as e" - " LEFT JOIN rejections as rej USING (event_id)" - " LEFT JOIN redactions as r ON e.event_id = r.redacts" - " WHERE e.event_id IN (%s)" - ) % (",".join(["?"]*len(evs)),) - - if txn: - txn.execute(sql, evs) - rows.extend(txn.fetchall()) - else: - res = yield self._execute("_fetch_events", None, sql, *evs) - rows.extend(res) - - res = yield defer.gatherResults( - [ - defer.maybeDeferred( - self._get_event_from_row, - txn, - row[0], row[1], row[2], - check_redacted=check_redacted, - get_prev_content=get_prev_content, - rejected_reason=row[3], - ) - for row in rows - ], - consumeErrors=True, - ) - - for e in res: - self._get_event_cache.prefill( - e.event_id, check_redacted, get_prev_content, e - ) - - defer.returnValue({ - e.event_id: e - for e in res if e - }) - - @defer.inlineCallbacks - def _get_event_from_row(self, txn, internal_metadata, js, redacted, - check_redacted=True, get_prev_content=False, - rejected_reason=None): - d = json.loads(js) - internal_metadata = json.loads(internal_metadata) - - def select(txn, *args, **kwargs): - if txn: - return self._simple_select_one_onecol_txn(txn, *args, **kwargs) - else: - return self._simple_select_one_onecol( - *args, - desc="_get_event_from_row", **kwargs - ) - - def get_event(txn, *args, **kwargs): - if txn: - return self._get_event_txn(txn, *args, **kwargs) - else: - return self.get_event(*args, **kwargs) - - if rejected_reason: - rejected_reason = yield select( - txn, - table="rejections", - keyvalues={"event_id": rejected_reason}, - retcol="reason", - ) - - ev = FrozenEvent( - d, - internal_metadata_dict=internal_metadata, - rejected_reason=rejected_reason, - ) - - if check_redacted and redacted: - ev = prune_event(ev) - - redaction_id = yield select( - txn, - table="redactions", - keyvalues={"redacts": ev.event_id}, - retcol="event_id", - ) - - ev.unsigned["redacted_by"] = redaction_id - # Get the redaction event. - - because = yield get_event( - txn, - redaction_id, - check_redacted=False - ) - - if because: - ev.unsigned["redacted_because"] = because - - if get_prev_content and "replaces_state" in ev.unsigned: - prev = yield get_event( - txn, - ev.unsigned["replaces_state"], - get_prev_content=False, - ) - if prev: - ev.unsigned["prev_content"] = prev.get_dict()["content"] - - defer.returnValue(ev) - - def _parse_events(self, rows): - return self.runInteraction( - "_parse_events", self._parse_events_txn, rows - ) - - def _parse_events_txn(self, txn, rows): - event_ids = [r["event_id"] for r in rows] - - return self._get_events_txn(txn, event_ids) - - def _has_been_redacted_txn(self, txn, event): - sql = "SELECT event_id FROM redactions WHERE redacts = ?" - txn.execute(sql, (event.event_id,)) - result = txn.fetchone() - return result[0] if result else None - def get_next_stream_id(self): with self._next_stream_id_lock: i = self._next_stream_id diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 9242b0a84..f960ef835 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -17,6 +17,10 @@ from _base import SQLBaseStore, _RollbackButIsFineException from twisted.internet import defer +from synapse.events import FrozenEvent +from synapse.events.utils import prune_event +from synapse.util import unwrap_deferred + from synapse.util.logutils import log_function from synapse.api.constants import EventTypes from synapse.crypto.event_signing import compute_event_reference_hash @@ -26,6 +30,7 @@ from syutil.jsonutil import encode_canonical_json from contextlib import contextmanager import logging +import simplejson as json logger = logging.getLogger(__name__) @@ -393,3 +398,244 @@ class EventsStore(SQLBaseStore): return self.runInteraction( "have_events", f, ) + + @defer.inlineCallbacks + def _get_events(self, event_ids, check_redacted=True, + get_prev_content=False, allow_rejected=False, txn=None): + if not event_ids: + defer.returnValue([]) + + event_map = self._get_events_from_cache( + event_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + missing_events_ids = [e for e in event_ids if e not in event_map] + + if not missing_events_ids: + defer.returnValue([ + event_map[e_id] for e_id in event_ids + if e_id in event_map and event_map[e_id] + ]) + + if not txn: + missing_events = yield self.runInteraction( + "_get_events", + self._fetch_events_txn, + missing_events_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + else: + missing_events = yield self._fetch_events( + txn, + missing_events_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + event_map.update(missing_events) + + defer.returnValue([ + event_map[e_id] for e_id in event_ids + if e_id in event_map and event_map[e_id] + ]) + + + def _get_events_txn(self, txn, event_ids, check_redacted=True, + get_prev_content=False, allow_rejected=False): + return unwrap_deferred(self._get_events( + event_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + txn=txn, + )) + + def _invalidate_get_event_cache(self, event_id): + for check_redacted in (False, True): + for get_prev_content in (False, True): + self._get_event_cache.invalidate(event_id, check_redacted, + get_prev_content) + + def _get_event_txn(self, txn, event_id, check_redacted=True, + get_prev_content=False, allow_rejected=False): + + events = self._get_events_txn( + txn, [event_id], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + return events[0] if events else None + + def _get_events_from_cache(self, events, check_redacted, get_prev_content, + allow_rejected): + event_map = {} + + for event_id in events: + try: + ret = self._get_event_cache.get( + event_id, check_redacted, get_prev_content + ) + + if allow_rejected or not ret.rejected_reason: + event_map[event_id] = ret + else: + event_map[event_id] = None + except KeyError: + pass + + return event_map + + def _fetch_events_txn(self, txn, events, check_redacted=True, + get_prev_content=False, allow_rejected=False): + return unwrap_deferred(self._fetch_events( + txn, events, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + )) + + @defer.inlineCallbacks + def _fetch_events(self, txn, events, check_redacted=True, + get_prev_content=False, allow_rejected=False): + if not events: + defer.returnValue({}) + + rows = [] + N = 200 + for i in range(1 + len(events) / N): + evs = events[i*N:(i + 1)*N] + if not evs: + break + + sql = ( + "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " + " FROM event_json as e" + " LEFT JOIN rejections as rej USING (event_id)" + " LEFT JOIN redactions as r ON e.event_id = r.redacts" + " WHERE e.event_id IN (%s)" + ) % (",".join(["?"]*len(evs)),) + + if txn: + txn.execute(sql, evs) + rows.extend(txn.fetchall()) + else: + res = yield self._execute("_fetch_events", None, sql, *evs) + rows.extend(res) + + res = yield defer.gatherResults( + [ + defer.maybeDeferred( + self._get_event_from_row, + txn, + row[0], row[1], row[2], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=row[3], + ) + for row in rows + ], + consumeErrors=True, + ) + + for e in res: + self._get_event_cache.prefill( + e.event_id, check_redacted, get_prev_content, e + ) + + defer.returnValue({ + e.event_id: e + for e in res if e + }) + + @defer.inlineCallbacks + def _get_event_from_row(self, txn, internal_metadata, js, redacted, + check_redacted=True, get_prev_content=False, + rejected_reason=None): + d = json.loads(js) + internal_metadata = json.loads(internal_metadata) + + def select(txn, *args, **kwargs): + if txn: + return self._simple_select_one_onecol_txn(txn, *args, **kwargs) + else: + return self._simple_select_one_onecol( + *args, + desc="_get_event_from_row", **kwargs + ) + + def get_event(txn, *args, **kwargs): + if txn: + return self._get_event_txn(txn, *args, **kwargs) + else: + return self.get_event(*args, **kwargs) + + if rejected_reason: + rejected_reason = yield select( + txn, + table="rejections", + keyvalues={"event_id": rejected_reason}, + retcol="reason", + ) + + ev = FrozenEvent( + d, + internal_metadata_dict=internal_metadata, + rejected_reason=rejected_reason, + ) + + if check_redacted and redacted: + ev = prune_event(ev) + + redaction_id = yield select( + txn, + table="redactions", + keyvalues={"redacts": ev.event_id}, + retcol="event_id", + ) + + ev.unsigned["redacted_by"] = redaction_id + # Get the redaction event. + + because = yield get_event( + txn, + redaction_id, + check_redacted=False + ) + + if because: + ev.unsigned["redacted_because"] = because + + if get_prev_content and "replaces_state" in ev.unsigned: + prev = yield get_event( + txn, + ev.unsigned["replaces_state"], + get_prev_content=False, + ) + if prev: + ev.unsigned["prev_content"] = prev.get_dict()["content"] + + defer.returnValue(ev) + + def _parse_events(self, rows): + return self.runInteraction( + "_parse_events", self._parse_events_txn, rows + ) + + def _parse_events_txn(self, txn, rows): + event_ids = [r["event_id"] for r in rows] + + return self._get_events_txn(txn, event_ids) + + def _has_been_redacted_txn(self, txn, event): + sql = "SELECT event_id FROM redactions WHERE redacts = ?" + txn.execute(sql, (event.event_id,)) + result = txn.fetchone() + return result[0] if result else None From f4d58deba19bde81b629c8ceb8df3aca60aa1e15 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 14:45:42 +0100 Subject: [PATCH 067/175] PEP8 --- synapse/storage/events.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f960ef835..28d2c0896 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -445,7 +445,6 @@ class EventsStore(SQLBaseStore): if e_id in event_map and event_map[e_id] ]) - def _get_events_txn(self, txn, event_ids, check_redacted=True, get_prev_content=False, allow_rejected=False): return unwrap_deferred(self._get_events( @@ -494,7 +493,7 @@ class EventsStore(SQLBaseStore): return event_map def _fetch_events_txn(self, txn, events, check_redacted=True, - get_prev_content=False, allow_rejected=False): + get_prev_content=False, allow_rejected=False): return unwrap_deferred(self._fetch_events( txn, events, check_redacted=check_redacted, From 7f4105a5c99d72662db0704ab03bff24a951005b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 14:51:06 +0100 Subject: [PATCH 068/175] Turn off preemptive transactions --- synapse/storage/events.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 28d2c0896..0aa4e0d44 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -420,23 +420,13 @@ class EventsStore(SQLBaseStore): if e_id in event_map and event_map[e_id] ]) - if not txn: - missing_events = yield self.runInteraction( - "_get_events", - self._fetch_events_txn, - missing_events_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) - else: - missing_events = yield self._fetch_events( - txn, - missing_events_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) + missing_events = yield self._fetch_events( + txn, + missing_events_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) event_map.update(missing_events) From c5d1b4986bbb5983054b64fdc3dd3c32e80e3c17 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 14 May 2015 14:59:31 +0100 Subject: [PATCH 069/175] Remove unused arguments and doc PresenceHandler.push_update_to_clients --- synapse/handlers/presence.py | 20 ++++++--------- tests/handlers/test_presence.py | 22 ++++------------ tests/handlers/test_presencelike.py | 39 ++++++----------------------- 3 files changed, 21 insertions(+), 60 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 1edab0549..0c246958a 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -496,9 +496,7 @@ class PresenceHandler(BaseHandler): # We want to tell the person that just came online # presence state of people they are interested in? self.push_update_to_clients( - observed_user=target_user, users_to_push=[user], - statuscache=self._get_or_offline_usercache(target_user), ) deferreds = [] @@ -712,10 +710,7 @@ class PresenceHandler(BaseHandler): continue self.push_update_to_clients( - observed_user=user, - users_to_push=observers, - room_ids=room_ids, - statuscache=statuscache, + users_to_push=observers, room_ids=room_ids ) user_id = user.to_string() @@ -779,10 +774,7 @@ class PresenceHandler(BaseHandler): localusers = set(localusers) self.push_update_to_clients( - observed_user=observed_user, - users_to_push=localusers, - room_ids=room_ids, - statuscache=statuscache, + users_to_push=localusers, room_ids=room_ids ) remote_domains = set(remote_domains) @@ -807,8 +799,12 @@ class PresenceHandler(BaseHandler): defer.returnValue((localusers, remote_domains)) - def push_update_to_clients(self, observed_user, users_to_push=[], - room_ids=[], statuscache=None): + def push_update_to_clients(self, users_to_push=[], room_ids=[]): + """Notify clients of a new presence event. + Args: + users_to_push(list): List of users to notify. + room_ids(list): List of room_ids to notify. + """ with PreserveLoggingContext(): self.notifier.on_new_user_event( users_to_push, diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 70147b017..ee773797e 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -1097,12 +1097,8 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase): # apple should see both banana and clementine currently offline self.mock_update_client.assert_has_calls([ - call(users_to_push=[self.u_apple], - observed_user=self.u_banana, - statuscache=ANY), - call(users_to_push=[self.u_apple], - observed_user=self.u_clementine, - statuscache=ANY), + call(users_to_push=[self.u_apple]), + call(users_to_push=[self.u_apple]), ], any_order=True) # Gut-wrenching tests @@ -1121,13 +1117,8 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase): # apple and banana should now both see each other online self.mock_update_client.assert_has_calls([ - call(users_to_push=set([self.u_apple]), - observed_user=self.u_banana, - room_ids=[], - statuscache=ANY), - call(users_to_push=[self.u_banana], - observed_user=self.u_apple, - statuscache=ANY), + call(users_to_push=set([self.u_apple]), room_ids=[]), + call(users_to_push=[self.u_banana]), ], any_order=True) self.assertTrue("apple" in self.handler._local_pushmap) @@ -1143,10 +1134,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase): # banana should now be told apple is offline self.mock_update_client.assert_has_calls([ - call(users_to_push=set([self.u_banana, self.u_apple]), - observed_user=self.u_apple, - room_ids=[], - statuscache=ANY), + call(users_to_push=set([self.u_banana, self.u_apple]), room_ids=[]), ], any_order=True) self.assertFalse("banana" in self.handler._local_pushmap) diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 977e832da..1f2e66ac1 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -209,20 +209,12 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): ], presence) self.mock_update_client.assert_has_calls([ - call(users_to_push=set([self.u_apple, self.u_banana, self.u_clementine]), - room_ids=[], - observed_user=self.u_apple, - statuscache=ANY), # self-reflection + call( + users_to_push={self.u_apple, self.u_banana, self.u_clementine}, + room_ids=[] + ), ], any_order=True) - statuscache = self.mock_update_client.call_args[1]["statuscache"] - self.assertEquals({ - "presence": ONLINE, - "last_active": 1000000, # MockClock - "displayname": "Frank", - "avatar_url": "http://foo", - }, statuscache.state) - self.mock_update_client.reset_mock() self.datastore.set_profile_displayname.return_value = defer.succeed( @@ -232,21 +224,12 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): self.u_apple, "I am an Apple") self.mock_update_client.assert_has_calls([ - call(users_to_push=set([self.u_apple, self.u_banana, self.u_clementine]), + call( + users_to_push={self.u_apple, self.u_banana, self.u_clementine}, room_ids=[], - observed_user=self.u_apple, - statuscache=ANY), # self-reflection + ), ], any_order=True) - statuscache = self.mock_update_client.call_args[1]["statuscache"] - self.assertEquals({ - "presence": ONLINE, - "last_active": 1000000, # MockClock - "displayname": "I am an Apple", - "avatar_url": "http://foo", - }, statuscache.state) - - @defer.inlineCallbacks def test_push_remote(self): self.presence_list = [ @@ -314,13 +297,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): self.mock_update_client.assert_called_with( users_to_push=set([self.u_apple]), room_ids=[], - observed_user=self.u_potato, - statuscache=ANY) - - statuscache = self.mock_update_client.call_args[1]["statuscache"] - self.assertEquals({"presence": ONLINE, - "displayname": "Frank", - "avatar_url": "http://foo"}, statuscache.state) + ) state = yield self.handlers.presence_handler.get_state(self.u_potato, self.u_apple) From 7cd6a6f6cf03e5bfde99b839bdc67f8f5513cdc5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 15:34:02 +0100 Subject: [PATCH 070/175] Awful idea for speeding up fetching of events --- synapse/storage/_base.py | 4 + synapse/storage/events.py | 167 ++++++++++++++++++++++++++++++-------- synapse/util/__init__.py | 8 +- 3 files changed, 139 insertions(+), 40 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a8f8989e3..c20ff3a57 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -299,6 +299,10 @@ class SQLBaseStore(object): self._get_event_cache = Cache("*getEvent*", keylen=3, lru=True, max_entries=hs.config.event_cache_size) + self._event_fetch_lock = threading.Lock() + self._event_fetch_list = [] + self._event_fetch_ongoing = False + self.database_engine = hs.database_engine self._stream_id_gen = StreamIdGenerator() diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0aa4e0d44..be88328ce 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -15,7 +15,7 @@ from _base import SQLBaseStore, _RollbackButIsFineException -from twisted.internet import defer +from twisted.internet import defer, reactor from synapse.events import FrozenEvent from synapse.events.utils import prune_event @@ -89,18 +89,17 @@ class EventsStore(SQLBaseStore): Returns: Deferred : A FrozenEvent. """ - event = yield self.runInteraction( - "get_event", self._get_event_txn, - event_id, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, + events = yield self._get_events( + [event_id], + check_redacted=True, + get_prev_content=False, + allow_rejected=False, ) - if not event and not allow_none: + if not events and not allow_none: raise RuntimeError("Could not find event %s" % (event_id,)) - defer.returnValue(event) + defer.returnValue(events[0] if events else None) @log_function def _persist_event_txn(self, txn, event, context, backfilled, @@ -420,13 +419,21 @@ class EventsStore(SQLBaseStore): if e_id in event_map and event_map[e_id] ]) - missing_events = yield self._fetch_events( - txn, - missing_events_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) + if not txn: + missing_events = yield self._enqueue_events( + missing_events_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + else: + missing_events = self._fetch_events_txn( + txn, + missing_events_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) event_map.update(missing_events) @@ -492,11 +499,82 @@ class EventsStore(SQLBaseStore): )) @defer.inlineCallbacks - def _fetch_events(self, txn, events, check_redacted=True, - get_prev_content=False, allow_rejected=False): + def _enqueue_events(self, events, check_redacted=True, + get_prev_content=False, allow_rejected=False): if not events: defer.returnValue({}) + def do_fetch(txn): + event_list = [] + try: + with self._event_fetch_lock: + event_list = self._event_fetch_list + self._event_fetch_list = [] + + if not event_list: + return + + event_id_lists = zip(*event_list)[0] + event_ids = [ + item for sublist in event_id_lists for item in sublist + ] + rows = self._fetch_event_rows(txn, event_ids) + + row_dict = { + r["event_id"]: r + for r in rows + } + + for ids, d in event_list: + d.callback( + [ + row_dict[i] for i in ids + if i in row_dict + ] + ) + except Exception as e: + for _, d in event_list: + try: + reactor.callFromThread(d.errback, e) + except: + pass + finally: + with self._event_fetch_lock: + self._event_fetch_ongoing = False + + def cb(rows): + return defer.gatherResults([ + self._get_event_from_row( + None, + row["internal_metadata"], row["json"], row["redacts"], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=row["rejects"], + ) + for row in rows + ]) + + d = defer.Deferred() + d.addCallback(cb) + with self._event_fetch_lock: + self._event_fetch_list.append( + (events, d) + ) + + if not self._event_fetch_ongoing: + self.runInteraction( + "do_fetch", + do_fetch + ) + + res = yield d + + defer.returnValue({ + e.event_id: e + for e in res if e + }) + + def _fetch_event_rows(self, txn, events): rows = [] N = 200 for i in range(1 + len(events) / N): @@ -505,43 +583,56 @@ class EventsStore(SQLBaseStore): break sql = ( - "SELECT e.internal_metadata, e.json, r.redacts, rej.event_id " + "SELECT " + " e.event_id as event_id, " + " e.internal_metadata," + " e.json," + " r.redacts as redacts," + " rej.event_id as rejects " " FROM event_json as e" " LEFT JOIN rejections as rej USING (event_id)" " LEFT JOIN redactions as r ON e.event_id = r.redacts" " WHERE e.event_id IN (%s)" ) % (",".join(["?"]*len(evs)),) - if txn: - txn.execute(sql, evs) - rows.extend(txn.fetchall()) - else: - res = yield self._execute("_fetch_events", None, sql, *evs) - rows.extend(res) + txn.execute(sql, evs) + rows.extend(self.cursor_to_dict(txn)) + + return rows + + @defer.inlineCallbacks + def _fetch_events(self, txn, events, check_redacted=True, + get_prev_content=False, allow_rejected=False): + if not events: + defer.returnValue({}) + + if txn: + rows = self._fetch_event_rows( + txn, events, + ) + else: + rows = yield self.runInteraction( + self._fetch_event_rows, + events, + ) res = yield defer.gatherResults( [ defer.maybeDeferred( self._get_event_from_row, txn, - row[0], row[1], row[2], + row["internal_metadata"], row["json"], row["redacts"], check_redacted=check_redacted, get_prev_content=get_prev_content, - rejected_reason=row[3], + rejected_reason=row["rejects"], ) for row in rows - ], - consumeErrors=True, + ] ) - for e in res: - self._get_event_cache.prefill( - e.event_id, check_redacted, get_prev_content, e - ) - defer.returnValue({ - e.event_id: e - for e in res if e + r.event_id: r + for r in res }) @defer.inlineCallbacks @@ -611,6 +702,10 @@ class EventsStore(SQLBaseStore): if prev: ev.unsigned["prev_content"] = prev.get_dict()["content"] + self._get_event_cache.prefill( + ev.event_id, check_redacted, get_prev_content, ev + ) + defer.returnValue(ev) def _parse_events(self, rows): diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index b9afb3364..260714ccc 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -80,16 +80,16 @@ class Clock(object): def stop_looping_call(self, loop): loop.stop() - def call_later(self, delay, callback): + def call_later(self, delay, callback, *args, **kwargs): current_context = LoggingContext.current_context() - def wrapped_callback(): + def wrapped_callback(*args, **kwargs): with PreserveLoggingContext(): LoggingContext.thread_local.current_context = current_context - callback() + callback(*args, **kwargs) with PreserveLoggingContext(): - return reactor.callLater(delay, wrapped_callback) + return reactor.callLater(delay, wrapped_callback, *args, **kwargs) def cancel_call_later(self, timer): timer.cancel() From 96c5b9f87cd5d959d0976487399d4b913082598d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 15:36:04 +0100 Subject: [PATCH 071/175] Don't start up more fetch_events --- synapse/storage/events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index be88328ce..0859518b1 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -566,6 +566,7 @@ class EventsStore(SQLBaseStore): "do_fetch", do_fetch ) + self._event_fetch_ongoing = True res = yield d From 142934084a374c3e47b63939526827f5afa7410d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 15:40:21 +0100 Subject: [PATCH 072/175] Count and loop --- synapse/storage/_base.py | 2 +- synapse/storage/events.py | 64 +++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c20ff3a57..97bf42469 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -301,7 +301,7 @@ class SQLBaseStore(object): self._event_fetch_lock = threading.Lock() self._event_fetch_list = [] - self._event_fetch_ongoing = False + self._event_fetch_ongoing = 0 self.database_engine = hs.database_engine diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0859518b1..a6b2e7677 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -506,41 +506,39 @@ class EventsStore(SQLBaseStore): def do_fetch(txn): event_list = [] - try: - with self._event_fetch_lock: - event_list = self._event_fetch_list - self._event_fetch_list = [] + while True: + try: + with self._event_fetch_lock: + event_list = self._event_fetch_list + self._event_fetch_list = [] - if not event_list: - return + if not event_list: + return - event_id_lists = zip(*event_list)[0] - event_ids = [ - item for sublist in event_id_lists for item in sublist - ] - rows = self._fetch_event_rows(txn, event_ids) + event_id_lists = zip(*event_list)[0] + event_ids = [ + item for sublist in event_id_lists for item in sublist + ] + rows = self._fetch_event_rows(txn, event_ids) - row_dict = { - r["event_id"]: r - for r in rows - } + row_dict = { + r["event_id"]: r + for r in rows + } - for ids, d in event_list: - d.callback( - [ - row_dict[i] for i in ids - if i in row_dict - ] - ) - except Exception as e: - for _, d in event_list: - try: - reactor.callFromThread(d.errback, e) - except: - pass - finally: - with self._event_fetch_lock: - self._event_fetch_ongoing = False + for ids, d in event_list: + d.callback( + [ + row_dict[i] for i in ids + if i in row_dict + ] + ) + except Exception as e: + for _, d in event_list: + try: + reactor.callFromThread(d.errback, e) + except: + pass def cb(rows): return defer.gatherResults([ @@ -561,12 +559,12 @@ class EventsStore(SQLBaseStore): (events, d) ) - if not self._event_fetch_ongoing: + if self._event_fetch_ongoing < 3: + self._event_fetch_ongoing += 1 self.runInteraction( "do_fetch", do_fetch ) - self._event_fetch_ongoing = True res = yield d From ef3d8754f51e38375dc8d5144d2224d5e86ce458 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 15:41:55 +0100 Subject: [PATCH 073/175] Call from right thread --- synapse/storage/events.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a6b2e7677..59af21a2c 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -513,6 +513,7 @@ class EventsStore(SQLBaseStore): self._event_fetch_list = [] if not event_list: + self._event_fetch_ongoing -= 1 return event_id_lists = zip(*event_list)[0] @@ -527,7 +528,8 @@ class EventsStore(SQLBaseStore): } for ids, d in event_list: - d.callback( + reactor.callFromThread( + d.callback, [ row_dict[i] for i in ids if i in row_dict From 6e1ad283cf671e0ff1b04856a2f711643900c436 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 14 May 2015 16:39:19 +0100 Subject: [PATCH 074/175] Support gzip encoding for client, client v2 and web client resources (SYN-176). --- synapse/app/homeserver.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index c22726519..dfb5314ff 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -32,9 +32,9 @@ from synapse.server import HomeServer from twisted.internet import reactor from twisted.application import service from twisted.enterprise import adbapi -from twisted.web.resource import Resource +from twisted.web.resource import Resource, EncodingResourceWrapper from twisted.web.static import File -from twisted.web.server import Site +from twisted.web.server import Site, GzipEncoderFactory from twisted.web.http import proxiedLogFormatter, combinedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource @@ -69,16 +69,26 @@ import subprocess logger = logging.getLogger("synapse.app.homeserver") +class GzipFile(File): + def getChild(self, path, request): + child = File.getChild(self, path, request) + return EncodingResourceWrapper(child, [GzipEncoderFactory()]) + + +def gz_wrap(r): + return EncodingResourceWrapper(r, [GzipEncoderFactory()]) + + class SynapseHomeServer(HomeServer): def build_http_client(self): return MatrixFederationHttpClient(self) def build_resource_for_client(self): - return ClientV1RestResource(self) + return gz_wrap(ClientV1RestResource(self)) def build_resource_for_client_v2_alpha(self): - return ClientV2AlphaRestResource(self) + return gz_wrap(ClientV2AlphaRestResource(self)) def build_resource_for_federation(self): return JsonResource(self) @@ -87,9 +97,10 @@ class SynapseHomeServer(HomeServer): import syweb syweb_path = os.path.dirname(syweb.__file__) webclient_path = os.path.join(syweb_path, "webclient") - return File(webclient_path) # TODO configurable? + return GzipFile(webclient_path) # TODO configurable? def build_resource_for_static_content(self): + # This is old and should go away: not going to bother adding gzip return File("static") def build_resource_for_content_repo(self): From 1d566edb81e1dffea026d4e603a12cee664a8eda Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 May 2015 16:54:35 +0100 Subject: [PATCH 075/175] Remove race condition --- synapse/storage/_base.py | 176 ++++++++++++++++++---------- synapse/storage/engines/postgres.py | 2 + synapse/storage/engines/sqlite3.py | 2 + synapse/storage/events.py | 81 +++++++------ 4 files changed, 161 insertions(+), 100 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 97bf42469..ceff99c16 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -26,6 +26,8 @@ from util.id_generators import IdGenerator, StreamIdGenerator from twisted.internet import defer from collections import namedtuple, OrderedDict + +import contextlib import functools import sys import time @@ -299,7 +301,7 @@ class SQLBaseStore(object): self._get_event_cache = Cache("*getEvent*", keylen=3, lru=True, max_entries=hs.config.event_cache_size) - self._event_fetch_lock = threading.Lock() + self._event_fetch_lock = threading.Condition() self._event_fetch_list = [] self._event_fetch_ongoing = 0 @@ -342,6 +344,84 @@ class SQLBaseStore(object): self._clock.looping_call(loop, 10000) + @contextlib.contextmanager + def _new_transaction(self, conn, desc, after_callbacks): + start = time.time() * 1000 + txn_id = self._TXN_ID + + # We don't really need these to be unique, so lets stop it from + # growing really large. + self._TXN_ID = (self._TXN_ID + 1) % (sys.maxint - 1) + + name = "%s-%x" % (desc, txn_id, ) + + transaction_logger.debug("[TXN START] {%s}", name) + + try: + i = 0 + N = 5 + while True: + try: + txn = conn.cursor() + txn = LoggingTransaction( + txn, name, self.database_engine, after_callbacks + ) + except self.database_engine.module.OperationalError as e: + # This can happen if the database disappears mid + # transaction. + logger.warn( + "[TXN OPERROR] {%s} %s %d/%d", + name, e, i, N + ) + if i < N: + i += 1 + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) + continue + raise + except self.database_engine.module.DatabaseError as e: + if self.database_engine.is_deadlock(e): + logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) + if i < N: + i += 1 + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) + continue + raise + + try: + yield txn + conn.commit() + return + except: + try: + conn.rollback() + except: + pass + raise + except Exception as e: + logger.debug("[TXN FAIL] {%s} %s", name, e) + raise + finally: + end = time.time() * 1000 + duration = end - start + + transaction_logger.debug("[TXN END] {%s} %f", name, duration) + + self._current_txn_total_time += duration + self._txn_perf_counters.update(desc, start, end) + sql_txn_timer.inc_by(duration, desc) + @defer.inlineCallbacks def runInteraction(self, desc, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" @@ -353,75 +433,15 @@ class SQLBaseStore(object): def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: + sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) + if self.database_engine.is_connection_closed(conn): logger.debug("Reconnecting closed database connection") conn.reconnect() current_context.copy_to(context) - start = time.time() * 1000 - txn_id = self._TXN_ID - - # We don't really need these to be unique, so lets stop it from - # growing really large. - self._TXN_ID = (self._TXN_ID + 1) % (sys.maxint - 1) - - name = "%s-%x" % (desc, txn_id, ) - - sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) - transaction_logger.debug("[TXN START] {%s}", name) - try: - i = 0 - N = 5 - while True: - try: - txn = conn.cursor() - txn = LoggingTransaction( - txn, name, self.database_engine, after_callbacks - ) - return func(txn, *args, **kwargs) - except self.database_engine.module.OperationalError as e: - # This can happen if the database disappears mid - # transaction. - logger.warn( - "[TXN OPERROR] {%s} %s %d/%d", - name, e, i, N - ) - if i < N: - i += 1 - try: - conn.rollback() - except self.database_engine.module.Error as e1: - logger.warn( - "[TXN EROLL] {%s} %s", - name, e1, - ) - continue - except self.database_engine.module.DatabaseError as e: - if self.database_engine.is_deadlock(e): - logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) - if i < N: - i += 1 - try: - conn.rollback() - except self.database_engine.module.Error as e1: - logger.warn( - "[TXN EROLL] {%s} %s", - name, e1, - ) - continue - raise - except Exception as e: - logger.debug("[TXN FAIL] {%s} %s", name, e) - raise - finally: - end = time.time() * 1000 - duration = end - start - - transaction_logger.debug("[TXN END] {%s} %f", name, duration) - - self._current_txn_total_time += duration - self._txn_perf_counters.update(desc, start, end) - sql_txn_timer.inc_by(duration, desc) + with self._new_transaction(conn, desc, after_callbacks) as txn: + return func(txn, *args, **kwargs) result = yield preserve_context_over_fn( self._db_pool.runWithConnection, @@ -432,6 +452,32 @@ class SQLBaseStore(object): after_callback(*after_args) defer.returnValue(result) + @defer.inlineCallbacks + def runWithConnection(self, func, *args, **kwargs): + """Wraps the .runInteraction() method on the underlying db_pool.""" + current_context = LoggingContext.current_context() + + start_time = time.time() * 1000 + + def inner_func(conn, *args, **kwargs): + with LoggingContext("runWithConnection") as context: + sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) + + if self.database_engine.is_connection_closed(conn): + logger.debug("Reconnecting closed database connection") + conn.reconnect() + + current_context.copy_to(context) + + return func(conn, *args, **kwargs) + + result = yield preserve_context_over_fn( + self._db_pool.runWithConnection, + inner_func, *args, **kwargs + ) + + defer.returnValue(result) + def cursor_to_dict(self, cursor): """Converts a SQL cursor into an list of dicts. diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index a32302854..4a855ffd5 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -19,6 +19,8 @@ from ._base import IncorrectDatabaseSetup class PostgresEngine(object): + single_threaded = False + def __init__(self, database_module): self.module = database_module self.module.extensions.register_type(self.module.extensions.UNICODE) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index ff13d8006..d18e2808d 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -17,6 +17,8 @@ from synapse.storage import prepare_database, prepare_sqlite3_database class Sqlite3Engine(object): + single_threaded = True + def __init__(self, database_module): self.module = database_module diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 59af21a2c..b4abd8326 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -504,23 +504,26 @@ class EventsStore(SQLBaseStore): if not events: defer.returnValue({}) - def do_fetch(txn): + def do_fetch(conn): event_list = [] while True: try: with self._event_fetch_lock: - event_list = self._event_fetch_list - self._event_fetch_list = [] - - if not event_list: + i = 0 + while not self._event_fetch_list: self._event_fetch_ongoing -= 1 return + event_list = self._event_fetch_list + self._event_fetch_list = [] + event_id_lists = zip(*event_list)[0] event_ids = [ item for sublist in event_id_lists for item in sublist ] - rows = self._fetch_event_rows(txn, event_ids) + + with self._new_transaction(conn, "do_fetch", []) as txn: + rows = self._fetch_event_rows(txn, event_ids) row_dict = { r["event_id"]: r @@ -528,22 +531,44 @@ class EventsStore(SQLBaseStore): } for ids, d in event_list: - reactor.callFromThread( - d.callback, - [ - row_dict[i] for i in ids - if i in row_dict - ] - ) + def fire(): + if not d.called: + d.callback( + [ + row_dict[i] + for i in ids + if i in row_dict + ] + ) + reactor.callFromThread(fire) except Exception as e: + logger.exception("do_fetch") for _, d in event_list: - try: + if not d.called: reactor.callFromThread(d.errback, e) - except: - pass - def cb(rows): - return defer.gatherResults([ + with self._event_fetch_lock: + self._event_fetch_ongoing -= 1 + return + + events_d = defer.Deferred() + with self._event_fetch_lock: + self._event_fetch_list.append( + (events, events_d) + ) + + self._event_fetch_lock.notify_all() + + # if self._event_fetch_ongoing < 5: + self._event_fetch_ongoing += 1 + self.runWithConnection( + do_fetch + ) + + rows = yield events_d + + res = yield defer.gatherResults( + [ self._get_event_from_row( None, row["internal_metadata"], row["json"], row["redacts"], @@ -552,23 +577,9 @@ class EventsStore(SQLBaseStore): rejected_reason=row["rejects"], ) for row in rows - ]) - - d = defer.Deferred() - d.addCallback(cb) - with self._event_fetch_lock: - self._event_fetch_list.append( - (events, d) - ) - - if self._event_fetch_ongoing < 3: - self._event_fetch_ongoing += 1 - self.runInteraction( - "do_fetch", - do_fetch - ) - - res = yield d + ], + consumeErrors=True + ) defer.returnValue({ e.event_id: e From 47ec693e29ce61885b605191b97a69c1cbf7ab09 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 14 May 2015 15:29:58 +0100 Subject: [PATCH 076/175] More doc-strings --- synapse/handlers/presence.py | 241 +++++++++++++++++++++++++++++------ 1 file changed, 202 insertions(+), 39 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 0c246958a..23302242b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -317,6 +317,13 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def user_joined_room(self, user, room_id): + """Called via the distributor whenever a user joins a room. + Notifies the new member of the presence of the current members. + Notifies the current members of the room of the new member's presence. + Args: + user(UserID): The user who joined the room. + room_id(str): The room id the user joined. + """ if self.hs.is_mine(user): statuscache = self._get_or_make_usercache(user) @@ -344,6 +351,7 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def send_invite(self, observer_user, observed_user): + """Request the presence of a local or remote user for a local user""" if not self.hs.is_mine(observer_user): raise SynapseError(400, "User is not hosted on this Home Server") @@ -378,6 +386,16 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def invite_presence(self, observed_user, observer_user): + """Handles a m.presence_invite EDU. A remote or local user has + requested presence updates for a local user. If the invite is accepted + then allow the local or remote user to see the presence of the local + user. + + Args: + observed_user(UserID): The local user whose presence is requested. + observer_user(UserID): The remote or local user requesting presence. + + """ accept = yield self._should_accept_invite(observed_user, observer_user) if accept: @@ -404,6 +422,14 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def accept_presence(self, observed_user, observer_user): + """Handles a m.presence_accept EDU. Mark a presence invite from a + local or remote user as accepted in a local user's presence list. + Starts polling for presence updates from the local or remote user. + + Args: + observed_user(UserID): The user to update in the presence list. + observer_user(UserID): The owner of the presence list to update. + """ yield self.store.set_presence_list_accepted( observer_user.localpart, observed_user.to_string() ) @@ -414,6 +440,15 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def deny_presence(self, observed_user, observer_user): + """Handle a m.presence_deny EDU. Removes a local or remote user from a + local user's presence list. + + Args: + observed_user: The local or remote user to remove from the list. + observer_user: The local owner of the presence list. + Returns: + A Deferred. + """ yield self.store.del_presence_list( observer_user.localpart, observed_user.to_string() ) @@ -422,6 +457,15 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def drop(self, observed_user, observer_user): + """Remove a local or remote user from a local user's presence list and + unsubscribe the local user from updates that user. + + Args: + observed_user: The local or remote user to remove from the list. + observer_user: The local owner of the presence list. + Returns: + A Deferred. + """ if not self.hs.is_mine(observer_user): raise SynapseError(400, "User is not hosted on this Home Server") @@ -435,6 +479,16 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def get_presence_list(self, observer_user, accepted=None): + """Get the presence list for a local user. The retured list includes + the current presence state for each user listed. + + Args: + observer_user(UserID): The local user whose presence list to fetch. + accepted(bool or None): If not none then only include users who + have or have not accepted the presence invite request. + Returns: + A Deferred list of presence state events. + """ if not self.hs.is_mine(observer_user): raise SynapseError(400, "User is not hosted on this Home Server") @@ -456,6 +510,23 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks @log_function def start_polling_presence(self, user, target_user=None, state=None): + """Subscribe a local user to presence updates from a local or remote + user. If no target_user is supplied then subscribe to all users stored + in the presence list for the local user. + + Additonally this pushes the current presence state of this user to all + target_users. That state can be provided directly or will be read from + the stored state for the local user. + + Also this attempts to notify the local user of the current state of + any local target users. + + Args: + user(UserID): The local user that whishes for presence updates. + target_user(UserID): The local or remote user whose updates are + wanted. + state(dict): Optional presence state for the local user. + """ logger.debug("Start polling for presence from %s", user) if target_user: @@ -513,6 +584,11 @@ class PresenceHandler(BaseHandler): yield defer.DeferredList(deferreds, consumeErrors=True) def _start_polling_local(self, user, target_user): + """Subscribe a local user to presence updates for a local user + Args: + user(UserId): The local user that wishes for updates. + target_user(UserId): The local users whose updates are wanted. + """ target_localpart = target_user.localpart if target_localpart not in self._local_pushmap: @@ -521,6 +597,16 @@ class PresenceHandler(BaseHandler): self._local_pushmap[target_localpart].add(user) def _start_polling_remote(self, user, domain, remoteusers): + """Subscribe a local user to presence updates for remote users on a + given domain. + Args: + user(UserID): The local user that wishes for updates. + domain(str): The remote server the local user wants updates from. + remoteusers(UserID): The remote users that local user wants to be + told about. + Returns: + A Deferred. + """ to_poll = set() for u in remoteusers: @@ -541,6 +627,16 @@ class PresenceHandler(BaseHandler): @log_function def stop_polling_presence(self, user, target_user=None): + """Unsubscribe a local user from presence updates from a local or + remote user. If no target user is supplied then unsubscribe the user + from all presence updates that the user had subscribed to. + Args: + user(UserID): The local user that no longer wishes for updates. + target_user(UserID or None): The user whose updates are no longer + wanted. + Returns: + A Deferred. + """ logger.debug("Stop polling for presence from %s", user) if not target_user or self.hs.is_mine(target_user): @@ -569,6 +665,12 @@ class PresenceHandler(BaseHandler): return defer.DeferredList(deferreds, consumeErrors=True) def _stop_polling_local(self, user, target_user): + """Unsubscribe a local user from presence updates from a local user on + this server. + Args: + user(UserID): The local user that no longer wishes for updates. + target_user(UserID): The user whose updates are no longer wanted. + """ for localpart in self._local_pushmap.keys(): if target_user and localpart != target_user.localpart: continue @@ -581,6 +683,16 @@ class PresenceHandler(BaseHandler): @log_function def _stop_polling_remote(self, user, domain, remoteusers): + """Unsubscribe a local user from presence updates from remote users on + a given domain. + Args: + user(UserID): The local user that no longer wishes for updates. + domain(str): The remote server to unsubscribe from. + remoteusers([UserID]): The users on that remote server that the + local user no longer wishes to be updated about. + Returns: + A Deferred. + """ to_unpoll = set() for u in remoteusers: @@ -602,6 +714,18 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks @log_function def push_presence(self, user, statuscache): + """ + Notify local and remote users of a change in presence of a local user. + Pushes the update to local clients and remote domains that are directly + subscribed to the presence of the local user. + Also pushes that update to any local user or remote domain that shares + a room with the local user. + Args: + user(UserID): The local user whose presence was updated. + statuscache(UserPresenceCache): Cache of the user's presence state + Returns: + A Deferred. + """ assert(self.hs.is_mine(user)) logger.debug("Pushing presence update from %s", user) @@ -628,45 +752,23 @@ class PresenceHandler(BaseHandler): ) yield self.distributor.fire("user_presence_changed", user, statuscache) - @defer.inlineCallbacks - def _push_presence_remote(self, user, destination, state=None): - if state is None: - state = yield self.store.get_presence_state(user.localpart) - del state["mtime"] - state["presence"] = state.pop("state") - - if user in self._user_cachemap: - state["last_active"] = ( - self._user_cachemap[user].get_state()["last_active"] - ) - - yield self.distributor.fire( - "collect_presencelike_data", user, state - ) - - if "last_active" in state: - state = dict(state) - state["last_active_ago"] = int( - self.clock.time_msec() - state.pop("last_active") - ) - - user_state = { - "user_id": user.to_string(), - } - user_state.update(**state) - - yield self.federation.send_edu( - destination=destination, - edu_type="m.presence", - content={ - "push": [ - user_state, - ], - } - ) - @defer.inlineCallbacks def incoming_presence(self, origin, content): + """Handle an incoming m.presence EDU. + For each presence update in the "push" list update our local cache and + notify the appropriate local clients. Only clients that share a room + or are directly subscribed to the presence for a user should be + notified of the update. + For each subscription request in the "poll" list start pushing presence + updates to the remote server. + For unsubscribe request in the "unpoll" list stop pushing presence + updates to the remote server. + Args: + orgin(str): The source of this m.presence EDU. + content(dict): The content of this m.presence EDU. + Returns: + A Deferred. + """ deferreds = [] for push in content.get("push", []): @@ -765,6 +867,22 @@ class PresenceHandler(BaseHandler): def push_update_to_local_and_remote(self, observed_user, statuscache, users_to_push=[], room_ids=[], remote_domains=[]): + """Notify local clients and remote servers of a change in the presence + of a user. + Args: + observed_user(UserID): The user to push the presence state for. + statuscache(UserPresenceCache): The cache for the presence state to + push. + users_to_push([UserID]): A list of local and remote users to + notify. + room_ids([str]): Notify the local and remote occupants of these + rooms. + remote_domains([str]): A list of remote servers to notify in + addition to those implied by the users_to_push and the + room_ids. + Returns: + A Deferred. + """ localusers, remoteusers = partitionbool( users_to_push, @@ -802,8 +920,8 @@ class PresenceHandler(BaseHandler): def push_update_to_clients(self, users_to_push=[], room_ids=[]): """Notify clients of a new presence event. Args: - users_to_push(list): List of users to notify. - room_ids(list): List of room_ids to notify. + users_to_push([UserID]): List of users to notify. + room_ids([str]): List of room_ids to notify. """ with PreserveLoggingContext(): self.notifier.on_new_user_event( @@ -811,6 +929,51 @@ class PresenceHandler(BaseHandler): room_ids, ) + @defer.inlineCallbacks + def _push_presence_remote(self, user, destination, state=None): + """Push a user's presence to a remote server. If a presence state event + that event is sent. Otherwise a new state event is constructed from the + stored presence state. + The last_active is replaced with last_active_ago in case the wallclock + time on the remote server is different to the time on this server. + Sends an EDU to the remote server with the current presence state. + Args: + user(UserID): The user to push the presence state for. + destination(str): The remote server to send state to. + state(dict): The state to push, or None to use the current stored + state. + Returns: + A Deferred. + """ + if state is None: + state = yield self.store.get_presence_state(user.localpart) + del state["mtime"] + state["presence"] = state.pop("state") + + if user in self._user_cachemap: + state["last_active"] = ( + self._user_cachemap[user].get_state()["last_active"] + ) + + yield self.distributor.fire( + "collect_presencelike_data", user, state + ) + + if "last_active" in state: + state = dict(state) + state["last_active_ago"] = int( + self.clock.time_msec() - state.pop("last_active") + ) + + user_state = {"user_id": user.to_string(), } + user_state.update(state) + + yield self.federation.send_edu( + destination=destination, + edu_type="m.presence", + content={"push": [user_state, ], } + ) + class PresenceEventSource(object): def __init__(self, hs): From 0a4330cd5d5e2230fb9e1ff4e24952829d03ef76 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 14 May 2015 17:48:12 +0100 Subject: [PATCH 077/175] Add some missed argument types, cleanup the whitespace a bit --- synapse/handlers/presence.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 23302242b..9638faf4b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -394,7 +394,6 @@ class PresenceHandler(BaseHandler): Args: observed_user(UserID): The local user whose presence is requested. observer_user(UserID): The remote or local user requesting presence. - """ accept = yield self._should_accept_invite(observed_user, observer_user) @@ -444,8 +443,9 @@ class PresenceHandler(BaseHandler): local user's presence list. Args: - observed_user: The local or remote user to remove from the list. - observer_user: The local owner of the presence list. + observed_user(UserID): The local or remote user to remove from the + list. + observer_user(UserID): The local owner of the presence list. Returns: A Deferred. """ @@ -461,8 +461,9 @@ class PresenceHandler(BaseHandler): unsubscribe the local user from updates that user. Args: - observed_user: The local or remote user to remove from the list. - observer_user: The local owner of the presence list. + observed_user(UserId): The local or remote user to remove from the + list. + observer_user(UserId): The local owner of the presence list. Returns: A Deferred. """ @@ -585,6 +586,7 @@ class PresenceHandler(BaseHandler): def _start_polling_local(self, user, target_user): """Subscribe a local user to presence updates for a local user + Args: user(UserId): The local user that wishes for updates. target_user(UserId): The local users whose updates are wanted. @@ -598,7 +600,8 @@ class PresenceHandler(BaseHandler): def _start_polling_remote(self, user, domain, remoteusers): """Subscribe a local user to presence updates for remote users on a - given domain. + given remote domain. + Args: user(UserID): The local user that wishes for updates. domain(str): The remote server the local user wants updates from. @@ -630,6 +633,7 @@ class PresenceHandler(BaseHandler): """Unsubscribe a local user from presence updates from a local or remote user. If no target user is supplied then unsubscribe the user from all presence updates that the user had subscribed to. + Args: user(UserID): The local user that no longer wishes for updates. target_user(UserID or None): The user whose updates are no longer @@ -667,6 +671,7 @@ class PresenceHandler(BaseHandler): def _stop_polling_local(self, user, target_user): """Unsubscribe a local user from presence updates from a local user on this server. + Args: user(UserID): The local user that no longer wishes for updates. target_user(UserID): The user whose updates are no longer wanted. @@ -685,6 +690,7 @@ class PresenceHandler(BaseHandler): def _stop_polling_remote(self, user, domain, remoteusers): """Unsubscribe a local user from presence updates from remote users on a given domain. + Args: user(UserID): The local user that no longer wishes for updates. domain(str): The remote server to unsubscribe from. @@ -720,6 +726,7 @@ class PresenceHandler(BaseHandler): subscribed to the presence of the local user. Also pushes that update to any local user or remote domain that shares a room with the local user. + Args: user(UserID): The local user whose presence was updated. statuscache(UserPresenceCache): Cache of the user's presence state @@ -763,6 +770,7 @@ class PresenceHandler(BaseHandler): updates to the remote server. For unsubscribe request in the "unpoll" list stop pushing presence updates to the remote server. + Args: orgin(str): The source of this m.presence EDU. content(dict): The content of this m.presence EDU. @@ -869,6 +877,7 @@ class PresenceHandler(BaseHandler): remote_domains=[]): """Notify local clients and remote servers of a change in the presence of a user. + Args: observed_user(UserID): The user to push the presence state for. statuscache(UserPresenceCache): The cache for the presence state to @@ -919,6 +928,7 @@ class PresenceHandler(BaseHandler): def push_update_to_clients(self, users_to_push=[], room_ids=[]): """Notify clients of a new presence event. + Args: users_to_push([UserID]): List of users to notify. room_ids([str]): List of room_ids to notify. @@ -937,6 +947,7 @@ class PresenceHandler(BaseHandler): The last_active is replaced with last_active_ago in case the wallclock time on the remote server is different to the time on this server. Sends an EDU to the remote server with the current presence state. + Args: user(UserID): The user to push the presence state for. destination(str): The remote server to send state to. From a2c4f3f150f63c720370f6882da804c8ac20fd69 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 10:54:04 +0100 Subject: [PATCH 078/175] Fix daedlock --- synapse/federation/federation_client.py | 15 ++- synapse/federation/federation_server.py | 2 + synapse/handlers/message.py | 33 ++++--- synapse/storage/_base.py | 26 ++---- synapse/storage/events.py | 117 ++++++++++++++---------- synapse/storage/stream.py | 2 + tests/storage/test_base.py | 3 +- 7 files changed, 118 insertions(+), 80 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 904c7c094..c255df1bb 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -222,7 +222,7 @@ class FederationClient(FederationBase): for p in transaction_data["pdus"] ] - if pdu_list: + if pdu_list and pdu_list[0]: pdu = pdu_list[0] # Check signatures are correct. @@ -255,7 +255,7 @@ class FederationClient(FederationBase): ) continue - if self._get_pdu_cache is not None: + if self._get_pdu_cache is not None and pdu: self._get_pdu_cache[event_id] = pdu defer.returnValue(pdu) @@ -475,6 +475,9 @@ class FederationClient(FederationBase): limit (int): Maximum number of events to return. min_depth (int): Minimum depth of events tor return. """ + logger.debug("get_missing_events: latest_events: %r", latest_events) + logger.debug("get_missing_events: earliest_events_ids: %r", earliest_events_ids) + try: content = yield self.transport_layer.get_missing_events( destination=destination, @@ -485,6 +488,8 @@ class FederationClient(FederationBase): min_depth=min_depth, ) + logger.debug("get_missing_events: Got content: %r", content) + events = [ self.event_from_pdu_json(e) for e in content.get("events", []) @@ -494,6 +499,8 @@ class FederationClient(FederationBase): destination, events, outlier=False ) + logger.debug("get_missing_events: signed_events: %r", signed_events) + have_gotten_all_from_destination = True except HttpResponseException as e: if not e.code == 400: @@ -518,6 +525,8 @@ class FederationClient(FederationBase): # Are we missing any? seen_events = set(earliest_events_ids) + + logger.debug("get_missing_events: signed_events2: %r", signed_events) seen_events.update(e.event_id for e in signed_events) missing_events = {} @@ -561,7 +570,7 @@ class FederationClient(FederationBase): res = yield defer.DeferredList(deferreds, consumeErrors=True) for (result, val), (e_id, _) in zip(res, ordered_missing): - if result: + if result and val: signed_events.append(val) else: failed_to_fetch.add(e_id) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index cd79e23f4..2c6488dd1 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -415,6 +415,8 @@ class FederationServer(FederationBase): pdu.internal_metadata.outlier = True elif min_depth and pdu.depth > min_depth: if get_missing and prevs - seen: + logger.debug("We're missing: %r", prevs-seen) + latest = yield self.store.get_latest_event_ids_in_room( pdu.room_id ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 867fdbefb..6a1b25d11 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -303,18 +303,27 @@ class MessageHandler(BaseHandler): if event.membership != Membership.JOIN: return try: - (messages, token), current_state = yield defer.gatherResults( - [ - self.store.get_recent_events_for_room( - event.room_id, - limit=limit, - end_token=now_token.room_key, - ), - self.state_handler.get_current_state( - event.room_id - ), - ] - ).addErrback(unwrapFirstError) + # (messages, token), current_state = yield defer.gatherResults( + # [ + # self.store.get_recent_events_for_room( + # event.room_id, + # limit=limit, + # end_token=now_token.room_key, + # ), + # self.state_handler.get_current_state( + # event.room_id + # ), + # ] + # ).addErrback(unwrapFirstError) + + messages, token = yield self.store.get_recent_events_for_room( + event.room_id, + limit=limit, + end_token=now_token.room_key, + ) + current_state = yield self.state_handler.get_current_state( + event.room_id + ) start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ceff99c16..0df1b46ed 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -301,10 +301,12 @@ class SQLBaseStore(object): self._get_event_cache = Cache("*getEvent*", keylen=3, lru=True, max_entries=hs.config.event_cache_size) - self._event_fetch_lock = threading.Condition() + self._event_fetch_lock = threading.Lock() self._event_fetch_list = [] self._event_fetch_ongoing = 0 + self._pending_ds = [] + self.database_engine = hs.database_engine self._stream_id_gen = StreamIdGenerator() @@ -344,8 +346,7 @@ class SQLBaseStore(object): self._clock.looping_call(loop, 10000) - @contextlib.contextmanager - def _new_transaction(self, conn, desc, after_callbacks): + def _new_transaction(self, conn, desc, after_callbacks, func, *args, **kwargs): start = time.time() * 1000 txn_id = self._TXN_ID @@ -366,6 +367,9 @@ class SQLBaseStore(object): txn = LoggingTransaction( txn, name, self.database_engine, after_callbacks ) + r = func(txn, *args, **kwargs) + conn.commit() + return r except self.database_engine.module.OperationalError as e: # This can happen if the database disappears mid # transaction. @@ -398,17 +402,6 @@ class SQLBaseStore(object): ) continue raise - - try: - yield txn - conn.commit() - return - except: - try: - conn.rollback() - except: - pass - raise except Exception as e: logger.debug("[TXN FAIL] {%s} %s", name, e) raise @@ -440,8 +433,9 @@ class SQLBaseStore(object): conn.reconnect() current_context.copy_to(context) - with self._new_transaction(conn, desc, after_callbacks) as txn: - return func(txn, *args, **kwargs) + return self._new_transaction( + conn, desc, after_callbacks, func, *args, **kwargs + ) result = yield preserve_context_over_fn( self._db_pool.runWithConnection, diff --git a/synapse/storage/events.py b/synapse/storage/events.py index b4abd8326..260bdf0ec 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -420,12 +420,14 @@ class EventsStore(SQLBaseStore): ]) if not txn: + logger.debug("enqueue before") missing_events = yield self._enqueue_events( missing_events_ids, check_redacted=check_redacted, get_prev_content=get_prev_content, allow_rejected=allow_rejected, ) + logger.debug("enqueue after") else: missing_events = self._fetch_events_txn( txn, @@ -498,41 +500,39 @@ class EventsStore(SQLBaseStore): allow_rejected=allow_rejected, )) - @defer.inlineCallbacks - def _enqueue_events(self, events, check_redacted=True, - get_prev_content=False, allow_rejected=False): - if not events: - defer.returnValue({}) - - def do_fetch(conn): - event_list = [] + def _do_fetch(self, conn): + event_list = [] + try: while True: - try: - with self._event_fetch_lock: - i = 0 - while not self._event_fetch_list: - self._event_fetch_ongoing -= 1 - return + logger.debug("do_fetch getting lock") + with self._event_fetch_lock: + logger.debug("do_fetch go lock: %r", self._event_fetch_list) + event_list = self._event_fetch_list + self._event_fetch_list = [] + if not event_list: + self._event_fetch_ongoing -= 1 + return - event_list = self._event_fetch_list - self._event_fetch_list = [] + event_id_lists = zip(*event_list)[0] + event_ids = [ + item for sublist in event_id_lists for item in sublist + ] - event_id_lists = zip(*event_list)[0] - event_ids = [ - item for sublist in event_id_lists for item in sublist - ] + rows = self._new_transaction( + conn, "do_fetch", [], self._fetch_event_rows, event_ids + ) - with self._new_transaction(conn, "do_fetch", []) as txn: - rows = self._fetch_event_rows(txn, event_ids) + row_dict = { + r["event_id"]: r + for r in rows + } - row_dict = { - r["event_id"]: r - for r in rows - } + logger.debug("do_fetch got events: %r", row_dict.keys()) - for ids, d in event_list: - def fire(): - if not d.called: + def fire(evs): + for ids, d in evs: + if not d.called: + try: d.callback( [ row_dict[i] @@ -540,32 +540,51 @@ class EventsStore(SQLBaseStore): if i in row_dict ] ) - reactor.callFromThread(fire) - except Exception as e: - logger.exception("do_fetch") - for _, d in event_list: - if not d.called: - reactor.callFromThread(d.errback, e) + except: + logger.exception("Failed to callback") + reactor.callFromThread(fire, event_list) + except Exception as e: + logger.exception("do_fetch") - with self._event_fetch_lock: - self._event_fetch_ongoing -= 1 - return + def fire(evs): + for _, d in evs: + if not d.called: + d.errback(e) + + if event_list: + reactor.callFromThread(fire, event_list) + + @defer.inlineCallbacks + def _enqueue_events(self, events, check_redacted=True, + get_prev_content=False, allow_rejected=False): + if not events: + defer.returnValue({}) events_d = defer.Deferred() - with self._event_fetch_lock: - self._event_fetch_list.append( - (events, events_d) - ) + try: + logger.debug("enqueueueueue getting lock") + with self._event_fetch_lock: + logger.debug("enqueue go lock") + self._event_fetch_list.append( + (events, events_d) + ) - self._event_fetch_lock.notify_all() + self._event_fetch_ongoing += 1 - # if self._event_fetch_ongoing < 5: - self._event_fetch_ongoing += 1 self.runWithConnection( - do_fetch + self._do_fetch ) - rows = yield events_d + except Exception as e: + if not events_d.called: + events_d.errback(e) + + logger.debug("events_d before") + try: + rows = yield events_d + except: + logger.exception("events_d") + logger.debug("events_d after") res = yield defer.gatherResults( [ @@ -580,6 +599,7 @@ class EventsStore(SQLBaseStore): ], consumeErrors=True ) + logger.debug("gatherResults after") defer.returnValue({ e.event_id: e @@ -639,7 +659,8 @@ class EventsStore(SQLBaseStore): rejected_reason=row["rejects"], ) for row in rows - ] + ], + consumeErrors=True, ) defer.returnValue({ diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index d16b57c51..af45fc561 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -357,10 +357,12 @@ class StreamStore(SQLBaseStore): "get_recent_events_for_room", get_recent_events_for_room_txn ) + logger.debug("stream before") events = yield self._get_events( [r["event_id"] for r in rows], get_prev_content=True ) + logger.debug("stream after") self._set_before_and_after(events, rows) diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 8c348ecc9..8573f18b5 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -33,8 +33,9 @@ class SQLBaseStoreTestCase(unittest.TestCase): def setUp(self): self.db_pool = Mock(spec=["runInteraction"]) self.mock_txn = Mock() - self.mock_conn = Mock(spec_set=["cursor"]) + self.mock_conn = Mock(spec_set=["cursor", "rollback", "commit"]) self.mock_conn.cursor.return_value = self.mock_txn + self.mock_conn.rollback.return_value = None # Our fake runInteraction just runs synchronously inline def runInteraction(func, *args, **kwargs): From de01438a578c285b632a4c791a56bebfe29fb06b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 11:00:50 +0100 Subject: [PATCH 079/175] Sort out error handling --- synapse/storage/events.py | 47 ++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 260bdf0ec..f2c181dde 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -502,8 +502,8 @@ class EventsStore(SQLBaseStore): def _do_fetch(self, conn): event_list = [] - try: - while True: + while True: + try: logger.debug("do_fetch getting lock") with self._event_fetch_lock: logger.debug("do_fetch go lock: %r", self._event_fetch_list) @@ -543,16 +543,16 @@ class EventsStore(SQLBaseStore): except: logger.exception("Failed to callback") reactor.callFromThread(fire, event_list) - except Exception as e: - logger.exception("do_fetch") + except Exception as e: + logger.exception("do_fetch") - def fire(evs): - for _, d in evs: - if not d.called: - d.errback(e) + def fire(evs): + for _, d in evs: + if not d.called: + d.errback(e) - if event_list: - reactor.callFromThread(fire, event_list) + if event_list: + reactor.callFromThread(fire, event_list) @defer.inlineCallbacks def _enqueue_events(self, events, check_redacted=True, @@ -561,29 +561,26 @@ class EventsStore(SQLBaseStore): defer.returnValue({}) events_d = defer.Deferred() - try: - logger.debug("enqueueueueue getting lock") - with self._event_fetch_lock: - logger.debug("enqueue go lock") - self._event_fetch_list.append( - (events, events_d) - ) + logger.debug("enqueueueueue getting lock") + with self._event_fetch_lock: + logger.debug("enqueue go lock") + self._event_fetch_list.append( + (events, events_d) + ) + if self._event_fetch_ongoing < 1: self._event_fetch_ongoing += 1 + should_start = True + else: + should_start = False + if should_start: self.runWithConnection( self._do_fetch ) - except Exception as e: - if not events_d.called: - events_d.errback(e) - logger.debug("events_d before") - try: - rows = yield events_d - except: - logger.exception("events_d") + rows = yield events_d logger.debug("events_d after") res = yield defer.gatherResults( From 415b158ce229d4f740bf577aca5cc3d5f73e1bf6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 15 May 2015 11:09:47 +0100 Subject: [PATCH 080/175] More whitespace --- synapse/handlers/presence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 9638faf4b..a01020e20 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -320,6 +320,7 @@ class PresenceHandler(BaseHandler): """Called via the distributor whenever a user joins a room. Notifies the new member of the presence of the current members. Notifies the current members of the room of the new member's presence. + Args: user(UserID): The user who joined the room. room_id(str): The room id the user joined. From 575ec91d82e6b283bd21471c5a874de968c97bff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 11:15:10 +0100 Subject: [PATCH 081/175] Correctly pass through params --- synapse/storage/events.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f2c181dde..143c24b10 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -91,9 +91,9 @@ class EventsStore(SQLBaseStore): """ events = yield self._get_events( [event_id], - check_redacted=True, - get_prev_content=False, - allow_rejected=False, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, ) if not events and not allow_none: From 372d4c6d7b38f89fb79509cf432915d96bdc8164 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 11:26:00 +0100 Subject: [PATCH 082/175] Srsly. Don't use closures. Baaaaaad --- synapse/storage/events.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 143c24b10..2c3e6d5a5 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -529,20 +529,18 @@ class EventsStore(SQLBaseStore): logger.debug("do_fetch got events: %r", row_dict.keys()) - def fire(evs): - for ids, d in evs: + def fire(lst, res): + for ids, d in lst: if not d.called: try: - d.callback( - [ - row_dict[i] - for i in ids - if i in row_dict - ] - ) + d.callback([ + res[i] + for i in ids + if i in res + ]) except: logger.exception("Failed to callback") - reactor.callFromThread(fire, event_list) + reactor.callFromThread(fire, event_list, row_dict) except Exception as e: logger.exception("do_fetch") From aa32bd38e40cd8d69406fe74581290ad7fe34f35 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 11:35:04 +0100 Subject: [PATCH 083/175] Add a wait --- synapse/storage/_base.py | 2 +- synapse/storage/events.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 0df1b46ed..5d86aa5cd 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -301,7 +301,7 @@ class SQLBaseStore(object): self._get_event_cache = Cache("*getEvent*", keylen=3, lru=True, max_entries=hs.config.event_cache_size) - self._event_fetch_lock = threading.Lock() + self._event_fetch_lock = threading.Condition() self._event_fetch_list = [] self._event_fetch_ongoing = 0 diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2c3e6d5a5..f694b877f 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -502,6 +502,7 @@ class EventsStore(SQLBaseStore): def _do_fetch(self, conn): event_list = [] + i = 0 while True: try: logger.debug("do_fetch getting lock") @@ -510,8 +511,14 @@ class EventsStore(SQLBaseStore): event_list = self._event_fetch_list self._event_fetch_list = [] if not event_list: - self._event_fetch_ongoing -= 1 - return + if self.database_engine.single_threaded or i > 5: + self._event_fetch_ongoing -= 1 + return + else: + self._event_fetch_lock.wait(0.1) + i += 1 + continue + i = 0 event_id_lists = zip(*event_list)[0] event_ids = [ @@ -566,6 +573,8 @@ class EventsStore(SQLBaseStore): (events, events_d) ) + self._event_fetch_lock.notify_all() + if self._event_fetch_ongoing < 1: self._event_fetch_ongoing += 1 should_start = True From e275a9c0d9365f77cb06f2a04c55f591ac5b54c7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 11:54:51 +0100 Subject: [PATCH 084/175] preserve log context --- synapse/storage/events.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f694b877f..c62151603 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -21,6 +21,7 @@ from synapse.events import FrozenEvent from synapse.events.utils import prune_event from synapse.util import unwrap_deferred +from synapse.util.logcontext import preserve_context_over_deferred from synapse.util.logutils import log_function from synapse.api.constants import EventTypes from synapse.crypto.event_signing import compute_event_reference_hash @@ -587,7 +588,7 @@ class EventsStore(SQLBaseStore): ) logger.debug("events_d before") - rows = yield events_d + rows = yield preserve_context_over_deferred(events_d) logger.debug("events_d after") res = yield defer.gatherResults( From 0f29cfabc3177c149a4c34fc05398b8d9f3a06ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 14:06:42 +0100 Subject: [PATCH 085/175] Remove debug logging --- synapse/storage/events.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index c62151603..716f10386 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -506,13 +506,11 @@ class EventsStore(SQLBaseStore): i = 0 while True: try: - logger.debug("do_fetch getting lock") with self._event_fetch_lock: - logger.debug("do_fetch go lock: %r", self._event_fetch_list) event_list = self._event_fetch_list self._event_fetch_list = [] if not event_list: - if self.database_engine.single_threaded or i > 5: + if self.database_engine.single_threaded or i > 3: self._event_fetch_ongoing -= 1 return else: @@ -535,8 +533,6 @@ class EventsStore(SQLBaseStore): for r in rows } - logger.debug("do_fetch got events: %r", row_dict.keys()) - def fire(lst, res): for ids, d in lst: if not d.called: @@ -567,16 +563,14 @@ class EventsStore(SQLBaseStore): defer.returnValue({}) events_d = defer.Deferred() - logger.debug("enqueueueueue getting lock") with self._event_fetch_lock: - logger.debug("enqueue go lock") self._event_fetch_list.append( (events, events_d) ) - self._event_fetch_lock.notify_all() + self._event_fetch_lock.notify() - if self._event_fetch_ongoing < 1: + if self._event_fetch_ongoing < 3: self._event_fetch_ongoing += 1 should_start = True else: @@ -587,9 +581,7 @@ class EventsStore(SQLBaseStore): self._do_fetch ) - logger.debug("events_d before") rows = yield preserve_context_over_deferred(events_d) - logger.debug("events_d after") res = yield defer.gatherResults( [ From d62dee7eae741034abfd050982b1cfc4e2181258 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 15:06:37 +0100 Subject: [PATCH 086/175] Remove more debug logging --- synapse/storage/events.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 716f10386..3cf2f7cff 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -421,14 +421,12 @@ class EventsStore(SQLBaseStore): ]) if not txn: - logger.debug("enqueue before") missing_events = yield self._enqueue_events( missing_events_ids, check_redacted=check_redacted, get_prev_content=get_prev_content, allow_rejected=allow_rejected, ) - logger.debug("enqueue after") else: missing_events = self._fetch_events_txn( txn, From acb12cc811d7ce7cb3c5b6544ed28f7d6592ef33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 15:20:05 +0100 Subject: [PATCH 087/175] Make store.get_current_state fetch events asyncly --- synapse/storage/events.py | 1 - synapse/storage/state.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 3cf2f7cff..066e1aab7 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -594,7 +594,6 @@ class EventsStore(SQLBaseStore): ], consumeErrors=True ) - logger.debug("gatherResults after") defer.returnValue({ e.event_id: e diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 3f5642642..b3f2a4dfa 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -159,11 +159,12 @@ class StateStore(SQLBaseStore): args = (room_id, ) txn.execute(sql, args) - results = self.cursor_to_dict(txn) + results = txn.fetchall() - return self._parse_events_txn(txn, results) + return [r[0] for r in results] - events = yield self.runInteraction("get_current_state", f) + event_ids = yield self.runInteraction("get_current_state", f) + events = yield self._get_events(event_ids) defer.returnValue(events) From 807229f2f2acbdcb471bdfd8458910879d183c63 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 15:20:29 +0100 Subject: [PATCH 088/175] Err, defer.gatherResults ftw --- synapse/handlers/message.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6a1b25d11..867fdbefb 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -303,27 +303,18 @@ class MessageHandler(BaseHandler): if event.membership != Membership.JOIN: return try: - # (messages, token), current_state = yield defer.gatherResults( - # [ - # self.store.get_recent_events_for_room( - # event.room_id, - # limit=limit, - # end_token=now_token.room_key, - # ), - # self.state_handler.get_current_state( - # event.room_id - # ), - # ] - # ).addErrback(unwrapFirstError) - - messages, token = yield self.store.get_recent_events_for_room( - event.room_id, - limit=limit, - end_token=now_token.room_key, - ) - current_state = yield self.state_handler.get_current_state( - event.room_id - ) + (messages, token), current_state = yield defer.gatherResults( + [ + self.store.get_recent_events_for_room( + event.room_id, + limit=limit, + end_token=now_token.room_key, + ), + self.state_handler.get_current_state( + event.room_id + ), + ] + ).addErrback(unwrapFirstError) start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) From 8763dd80efd19d562a97a8d5af59b85bc3678d46 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 15:33:01 +0100 Subject: [PATCH 089/175] Don't fetch prev_content for current_state --- synapse/storage/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index b3f2a4dfa..56f0572f7 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -164,7 +164,7 @@ class StateStore(SQLBaseStore): return [r[0] for r in results] event_ids = yield self.runInteraction("get_current_state", f) - events = yield self._get_events(event_ids) + events = yield self._get_events(event_ids, get_prev_content=False) defer.returnValue(events) From 70f272f71ca399205a72deb29b0f86ff3bf23618 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 16:34:17 +0100 Subject: [PATCH 090/175] Don't completely drain the list --- synapse/storage/events.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 066e1aab7..6ee2e9a48 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -505,8 +505,15 @@ class EventsStore(SQLBaseStore): while True: try: with self._event_fetch_lock: - event_list = self._event_fetch_list - self._event_fetch_list = [] + tot = 0 + for j, lst in enumerate(self._event_fetch_list): + if tot > 200: + break + tot += len(lst[0]) + + event_list = self._event_fetch_list[:j+1] + self._event_fetch_list = self._event_fetch_list[j+1:] + if not event_list: if self.database_engine.single_threaded or i > 3: self._event_fetch_ongoing -= 1 From 9ff7f66a2be6d3f0542b69e784b400df349869ff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 16:36:03 +0100 Subject: [PATCH 091/175] init j --- synapse/storage/events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 6ee2e9a48..f21364606 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -506,6 +506,7 @@ class EventsStore(SQLBaseStore): try: with self._event_fetch_lock: tot = 0 + j = 0 for j, lst in enumerate(self._event_fetch_list): if tot > 200: break From 6c74fd62a0b2292b4fce9d59ea9790685e256e0a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 16:45:35 +0100 Subject: [PATCH 092/175] Revert limiting of fetching, it didn't help perf. --- synapse/storage/events.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f21364606..98a566393 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -505,15 +505,8 @@ class EventsStore(SQLBaseStore): while True: try: with self._event_fetch_lock: - tot = 0 - j = 0 - for j, lst in enumerate(self._event_fetch_list): - if tot > 200: - break - tot += len(lst[0]) - - event_list = self._event_fetch_list[:j+1] - self._event_fetch_list = self._event_fetch_list[j+1:] + event_list = self._event_fetch_list + self._event_fetch_list = [] if not event_list: if self.database_engine.single_threaded or i > 3: From c3b37abdfd6d34ea7ff7698bd61dd37763e257b1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 May 2015 16:59:58 +0100 Subject: [PATCH 093/175] PEP8 --- synapse/storage/_base.py | 1 - synapse/storage/events.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5d86aa5cd..b529e0543 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -27,7 +27,6 @@ from twisted.internet import defer from collections import namedtuple, OrderedDict -import contextlib import functools import sys import time diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 98a566393..25fb49a28 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -629,7 +629,7 @@ class EventsStore(SQLBaseStore): @defer.inlineCallbacks def _fetch_events(self, txn, events, check_redacted=True, - get_prev_content=False, allow_rejected=False): + get_prev_content=False, allow_rejected=False): if not events: defer.returnValue({}) From f8bd4de87de9464c54dcc50371866e3537754b9b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 May 2015 09:58:03 +0100 Subject: [PATCH 094/175] Remove debug logging --- synapse/federation/federation_client.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index c255df1bb..fe5a7a9fa 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -475,9 +475,6 @@ class FederationClient(FederationBase): limit (int): Maximum number of events to return. min_depth (int): Minimum depth of events tor return. """ - logger.debug("get_missing_events: latest_events: %r", latest_events) - logger.debug("get_missing_events: earliest_events_ids: %r", earliest_events_ids) - try: content = yield self.transport_layer.get_missing_events( destination=destination, @@ -488,8 +485,6 @@ class FederationClient(FederationBase): min_depth=min_depth, ) - logger.debug("get_missing_events: Got content: %r", content) - events = [ self.event_from_pdu_json(e) for e in content.get("events", []) @@ -499,8 +494,6 @@ class FederationClient(FederationBase): destination, events, outlier=False ) - logger.debug("get_missing_events: signed_events: %r", signed_events) - have_gotten_all_from_destination = True except HttpResponseException as e: if not e.code == 400: @@ -526,7 +519,6 @@ class FederationClient(FederationBase): seen_events = set(earliest_events_ids) - logger.debug("get_missing_events: signed_events2: %r", signed_events) seen_events.update(e.event_id for e in signed_events) missing_events = {} From c71176858b9d58cfbe5520ad1dac8191c005fdc9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 May 2015 10:11:14 +0100 Subject: [PATCH 095/175] Newline, remove debug logging --- synapse/federation/federation_server.py | 2 -- synapse/storage/_base.py | 1 - synapse/storage/schema/delta/19/event_index.sql | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 2c6488dd1..cd79e23f4 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -415,8 +415,6 @@ class FederationServer(FederationBase): pdu.internal_metadata.outlier = True elif min_depth and pdu.depth > min_depth: if get_missing and prevs - seen: - logger.debug("We're missing: %r", prevs-seen) - latest = yield self.store.get_latest_event_ids_in_room( pdu.room_id ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b529e0543..d1f050394 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -15,7 +15,6 @@ import logging from synapse.api.errors import StoreError - from synapse.util.logutils import log_function from synapse.util.logcontext import preserve_context_over_fn, LoggingContext from synapse.util.lrucache import LruCache diff --git a/synapse/storage/schema/delta/19/event_index.sql b/synapse/storage/schema/delta/19/event_index.sql index f3792817b..3881fc989 100644 --- a/synapse/storage/schema/delta/19/event_index.sql +++ b/synapse/storage/schema/delta/19/event_index.sql @@ -16,4 +16,4 @@ CREATE INDEX events_order_topo_stream_room ON events( topological_ordering, stream_ordering, room_id -); \ No newline at end of file +); From d5cea26d45a50053fcb16296b73bbced49675a74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 May 2015 10:16:45 +0100 Subject: [PATCH 096/175] Remove pointless newline --- synapse/federation/federation_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index fe5a7a9fa..3a7bc0c9a 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -518,7 +518,6 @@ class FederationClient(FederationBase): # Are we missing any? seen_events = set(earliest_events_ids) - seen_events.update(e.event_id for e in signed_events) missing_events = {} From 10f1bdb9a27abe34c8b022c32209a4abfc8cb443 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 May 2015 10:21:40 +0100 Subject: [PATCH 097/175] Move get_events functions to storage.events --- synapse/storage/_base.py | 155 -------------------------------------- synapse/storage/events.py | 132 ++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 155 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 81052409b..ec80169c5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -15,8 +15,6 @@ import logging from synapse.api.errors import StoreError -from synapse.events import FrozenEvent -from synapse.events.utils import prune_event from synapse.util.logutils import log_function from synapse.util.logcontext import preserve_context_over_fn, LoggingContext from synapse.util.lrucache import LruCache @@ -28,7 +26,6 @@ from twisted.internet import defer from collections import namedtuple, OrderedDict import functools -import simplejson as json import sys import time import threading @@ -867,158 +864,6 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) - def _get_events(self, event_ids, check_redacted=True, - get_prev_content=False): - return self.runInteraction( - "_get_events", self._get_events_txn, event_ids, - check_redacted=check_redacted, get_prev_content=get_prev_content, - ) - - def _get_events_txn(self, txn, event_ids, check_redacted=True, - get_prev_content=False): - if not event_ids: - return [] - - events = [ - self._get_event_txn( - txn, event_id, - check_redacted=check_redacted, - get_prev_content=get_prev_content - ) - for event_id in event_ids - ] - - return [e for e in events if e] - - def _invalidate_get_event_cache(self, event_id): - for check_redacted in (False, True): - for get_prev_content in (False, True): - self._get_event_cache.invalidate(event_id, check_redacted, - get_prev_content) - - def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=False, allow_rejected=False): - - start_time = time.time() * 1000 - - def update_counter(desc, last_time): - curr_time = self._get_event_counters.update(desc, last_time) - sql_getevents_timer.inc_by(curr_time - last_time, desc) - return curr_time - - try: - ret = self._get_event_cache.get(event_id, check_redacted, get_prev_content) - - if allow_rejected or not ret.rejected_reason: - return ret - else: - return None - except KeyError: - pass - finally: - start_time = update_counter("event_cache", start_time) - - sql = ( - "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " - "FROM event_json as e " - "LEFT JOIN redactions as r ON e.event_id = r.redacts " - "LEFT JOIN rejections as rej on rej.event_id = e.event_id " - "WHERE e.event_id = ? " - "LIMIT 1 " - ) - - txn.execute(sql, (event_id,)) - - res = txn.fetchone() - - if not res: - return None - - internal_metadata, js, redacted, rejected_reason = res - - start_time = update_counter("select_event", start_time) - - result = self._get_event_from_row_txn( - txn, internal_metadata, js, redacted, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - rejected_reason=rejected_reason, - ) - self._get_event_cache.prefill(event_id, check_redacted, get_prev_content, result) - - if allow_rejected or not rejected_reason: - return result - else: - return None - - def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, - check_redacted=True, get_prev_content=False, - rejected_reason=None): - - start_time = time.time() * 1000 - - def update_counter(desc, last_time): - curr_time = self._get_event_counters.update(desc, last_time) - sql_getevents_timer.inc_by(curr_time - last_time, desc) - return curr_time - - d = json.loads(js) - start_time = update_counter("decode_json", start_time) - - internal_metadata = json.loads(internal_metadata) - start_time = update_counter("decode_internal", start_time) - - ev = FrozenEvent( - d, - internal_metadata_dict=internal_metadata, - rejected_reason=rejected_reason, - ) - start_time = update_counter("build_frozen_event", start_time) - - if check_redacted and redacted: - ev = prune_event(ev) - - ev.unsigned["redacted_by"] = redacted - # Get the redaction event. - - because = self._get_event_txn( - txn, - redacted, - check_redacted=False - ) - - if because: - ev.unsigned["redacted_because"] = because - start_time = update_counter("redact_event", start_time) - - if get_prev_content and "replaces_state" in ev.unsigned: - prev = self._get_event_txn( - txn, - ev.unsigned["replaces_state"], - get_prev_content=False, - ) - if prev: - ev.unsigned["prev_content"] = prev.get_dict()["content"] - start_time = update_counter("get_prev_content", start_time) - - return ev - - def _parse_events(self, rows): - return self.runInteraction( - "_parse_events", self._parse_events_txn, rows - ) - - def _parse_events_txn(self, txn, rows): - event_ids = [r["event_id"] for r in rows] - - return self._get_events_txn(txn, event_ids) - - def _has_been_redacted_txn(self, txn, event): - sql = "SELECT event_id FROM redactions WHERE redacts = ?" - txn.execute(sql, (event.event_id,)) - result = txn.fetchone() - return result[0] if result else None - def get_next_stream_id(self): with self._next_stream_id_lock: i = self._next_stream_id diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 9242b0a84..afdf0f719 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -17,6 +17,9 @@ from _base import SQLBaseStore, _RollbackButIsFineException from twisted.internet import defer +from synapse.events import FrozenEvent +from synapse.events.utils import prune_event + from synapse.util.logutils import log_function from synapse.api.constants import EventTypes from synapse.crypto.event_signing import compute_event_reference_hash @@ -26,6 +29,7 @@ from syutil.jsonutil import encode_canonical_json from contextlib import contextmanager import logging +import simplejson as json logger = logging.getLogger(__name__) @@ -393,3 +397,131 @@ class EventsStore(SQLBaseStore): return self.runInteraction( "have_events", f, ) + + def _get_events(self, event_ids, check_redacted=True, + get_prev_content=False): + return self.runInteraction( + "_get_events", self._get_events_txn, event_ids, + check_redacted=check_redacted, get_prev_content=get_prev_content, + ) + + def _get_events_txn(self, txn, event_ids, check_redacted=True, + get_prev_content=False): + if not event_ids: + return [] + + events = [ + self._get_event_txn( + txn, event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content + ) + for event_id in event_ids + ] + + return [e for e in events if e] + + def _invalidate_get_event_cache(self, event_id): + for check_redacted in (False, True): + for get_prev_content in (False, True): + self._get_event_cache.invalidate(event_id, check_redacted, + get_prev_content) + + def _get_event_txn(self, txn, event_id, check_redacted=True, + get_prev_content=False, allow_rejected=False): + + try: + ret = self._get_event_cache.get(event_id, check_redacted, get_prev_content) + + if allow_rejected or not ret.rejected_reason: + return ret + else: + return None + except KeyError: + pass + + sql = ( + "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " + "FROM event_json as e " + "LEFT JOIN redactions as r ON e.event_id = r.redacts " + "LEFT JOIN rejections as rej on rej.event_id = e.event_id " + "WHERE e.event_id = ? " + "LIMIT 1 " + ) + + txn.execute(sql, (event_id,)) + + res = txn.fetchone() + + if not res: + return None + + internal_metadata, js, redacted, rejected_reason = res + + result = self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=rejected_reason, + ) + self._get_event_cache.prefill(event_id, check_redacted, get_prev_content, result) + + if allow_rejected or not rejected_reason: + return result + else: + return None + + def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, + check_redacted=True, get_prev_content=False, + rejected_reason=None): + + d = json.loads(js) + internal_metadata = json.loads(internal_metadata) + + ev = FrozenEvent( + d, + internal_metadata_dict=internal_metadata, + rejected_reason=rejected_reason, + ) + + if check_redacted and redacted: + ev = prune_event(ev) + + ev.unsigned["redacted_by"] = redacted + # Get the redaction event. + + because = self._get_event_txn( + txn, + redacted, + check_redacted=False + ) + + if because: + ev.unsigned["redacted_because"] = because + + if get_prev_content and "replaces_state" in ev.unsigned: + prev = self._get_event_txn( + txn, + ev.unsigned["replaces_state"], + get_prev_content=False, + ) + if prev: + ev.unsigned["prev_content"] = prev.get_dict()["content"] + + return ev + + def _parse_events(self, rows): + return self.runInteraction( + "_parse_events", self._parse_events_txn, rows + ) + + def _parse_events_txn(self, txn, rows): + event_ids = [r["event_id"] for r in rows] + + return self._get_events_txn(txn, event_ids) + + def _has_been_redacted_txn(self, txn, event): + sql = "SELECT event_id FROM redactions WHERE redacts = ?" + txn.execute(sql, (event.event_id,)) + result = txn.fetchone() + return result[0] if result else None From 1e90715a3d5f8a910c187dec888283e110a3c04a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 18 May 2015 13:17:36 +0100 Subject: [PATCH 098/175] Make sure the notifier stream token goes forward when it is updated. Sort the pending events by the correct room_stream_id --- synapse/notifier.py | 8 ++++---- synapse/types.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 862b42cfc..0b5d97521 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -88,7 +88,7 @@ class _NotifierUserStream(object): stream_id(str): The new id for the stream the event came from. time_now_ms(int): The current time in milliseconds. """ - self.current_token = self.current_token.copy_and_replace( + self.current_token = self.current_token.copy_and_advance( stream_key, stream_id ) if self.listeners: @@ -192,7 +192,7 @@ class Notifier(object): yield run_on_reactor() self.pending_new_room_events.append(( - event, room_stream_id, extra_users + room_stream_id, event, extra_users )) self._notify_pending_new_room_events(max_room_stream_id) @@ -205,10 +205,10 @@ class Notifier(object): """ pending = sorted(self.pending_new_room_events) self.pending_new_room_events = [] - for event, room_stream_id, extra_users in pending: + for room_stream_id, event, extra_users in pending: if room_stream_id > max_room_stream_id: self.pending_new_room_events.append(( - event, room_stream_id, extra_users + room_stream_id, event, extra_users )) else: self._on_new_room_event(event, room_stream_id, extra_users) diff --git a/synapse/types.py b/synapse/types.py index d89a04f7c..1b21160c5 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -119,6 +119,7 @@ class StreamToken( @property def room_stream_id(self): # TODO(markjh): Awful hack to work around hacks in the presence tests + # which assume that the keys are integers. if type(self.room_key) is int: return self.room_key else: @@ -132,6 +133,22 @@ class StreamToken( or (int(other_token.typing_key) < int(self.typing_key)) ) + def copy_and_advance(self, key, new_value): + """Advance the given key in the token to a new value if and only if the + new value is after the old value. + """ + new_token = self.copy_and_replace(key, new_value) + if key == "room_key": + new_id = new_token.room_stream_id + old_id = self.room_stream_id + else: + new_id = int(getattr(new_token, key)) + old_id = int(getattr(self, key)) + if old_id < new_id: + return new_token + else: + return self + def copy_and_replace(self, key, new_value): d = self._asdict() d[key] = new_value From 755def8083ec887feabcb45b3bc111db4aef20ab Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 18 May 2015 13:46:47 +0100 Subject: [PATCH 099/175] Add more doc string, reduce C+P boilerplate for getting room list --- synapse/handlers/presence.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index a01020e20..ce9dd6439 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -297,7 +297,26 @@ class PresenceHandler(BaseHandler): self.changed_presencelike_data(user, {"last_active": now}) + def get_joined_rooms_for_user(self, user): + """Get the list of rooms a user is joined to. + + Args: + user(UserID): The user. + Returns: + A Deferred of a list of room id strings. + """ + rm_handler = self.homeserver.get_handlers().room_member_handler + return rm_handler.get_joined_rooms_for_user(user) + def changed_presencelike_data(self, user, state): + """Updates the presence state of a local user. + + Args: + user(UserID): The user being updated. + state(dict): The new presence state for the user. + Returns: + A Deferred + """ statuscache = self._get_or_make_usercache(user) self._user_cachemap_latest_serial += 1 @@ -544,8 +563,7 @@ class PresenceHandler(BaseHandler): # Also include people in all my rooms - rm_handler = self.homeserver.get_handlers().room_member_handler - room_ids = yield rm_handler.get_joined_rooms_for_user(user) + room_ids = yield self.get_joined_rooms_for_user(user) if state is None: state = yield self.store.get_presence_state(user.localpart) @@ -745,8 +763,7 @@ class PresenceHandler(BaseHandler): # and also user is informed of server-forced pushes localusers.add(user) - rm_handler = self.homeserver.get_handlers().room_member_handler - room_ids = yield rm_handler.get_joined_rooms_for_user(user) + room_ids = yield self.get_joined_rooms_for_user(user) if not localusers and not room_ids: defer.returnValue(None) @@ -791,8 +808,7 @@ class PresenceHandler(BaseHandler): " | %d interested local observers %r", len(observers), observers ) - rm_handler = self.homeserver.get_handlers().room_member_handler - room_ids = yield rm_handler.get_joined_rooms_for_user(user) + room_ids = yield self.get_joined_rooms_for_user(user) if room_ids: logger.debug(" | %d interested room IDs %r", len(room_ids), room_ids) From 4d1b6f4ad1bbae71c6c941b5951d9f9043ddace4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 May 2015 14:03:46 +0100 Subject: [PATCH 100/175] Remove rejected events if we don't want rejected events --- synapse/storage/events.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 25fb49a28..5a04d45e1 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -643,6 +643,9 @@ class EventsStore(SQLBaseStore): events, ) + if not allow_rejected: + rows[:] = [r for r in rows if not r["rejects"]] + res = yield defer.gatherResults( [ defer.maybeDeferred( From ad31fa304022e593b6c9354d3a42df761597e69b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 18 May 2015 14:04:58 +0100 Subject: [PATCH 101/175] Don't bother sorting by the room_stream_ids, it shouldn't matter which order they are notified in --- synapse/notifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 0b5d97521..1e73d52c4 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -203,7 +203,7 @@ class Notifier(object): max_room_stream_id(int): The highest stream_id below which all events have been persisted. """ - pending = sorted(self.pending_new_room_events) + pending = self.pending_new_room_events self.pending_new_room_events = [] for room_stream_id, event, extra_users in pending: if room_stream_id > max_room_stream_id: From 65878a2319366b268a98dcff48eeb632ef67cc0a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 May 2015 14:06:30 +0100 Subject: [PATCH 102/175] Remove unused metric --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d1f050394..0f146998d 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -46,7 +46,6 @@ sql_scheduling_timer = metrics.register_distribution("schedule_time") sql_query_timer = metrics.register_distribution("query_time", labels=["verb"]) sql_txn_timer = metrics.register_distribution("transaction_time", labels=["desc"]) -sql_getevents_timer = metrics.register_distribution("getEvents_time", labels=["desc"]) caches_by_name = {} cache_counter = metrics.register_cache( From 165eb2dbe63a42ef5fdd947544e8d7fda0e7234f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 May 2015 15:18:41 +0100 Subject: [PATCH 103/175] Comments and shuffle of functions --- synapse/storage/events.py | 80 ++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 5a04d45e1..9751f024d 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -402,6 +402,10 @@ class EventsStore(SQLBaseStore): @defer.inlineCallbacks def _get_events(self, event_ids, check_redacted=True, get_prev_content=False, allow_rejected=False, txn=None): + """Gets a collection of events. If `txn` is not None the we use the + current transaction to fetch events and we return a deferred that is + guarenteed to have resolved. + """ if not event_ids: defer.returnValue([]) @@ -490,16 +494,10 @@ class EventsStore(SQLBaseStore): return event_map - def _fetch_events_txn(self, txn, events, check_redacted=True, - get_prev_content=False, allow_rejected=False): - return unwrap_deferred(self._fetch_events( - txn, events, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - )) - def _do_fetch(self, conn): + """Takes a database connection and waits for requests for events from + the _event_fetch_list queue. + """ event_list = [] i = 0 while True: @@ -532,6 +530,7 @@ class EventsStore(SQLBaseStore): for r in rows } + # We only want to resolve deferreds from the main thread def fire(lst, res): for ids, d in lst: if not d.called: @@ -547,6 +546,7 @@ class EventsStore(SQLBaseStore): except Exception as e: logger.exception("do_fetch") + # We only want to resolve deferreds from the main thread def fire(evs): for _, d in evs: if not d.called: @@ -558,6 +558,10 @@ class EventsStore(SQLBaseStore): @defer.inlineCallbacks def _enqueue_events(self, events, check_redacted=True, get_prev_content=False, allow_rejected=False): + """Fetches events from the database using the _event_fetch_list. This + allows batch and bulk fetching of events - it allows us to fetch events + without having to create a new transaction for each request for events. + """ if not events: defer.returnValue({}) @@ -582,6 +586,9 @@ class EventsStore(SQLBaseStore): rows = yield preserve_context_over_deferred(events_d) + if not allow_rejected: + rows[:] = [r for r in rows if not r["rejects"]] + res = yield defer.gatherResults( [ self._get_event_from_row( @@ -627,49 +634,46 @@ class EventsStore(SQLBaseStore): return rows - @defer.inlineCallbacks - def _fetch_events(self, txn, events, check_redacted=True, - get_prev_content=False, allow_rejected=False): + def _fetch_events_txn(self, txn, events, check_redacted=True, + get_prev_content=False, allow_rejected=False): if not events: - defer.returnValue({}) + return {} - if txn: - rows = self._fetch_event_rows( - txn, events, - ) - else: - rows = yield self.runInteraction( - self._fetch_event_rows, - events, - ) + rows = self._fetch_event_rows( + txn, events, + ) if not allow_rejected: rows[:] = [r for r in rows if not r["rejects"]] - res = yield defer.gatherResults( - [ - defer.maybeDeferred( - self._get_event_from_row, - txn, - row["internal_metadata"], row["json"], row["redacts"], - check_redacted=check_redacted, - get_prev_content=get_prev_content, - rejected_reason=row["rejects"], - ) - for row in rows - ], - consumeErrors=True, - ) + res = [ + unwrap_deferred(self._get_event_from_row( + txn, + row["internal_metadata"], row["json"], row["redacts"], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=row["rejects"], + )) + for row in rows + ] - defer.returnValue({ + return { r.event_id: r for r in res - }) + } @defer.inlineCallbacks def _get_event_from_row(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False, rejected_reason=None): + """This is called when we have a row from the database that we want to + convert into an event. Depending on the given options it may do more + database ops to fill in extra information (e.g. previous content or + rejection reason.) + + `txn` may be None, and if so this creates new transactions for each + database op. + """ d = json.loads(js) internal_metadata = json.loads(internal_metadata) From e1150cac4bceab88097ea2421323f3b3852028e3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 18 May 2015 15:46:37 +0100 Subject: [PATCH 104/175] Move updating the serial and state of the presence cache into a single function --- synapse/handlers/presence.py | 60 ++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 0c3290b30..d129d4ca8 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -308,6 +308,11 @@ class PresenceHandler(BaseHandler): rm_handler = self.homeserver.get_handlers().room_member_handler return rm_handler.get_joined_rooms_for_user(user) + def get_joined_users_for_room_id(self, room_id): + rm_handler = self.homeserver.get_handlers().room_member_handler + return rm_handler.get_room_members(room_id) + + @defer.inlineCallbacks def changed_presencelike_data(self, user, state): """Updates the presence state of a local user. @@ -317,12 +322,9 @@ class PresenceHandler(BaseHandler): Returns: A Deferred """ - statuscache = self._get_or_make_usercache(user) - self._user_cachemap_latest_serial += 1 - statuscache.update(state, serial=self._user_cachemap_latest_serial) - - return self.push_presence(user, statuscache=statuscache) + statuscache = yield self.update_presence_cache(user, state) + yield self.push_presence(user, statuscache=statuscache) @log_function def started_user_eventstream(self, user): @@ -345,13 +347,12 @@ class PresenceHandler(BaseHandler): room_id(str): The room id the user joined. """ if self.hs.is_mine(user): - statuscache = self._get_or_make_usercache(user) - # No actual update but we need to bump the serial anyway for the # event source self._user_cachemap_latest_serial += 1 - statuscache.update({}, serial=self._user_cachemap_latest_serial) - + statuscache = yield self.update_presence_cache( + user, room_ids=[room_id] + ) self.push_update_to_local_and_remote( observed_user=user, room_ids=[room_id], @@ -359,16 +360,17 @@ class PresenceHandler(BaseHandler): ) # We also want to tell them about current presence of people. - rm_handler = self.homeserver.get_handlers().room_member_handler - curr_users = yield rm_handler.get_room_members(room_id) + curr_users = yield self.get_joined_users_for_room_id(room_id) for local_user in [c for c in curr_users if self.hs.is_mine(c)]: - statuscache = self._get_or_offline_usercache(local_user) - statuscache.update({}, serial=self._user_cachemap_latest_serial) + statuscache = yield self.update_presence_cache( + local_user, room_ids=[room_id], add_to_cache=False + ) + self.push_update_to_local_and_remote( observed_user=local_user, users_to_push=[user], - statuscache=self._get_or_offline_usercache(local_user), + statuscache=statuscache, ) @defer.inlineCallbacks @@ -829,10 +831,8 @@ class PresenceHandler(BaseHandler): self.clock.time_msec() - state.pop("last_active_ago") ) - statuscache = self._get_or_make_usercache(user) - self._user_cachemap_latest_serial += 1 - statuscache.update(state, serial=self._user_cachemap_latest_serial) + yield self.update_presence_cache(user, state, room_ids=room_ids) if not observers and not room_ids: logger.debug(" | no interested observers or room IDs") @@ -890,6 +890,32 @@ class PresenceHandler(BaseHandler): yield defer.DeferredList(deferreds, consumeErrors=True) + @defer.inlineCallbacks + def update_presence_cache(self, user, state={}, room_ids=None, + add_to_cache=True): + """Update the presence cache for a user with a new state and bump the + serial to the latest value. + + Args: + user(UserID): The user being updated + state(dict): The presence state being updated + room_ids(None or list of str): A list of room_ids to update. If + room_ids is None then fetch the list of room_ids the user is + joined to. + add_to_cache: Whether to add an entry to the presence cache if the + user isn't already in the cache. + Returns: + A Deferred UserPresenceCache for the user being updated. + """ + if room_ids is None: + room_ids = yield self.get_joined_rooms_for_user(user) + if add_to_cache: + statuscache = self._get_or_make_usercache(user) + else: + statuscache = self._get_or_offline_usercache(user) + statuscache.update(state, serial=self._user_cachemap_latest_serial) + defer.returnValue(statuscache) + @defer.inlineCallbacks def push_update_to_local_and_remote(self, observed_user, statuscache, users_to_push=[], room_ids=[], From 591c4bf223a4a8698f51ba258984e769f593e32b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 18 May 2015 16:21:51 +0100 Subject: [PATCH 105/175] Cache the most recent serial for each room --- synapse/handlers/presence.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index d129d4ca8..aa1d73f2f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -146,6 +146,10 @@ class PresenceHandler(BaseHandler): self._user_cachemap = {} self._user_cachemap_latest_serial = 0 + # map room_ids to the latest presence serial for a member of that + # room + self._room_serials = {} + metrics.register_callback( "userCachemap:size", lambda: len(self._user_cachemap), @@ -909,6 +913,9 @@ class PresenceHandler(BaseHandler): """ if room_ids is None: room_ids = yield self.get_joined_rooms_for_user(user) + + for room_id in room_ids: + self._room_serials[room_id] = self._user_cachemap_latest_serial if add_to_cache: statuscache = self._get_or_make_usercache(user) else: @@ -1069,8 +1076,6 @@ class PresenceEventSource(object): def get_new_events_for_user(self, user, from_key, limit): from_key = int(from_key) - observer_user = user - presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap @@ -1079,17 +1084,28 @@ class PresenceEventSource(object): clock = self.clock latest_serial = 0 + presence_list = yield presence.store.get_presence_list( + user.localpart, accepted=True + ) + if presence_list is None: + presence_list = () + user_ids_to_check = set( + UserID.from_string(p["observed_user_id"]) for p in presence_list + ) + room_ids = yield presence.get_joined_rooms_for_user(user) + for room_id in set(room_ids) & set(presence._room_serials): + if presence._room_serials[room_id] > from_key: + joined = yield presence.get_joined_users_for_room_id(room_id) + user_ids_to_check |= set(joined) + updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. - for observed_user in cachemap.keys(): + for observed_user in user_ids_to_check & set(cachemap): cached = cachemap[observed_user] if cached.serial <= from_key or cached.serial > max_serial: continue - if not (yield self.is_visible(observer_user, observed_user)): - continue - latest_serial = max(cached.serial, latest_serial) updates.append(cached.make_event(user=observed_user, clock=clock)) From ef910a0358d1a1bd608576cfc07edc0a4f2649aa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 May 2015 17:17:04 +0100 Subject: [PATCH 106/175] Do work in parellel when joining a room --- synapse/handlers/federation.py | 69 ++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 880cbd77e..78f2bfc21 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -516,30 +516,59 @@ class FederationHandler(BaseHandler): # FIXME pass + auth_ids_to_deferred = {} + + def process_auth_ev(ev): + auth_ids = [e_id for e_id, _ in ev.auth_events] + + prev_ds = [ + auth_ids_to_deferred[i] + for i in auth_ids + if i in auth_ids_to_deferred + ] + + d = defer.Deferred() + + auth_ids_to_deferred[ev.event_id] = d + + @defer.inlineCallbacks + def f(*_): + ev.internal_metadata.outlier = True + + try: + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + + yield self._handle_new_event( + origin, ev, auth_events=auth + ) + except: + logger.exception( + "Failed to handle auth event %s", + ev.event_id, + ) + + d.callback(None) + + if prev_ds: + dx = defer.DeferredList(prev_ds) + dx.addBoth(f) + else: + f() + for e in auth_chain: - e.internal_metadata.outlier = True - if e.event_id == event.event_id: - continue + return + process_auth_ev(e) - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - yield self._handle_new_event( - origin, e, auth_events=auth - ) - except: - logger.exception( - "Failed to handle auth event %s", - e.event_id, - ) + yield defer.DeferredList(auth_ids_to_deferred.values()) - for e in state: + @defer.inlineCallbacks + def handle_state(e): if e.event_id == event.event_id: - continue + return e.internal_metadata.outlier = True try: @@ -557,6 +586,8 @@ class FederationHandler(BaseHandler): e.event_id, ) + yield defer.DeferredList([handle_state(e) for e in state]) + auth_ids = [e_id for e_id, _ in event.auth_events] auth_events = { (e.type, e.state_key): e for e in auth_chain From e4c65b338d44fadc058cbd8e4cd79ae1601d3526 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 18 May 2015 18:21:06 +0100 Subject: [PATCH 107/175] Speed up the get_pagination_rows as well --- synapse/handlers/presence.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index aa1d73f2f..6537a3738 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1154,14 +1154,28 @@ class PresenceEventSource(object): presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap + user_ids_to_check = {user} + presence_list = yield presence.store.get_presence_list( + user.localpart, accepted=True + ) + if presence_list is None: + presence_list = () + user_ids_to_check |= set( + UserID.from_string(p["observed_user_id"]) for p in presence_list + ) + room_ids = yield presence.get_joined_rooms_for_user(user) + for room_id in set(room_ids) & set(presence._room_serials): + if presence._room_serials[room_id] >= from_key: + joined = yield presence.get_joined_users_for_room_id(room_id) + user_ids_to_check |= set(joined) + updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. - for observed_user in cachemap.keys(): + for observed_user in user_ids_to_check & set(cachemap): if not (to_key < cachemap[observed_user].serial <= from_key): continue - if (yield self.is_visible(observer_user, observed_user)): - updates.append((observed_user, cachemap[observed_user])) + updates.append((observed_user, cachemap[observed_user])) # TODO(paul): limit From c6a03c46e644061155587c141e75f44dc028ed75 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 19 May 2015 10:23:02 +0100 Subject: [PATCH 108/175] SYN-383: Extract the response list from 'server_keys' in the response JSON as it might work better than iterating over the top level dict --- synapse/crypto/keyring.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a859872ce..c7e47f6bf 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -188,7 +188,7 @@ class Keyring(object): # TODO(mark): Set the minimum_valid_until_ts to that needed by # the events being validated or the current time if validating # an incoming request. - responses = yield self.client.post_json( + query_response = yield self.client.post_json( destination=perspective_name, path=b"/_matrix/key/v2/query", data={ @@ -204,6 +204,8 @@ class Keyring(object): keys = {} + responses = query_response["server_keys"] + for response in responses: if (u"signatures" not in response or perspective_name not in response[u"signatures"]): From 2aeee2a90565969f4066783167ad8d6bb530f9d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 11:56:18 +0100 Subject: [PATCH 109/175] SYN-383: Fix parsing of verify_keys and catching of _DefGen_Return --- synapse/crypto/keyring.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index c7e47f6bf..c626f78f4 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -129,23 +129,24 @@ class Keyring(object): def _get_server_verify_key_impl(self, server_name, key_ids): keys = None - perspective_results = [] - for perspective_name, perspective_keys in self.perspective_servers.items(): - @defer.inlineCallbacks - def get_key(): - try: - result = yield self.get_server_verify_key_v2_indirect( - server_name, key_ids, perspective_name, perspective_keys - ) - defer.returnValue(result) - except: - logging.info( - "Unable to getting key %r for %r from %r", - key_ids, server_name, perspective_name, - ) - perspective_results.append(get_key()) + @defer.inlineCallbacks + def get_key(perspective_name, perspective_keys): + try: + result = yield self.get_server_verify_key_v2_indirect( + server_name, key_ids, perspective_name, perspective_keys + ) + defer.returnValue(result) + except Exception as e: + logging.info( + "Unable to getting key %r for %r from %r: %s %s", + key_ids, server_name, perspective_name, + type(e).__name__, str(e.message), + ) - perspective_results = yield defer.gatherResults(perspective_results) + perspective_results = yield defer.gatherResults([ + get_key(name, keys) + for name, keys in self.perspective_servers.items() + ]) for results in perspective_results: if results is not None: @@ -311,9 +312,8 @@ class Keyring(object): time_now_ms = self.clock.time_msec() response_keys = {} verify_keys = {} - for key_id, key_data in response_json["verify_keys"].items(): + for key_id, key_base64 in response_json["verify_keys"].items(): if is_signing_algorithm_supported(key_id): - key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) verify_key = decode_verify_key_bytes(key_id, key_bytes) verify_key.time_added = time_now_ms From 62ccc6d95f0ce1f9f23d7458b2d7f950360b8fb9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 11:58:04 +0100 Subject: [PATCH 110/175] Don't reuse var names --- synapse/crypto/keyring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index c626f78f4..1f24e58ba 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -144,8 +144,8 @@ class Keyring(object): ) perspective_results = yield defer.gatherResults([ - get_key(name, keys) - for name, keys in self.perspective_servers.items() + get_key(p_name, p_keys) + for p_name, p_keys in self.perspective_servers.items() ]) for results in perspective_results: From 8b256a72961ec5fc6ae912dff8190c421e2419df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 11:58:04 +0100 Subject: [PATCH 111/175] Don't reuse var names --- synapse/crypto/keyring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index c626f78f4..1f24e58ba 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -144,8 +144,8 @@ class Keyring(object): ) perspective_results = yield defer.gatherResults([ - get_key(name, keys) - for name, keys in self.perspective_servers.items() + get_key(p_name, p_keys) + for p_name, p_keys in self.perspective_servers.items() ]) for results in perspective_results: From 2b7120e233999c4c89edb46b2714238e828aacc6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 12:49:38 +0100 Subject: [PATCH 112/175] SYN-383: Handle the fact the server might not have signed things --- synapse/app/homeserver.py | 6 +++--- synapse/crypto/keyring.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index dfb5314ff..2e11ac062 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -85,10 +85,10 @@ class SynapseHomeServer(HomeServer): return MatrixFederationHttpClient(self) def build_resource_for_client(self): - return gz_wrap(ClientV1RestResource(self)) + return ClientV1RestResource(self) def build_resource_for_client_v2_alpha(self): - return gz_wrap(ClientV2AlphaRestResource(self)) + return ClientV2AlphaRestResource(self) def build_resource_for_federation(self): return JsonResource(self) @@ -97,7 +97,7 @@ class SynapseHomeServer(HomeServer): import syweb syweb_path = os.path.dirname(syweb.__file__) webclient_path = os.path.join(syweb_path, "webclient") - return GzipFile(webclient_path) # TODO configurable? + return File(webclient_path) # TODO configurable? def build_resource_for_static_content(self): # This is old and should go away: not going to bother adding gzip diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 1f24e58ba..a061def16 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -329,7 +329,7 @@ class Keyring(object): verify_key.time_added = time_now_ms old_verify_keys[key_id] = verify_key - for key_id in response_json["signatures"][server_name]: + for key_id in response_json["signatures"].get(server_name, {}): if key_id not in response_json["verify_keys"]: raise ValueError( "Key response must include verification keys for all" From 350b88656ab7a4ff871dfebc85b9db6d294a4295 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 13:01:57 +0100 Subject: [PATCH 113/175] SYN-383: Actually, we expect this value to be a dict --- synapse/crypto/keyring.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a061def16..2a5a8914c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -312,8 +312,9 @@ class Keyring(object): time_now_ms = self.clock.time_msec() response_keys = {} verify_keys = {} - for key_id, key_base64 in response_json["verify_keys"].items(): + for key_id, key_data in response_json["verify_keys"].items(): if is_signing_algorithm_supported(key_id): + key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) verify_key = decode_verify_key_bytes(key_id, key_bytes) verify_key.time_added = time_now_ms From 677be13ffc5c463acacf2d420ca9c47f2f7cba65 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 13:12:28 +0100 Subject: [PATCH 114/175] Revert accidental commit --- synapse/app/homeserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2e11ac062..dfb5314ff 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -85,10 +85,10 @@ class SynapseHomeServer(HomeServer): return MatrixFederationHttpClient(self) def build_resource_for_client(self): - return ClientV1RestResource(self) + return gz_wrap(ClientV1RestResource(self)) def build_resource_for_client_v2_alpha(self): - return ClientV2AlphaRestResource(self) + return gz_wrap(ClientV2AlphaRestResource(self)) def build_resource_for_federation(self): return JsonResource(self) @@ -97,7 +97,7 @@ class SynapseHomeServer(HomeServer): import syweb syweb_path = os.path.dirname(syweb.__file__) webclient_path = os.path.join(syweb_path, "webclient") - return File(webclient_path) # TODO configurable? + return GzipFile(webclient_path) # TODO configurable? def build_resource_for_static_content(self): # This is old and should go away: not going to bother adding gzip From d3e09f12d041c39a8b18df79abeb6383c708f73d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 13:12:41 +0100 Subject: [PATCH 115/175] SYN-383: Actually, we expect this value to be a dict --- synapse/crypto/keyring.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a061def16..2a5a8914c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -312,8 +312,9 @@ class Keyring(object): time_now_ms = self.clock.time_msec() response_keys = {} verify_keys = {} - for key_id, key_base64 in response_json["verify_keys"].items(): + for key_id, key_data in response_json["verify_keys"].items(): if is_signing_algorithm_supported(key_id): + key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) verify_key = decode_verify_key_bytes(key_id, key_bytes) verify_key.time_added = time_now_ms From 882ac83d8d314c5e1490197aaadd19efcbaaf168 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 13:12:55 +0100 Subject: [PATCH 116/175] Fix scripts-dev/convert_server_keys.py to have correct format --- scripts-dev/convert_server_keys.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py index 024ddcdbd..7080ec5fd 100644 --- a/scripts-dev/convert_server_keys.py +++ b/scripts-dev/convert_server_keys.py @@ -47,7 +47,10 @@ def convert_v1_to_v2(server_name, valid_until, keys, certificate): return { "old_verify_keys": {}, "server_name": server_name, - "verify_keys": keys, + "verify_keys": { + key_id: {"key": key} + for key_id, key in keys + }, "valid_until_ts": valid_until, "tls_fingerprints": [fingerprint(certificate)], } From df431b127b54df3d5e5c398e48a134dc66d29ebd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 13:14:21 +0100 Subject: [PATCH 117/175] Add forgotten .items() --- scripts-dev/convert_server_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py index 7080ec5fd..a1ee39059 100644 --- a/scripts-dev/convert_server_keys.py +++ b/scripts-dev/convert_server_keys.py @@ -49,7 +49,7 @@ def convert_v1_to_v2(server_name, valid_until, keys, certificate): "server_name": server_name, "verify_keys": { key_id: {"key": key} - for key_id, key in keys + for key_id, key in keys.items() }, "valid_until_ts": valid_until, "tls_fingerprints": [fingerprint(certificate)], From 19505e03921e2e239e99233f3faae083c105501a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 19 May 2015 13:19:47 +0100 Subject: [PATCH 118/175] Disable GZip encoding on static file resources as per comment --- synapse/app/homeserver.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index dfb5314ff..fa4321141 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -97,7 +97,13 @@ class SynapseHomeServer(HomeServer): import syweb syweb_path = os.path.dirname(syweb.__file__) webclient_path = os.path.join(syweb_path, "webclient") - return GzipFile(webclient_path) # TODO configurable? + # GZip is disabled here due to + # https://twistedmatrix.com/trac/ticket/7678 + # (It can stay enabled for the API resources: they call + # write() with the whole body and then finish() straight + # after and so do not trigger the bug. + # return GzipFile(webclient_path) # TODO configurable? + return File(webclient_path) # TODO configurable? def build_resource_for_static_content(self): # This is old and should go away: not going to bother adding gzip From 5ae4a84211e4ca0247ab3bca77b159f843d6ead2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 13:43:34 +0100 Subject: [PATCH 119/175] Don't always hit get_server_verify_key_v1_direct --- synapse/crypto/keyring.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 2a5a8914c..35f9ac351 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -164,12 +164,17 @@ class Keyring(object): keys = yield self.get_server_verify_key_v2_direct( server_name, key_ids ) - except: - pass + except Exception as e: + logging.info( + "Unable to getting key %r for %r directly: %s %s", + key_ids, server_name, + type(e).__name__, str(e.message), + ) - keys = yield self.get_server_verify_key_v1_direct( - server_name, key_ids - ) + if keys is None: + keys = yield self.get_server_verify_key_v1_direct( + server_name, key_ids + ) for key_id in key_ids: if key_id in keys: From 722312991694846045fae31ec6e0cbbbc59c6a33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 14:15:05 +0100 Subject: [PATCH 120/175] Don't apply new room join hack if depth > 5 --- synapse/handlers/federation.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 03f8444ad..d85b1cf5d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -924,9 +924,12 @@ class FederationHandler(BaseHandler): # This is a hack to fix some old rooms where the initial join event # didn't reference the create event in its auth events. if event.type == EventTypes.Member and not event.auth_events: - if len(event.prev_events) == 1: - c = yield self.store.get_event(event.prev_events[0][0]) - if c.type == EventTypes.Create: + if len(event.prev_events) == 1 and event.depth < 5: + c = yield self.store.get_event( + event.prev_events[0][0], + allow_none=True, + ) + if c and c.type == EventTypes.Create: auth_events[(c.type, c.state_key)] = c try: From 6837c5edab94df93addc85d1900011ba2182e0f1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 14:27:11 +0100 Subject: [PATCH 121/175] Handle the case when things return empty but non none things --- synapse/crypto/keyring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 35f9ac351..aff69c5f8 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -159,7 +159,7 @@ class Keyring(object): ) with limiter: - if keys is None: + if not keys: try: keys = yield self.get_server_verify_key_v2_direct( server_name, key_ids @@ -171,7 +171,7 @@ class Keyring(object): type(e).__name__, str(e.message), ) - if keys is None: + if not keys: keys = yield self.get_server_verify_key_v1_direct( server_name, key_ids ) From 253f76a0a544982cee2e908bf73ad450c7321fcb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 13:43:34 +0100 Subject: [PATCH 122/175] Don't always hit get_server_verify_key_v1_direct --- synapse/crypto/keyring.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 2a5a8914c..35f9ac351 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -164,12 +164,17 @@ class Keyring(object): keys = yield self.get_server_verify_key_v2_direct( server_name, key_ids ) - except: - pass + except Exception as e: + logging.info( + "Unable to getting key %r for %r directly: %s %s", + key_ids, server_name, + type(e).__name__, str(e.message), + ) - keys = yield self.get_server_verify_key_v1_direct( - server_name, key_ids - ) + if keys is None: + keys = yield self.get_server_verify_key_v1_direct( + server_name, key_ids + ) for key_id in key_ids: if key_id in keys: From 291cba284bc79c68caa73c09a382e5bea38375d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 14:27:11 +0100 Subject: [PATCH 123/175] Handle the case when things return empty but non none things --- synapse/crypto/keyring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 35f9ac351..aff69c5f8 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -159,7 +159,7 @@ class Keyring(object): ) with limiter: - if keys is None: + if not keys: try: keys = yield self.get_server_verify_key_v2_direct( server_name, key_ids @@ -171,7 +171,7 @@ class Keyring(object): type(e).__name__, str(e.message), ) - if keys is None: + if not keys: keys = yield self.get_server_verify_key_v1_direct( server_name, key_ids ) From 5b1631a4a9ad4c1ed0adaff3ffc8238014359e95 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 14:53:32 +0100 Subject: [PATCH 124/175] Add a timeout param to get_event --- synapse/federation/federation_base.py | 1 + synapse/federation/federation_client.py | 23 ++++++++++++++--------- synapse/federation/transport/client.py | 4 ++-- synapse/http/matrixfederationclient.py | 13 ++++++++----- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 5217d91aa..f0430b2cb 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -80,6 +80,7 @@ class FederationBase(object): destinations=[pdu.origin], event_id=pdu.event_id, outlier=outlier, + timeout=10000, ) if new_pdu: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 904c7c094..a163b2674 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -22,6 +22,7 @@ from .units import Edu from synapse.api.errors import ( CodeMessageException, HttpResponseException, SynapseError, ) +from synapse.util import unwrapFirstError from synapse.util.expiringcache import ExpiringCache from synapse.util.logutils import log_function from synapse.events import FrozenEvent @@ -173,7 +174,7 @@ class FederationClient(FederationBase): @defer.inlineCallbacks @log_function - def get_pdu(self, destinations, event_id, outlier=False): + def get_pdu(self, destinations, event_id, outlier=False, timeout=None): """Requests the PDU with given origin and ID from the remote home servers. @@ -212,7 +213,7 @@ class FederationClient(FederationBase): with limiter: transaction_data = yield self.transport_layer.get_event( - destination, event_id + destination, event_id, timeout=timeout, ) logger.debug("transaction_data %r", transaction_data) @@ -370,13 +371,17 @@ class FederationClient(FederationBase): for p in content.get("auth_chain", []) ] - signed_state = yield self._check_sigs_and_hash_and_fetch( - destination, state, outlier=True - ) - - signed_auth = yield self._check_sigs_and_hash_and_fetch( - destination, auth_chain, outlier=True - ) + signed_state, signed_auth = yield defer.gatherResults( + [ + self._check_sigs_and_hash_and_fetch( + destination, state, outlier=True + ), + self._check_sigs_and_hash_and_fetch( + destination, auth_chain, outlier=True + ) + ], + consumeErrors=True + ).addErrback(unwrapFirstError) auth_chain.sort(key=lambda e: e.depth) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 80d03012b..c2b53b78b 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -50,7 +50,7 @@ class TransportLayerClient(object): ) @log_function - def get_event(self, destination, event_id): + def get_event(self, destination, event_id, timeout=None): """ Requests the pdu with give id and origin from the given server. Args: @@ -65,7 +65,7 @@ class TransportLayerClient(object): destination, event_id) path = PREFIX + "/event/%s/" % (event_id, ) - return self.client.get_json(destination, path=path) + return self.client.get_json(destination, path=path, timeout=timeout) @log_function def backfill(self, destination, room_id, event_tuples, limit): diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c99d237c7..312bbcc6b 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -110,7 +110,8 @@ class MatrixFederationHttpClient(object): @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, body_callback, headers_dict={}, param_bytes=b"", - query_bytes=b"", retry_on_dns_fail=True): + query_bytes=b"", retry_on_dns_fail=True, + timeout=None): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [self.version_string] @@ -158,7 +159,7 @@ class MatrixFederationHttpClient(object): response = yield self.clock.time_bound_deferred( request_deferred, - time_out=60, + time_out=timeout/1000. if timeout else 60, ) logger.debug("Got response to %s", method) @@ -181,7 +182,7 @@ class MatrixFederationHttpClient(object): _flatten_response_never_received(e), ) - if retries_left: + if retries_left and not timeout: yield sleep(2 ** (5 - retries_left)) retries_left -= 1 else: @@ -334,7 +335,8 @@ class MatrixFederationHttpClient(object): defer.returnValue(json.loads(body)) @defer.inlineCallbacks - def get_json(self, destination, path, args={}, retry_on_dns_fail=True): + def get_json(self, destination, path, args={}, retry_on_dns_fail=True, + timeout=None): """ GETs some json from the given host homeserver and path Args: @@ -370,7 +372,8 @@ class MatrixFederationHttpClient(object): path.encode("ascii"), query_bytes=query_bytes, body_callback=body_callback, - retry_on_dns_fail=retry_on_dns_fail + retry_on_dns_fail=retry_on_dns_fail, + timeout=timeout, ) if 200 <= response.code < 300: From aa729349ddf23c0abb5096581a317783a8f60aa6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 15:27:00 +0100 Subject: [PATCH 125/175] Fix event_backwards_extrem insertion to ignore outliers --- synapse/storage/event_federation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index a1982dfbb..288085050 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -337,12 +337,13 @@ class EventFederationStore(SQLBaseStore): " WHERE event_id = ? AND room_id = ?" " )" " AND NOT EXISTS (" - " SELECT 1 FROM events WHERE event_id = ? AND room_id = ?" + " SELECT 1 FROM events WHERE event_id = ? AND room_id = ? " + " AND outlier = ?" " )" ) txn.executemany(query, [ - (e_id, room_id, e_id, room_id, e_id, room_id, ) + (e_id, room_id, e_id, room_id, e_id, room_id, False) for e_id, _ in prev_events ]) From 3a653515ec3bba9d5b143e37bc9569d5caa50a5b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 15:27:09 +0100 Subject: [PATCH 126/175] Add None check --- synapse/federation/federation_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index a163b2674..4b3bf9783 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -523,7 +523,7 @@ class FederationClient(FederationBase): # Are we missing any? seen_events = set(earliest_events_ids) - seen_events.update(e.event_id for e in signed_events) + seen_events.update(e.event_id for e in signed_events if e) missing_events = {} for e in itertools.chain(latest_events, signed_events): From 9084cdd70f0b9c03de4e72897cabd30f545b6cb5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 May 2015 16:34:31 +0100 Subject: [PATCH 127/175] Ensure event_results is a set --- synapse/storage/event_federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 288085050..2f913adf2 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -378,7 +378,7 @@ class EventFederationStore(SQLBaseStore): room_id, repr(event_list), limit ) - event_results = event_list + event_results = set(event_list) front = event_list From 20814fabdd001ee6a04efc5277d71e80fdbf5a14 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 May 2015 11:59:02 +0100 Subject: [PATCH 128/175] Actually fetch state for new backwards extremeties when backfilling. --- synapse/federation/federation_client.py | 6 +- synapse/handlers/federation.py | 164 +++++++++++++++--------- 2 files changed, 108 insertions(+), 62 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 4b3bf9783..6febc8618 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -168,7 +168,11 @@ class FederationClient(FederationBase): for i, pdu in enumerate(pdus): pdus[i] = yield self._check_sigs_and_hash(pdu) - # FIXME: We should handle signature failures more gracefully. + # FIXME: We should handle signature failures more gracefully. + pdus[:] = yield defer.gatherResults( + [self._check_sigs_and_hash(pdu) for pdu in pdus], + consumeErrors=True, + ).addErrback(unwrapFirstError) defer.returnValue(pdus) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d85b1cf5d..46ce3699d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -230,27 +230,65 @@ class FederationHandler(BaseHandler): if not extremities: extremities = yield self.store.get_oldest_events_in_room(room_id) - pdus = yield self.replication_layer.backfill( + events = yield self.replication_layer.backfill( dest, room_id, - limit, + limit=limit, extremities=extremities, ) - events = [] + event_map = {e.event_id: e for e in events} - for pdu in pdus: - event = pdu + event_ids = set(e.event_id for e in events) - # FIXME (erikj): Not sure this actually works :/ - context = yield self.state_handler.compute_event_context(event) + edges = [ + ev.event_id + for ev in events + if set(e_id for e_id, _ in ev.prev_events) - event_ids + ] - events.append((event, context)) + # For each edge get the current state. - yield self.store.persist_event( - event, - context=context, - backfilled=True + auth_events = {} + events_to_state = {} + for e_id in edges: + state, auth = yield self.replication_layer.get_state_for_room( + destination=dest, + room_id=room_id, + event_id=e_id + ) + auth_events.update({a.event_id: a for a in auth}) + events_to_state[e_id] = state + + yield defer.gatherResults( + [ + self._handle_new_event(dest, a) + for a in auth_events.values() + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) + + yield defer.gatherResults( + [ + self._handle_new_event( + dest, event_map[e_id], + state=events_to_state[e_id], + backfilled=True, + ) + for e_id in events_to_state + ], + consumeErrors=True + ).addErrback(unwrapFirstError) + + events.sort(key=lambda e: e.depth) + + for event in events: + if event in events_to_state: + continue + + yield self._handle_new_event( + dest, event, + backfilled=True, ) defer.returnValue(events) @@ -347,7 +385,7 @@ class FederationHandler(BaseHandler): logger.info(e.message) continue except Exception as e: - logger.warn( + logger.exception( "Failed to backfill from %s because %s", dom, e, ) @@ -517,54 +555,9 @@ class FederationHandler(BaseHandler): # FIXME pass - auth_ids_to_deferred = {} - - def process_auth_ev(ev): - auth_ids = [e_id for e_id, _ in ev.auth_events] - - prev_ds = [ - auth_ids_to_deferred[i] - for i in auth_ids - if i in auth_ids_to_deferred - ] - - d = defer.Deferred() - - auth_ids_to_deferred[ev.event_id] = d - - @defer.inlineCallbacks - def f(*_): - ev.internal_metadata.outlier = True - - try: - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - - yield self._handle_new_event( - origin, ev, auth_events=auth - ) - except: - logger.exception( - "Failed to handle auth event %s", - ev.event_id, - ) - - d.callback(None) - - if prev_ds: - dx = defer.DeferredList(prev_ds) - dx.addBoth(f) - else: - f() - - for e in auth_chain: - if e.event_id == event.event_id: - return - process_auth_ev(e) - - yield defer.DeferredList(auth_ids_to_deferred.values()) + yield self._handle_auth_events( + origin, [e for e in auth_chain if e.event_id != event.event_id] + ) @defer.inlineCallbacks def handle_state(e): @@ -1348,3 +1341,52 @@ class FederationHandler(BaseHandler): }, "missing": [e.event_id for e in missing_locals], }) + + @defer.inlineCallbacks + def _handle_auth_events(self, origin, auth_events): + auth_ids_to_deferred = {} + + def process_auth_ev(ev): + auth_ids = [e_id for e_id, _ in ev.auth_events] + + prev_ds = [ + auth_ids_to_deferred[i] + for i in auth_ids + if i in auth_ids_to_deferred + ] + + d = defer.Deferred() + + auth_ids_to_deferred[ev.event_id] = d + + @defer.inlineCallbacks + def f(*_): + ev.internal_metadata.outlier = True + + try: + auth = { + (e.type, e.state_key): e for e in auth_events + if e.event_id in auth_ids + } + + yield self._handle_new_event( + origin, ev, auth_events=auth + ) + except: + logger.exception( + "Failed to handle auth event %s", + ev.event_id, + ) + + d.callback(None) + + if prev_ds: + dx = defer.DeferredList(prev_ds) + dx.addBoth(f) + else: + f() + + for e in auth_events: + process_auth_ev(e) + + yield defer.DeferredList(auth_ids_to_deferred.values()) From 2bc60c55af8a77d4cf229c4c1d3a05f488c4af6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 May 2015 12:57:00 +0100 Subject: [PATCH 129/175] Fix _get_backfill_events to return events in the correct order --- synapse/storage/event_federation.py | 55 +++++++++++++++-------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 2f913adf2..34948c30c 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -17,6 +17,7 @@ from ._base import SQLBaseStore, cached from syutil.base64util import encode_base64 import logging +from Queue import PriorityQueue logger = logging.getLogger(__name__) @@ -380,41 +381,41 @@ class EventFederationStore(SQLBaseStore): event_results = set(event_list) - front = event_list + # We want to make sure that we do a breadth-first, "depth" ordered + # search. query = ( - "SELECT prev_event_id FROM event_edges " - "WHERE room_id = ? AND event_id = ? " - "LIMIT ?" + "SELECT depth, prev_event_id FROM event_edges" + " INNER JOIN events" + " ON prev_event_id = events.event_id" + " AND event_edges.room_id = events.room_id" + " WHERE event_edges.room_id = ? AND event_edges.event_id = ?" + " LIMIT ?" ) - # We iterate through all event_ids in `front` to select their previous - # events. These are dumped in `new_front`. - # We continue until we reach the limit *or* new_front is empty (i.e., - # we've run out of things to select - while front and len(event_results) < limit: + queue = PriorityQueue() - new_front = [] - for event_id in front: - logger.debug( - "_backfill_interaction: id=%s", - event_id - ) + for event_id in event_list: + txn.execute( + query, + (room_id, event_id, limit - len(event_results)) + ) - txn.execute( - query, - (room_id, event_id, limit - len(event_results)) - ) + for row in txn.fetchall(): + queue.put(row) - for row in txn.fetchall(): - logger.debug( - "_backfill_interaction: got id=%s", - *row - ) - new_front.append(row[0]) + while not queue.empty() and len(event_results) < limit: + _, event_id = queue.get_nowait() - front = new_front - event_results += new_front + event_results.add(event_id) + + txn.execute( + query, + (room_id, event_id, limit - len(event_results)) + ) + + for row in txn.fetchall(): + queue.put(row) return self._get_events_txn(txn, event_results) From 227f8ef03129a3eb9fe4e4a3f21e397b11032836 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 May 2015 13:00:57 +0100 Subject: [PATCH 130/175] Split out _get_event_from_row back into defer and _txn version --- synapse/storage/events.py | 68 ++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 9751f024d..b0d474b8e 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -592,7 +592,6 @@ class EventsStore(SQLBaseStore): res = yield defer.gatherResults( [ self._get_event_from_row( - None, row["internal_metadata"], row["json"], row["redacts"], check_redacted=check_redacted, get_prev_content=get_prev_content, @@ -647,13 +646,13 @@ class EventsStore(SQLBaseStore): rows[:] = [r for r in rows if not r["rejects"]] res = [ - unwrap_deferred(self._get_event_from_row( + self._get_event_from_row_txn( txn, row["internal_metadata"], row["json"], row["redacts"], check_redacted=check_redacted, get_prev_content=get_prev_content, rejected_reason=row["rejects"], - )) + ) for row in rows ] @@ -663,17 +662,64 @@ class EventsStore(SQLBaseStore): } @defer.inlineCallbacks - def _get_event_from_row(self, txn, internal_metadata, js, redacted, + def _get_event_from_row(self, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False, rejected_reason=None): - """This is called when we have a row from the database that we want to - convert into an event. Depending on the given options it may do more - database ops to fill in extra information (e.g. previous content or - rejection reason.) + d = json.loads(js) + internal_metadata = json.loads(internal_metadata) - `txn` may be None, and if so this creates new transactions for each - database op. - """ + if rejected_reason: + rejected_reason = yield self._simple_select_one_onecol( + table="rejections", + keyvalues={"event_id": rejected_reason}, + retcol="reason", + desc="_get_event_from_row", + ) + + ev = FrozenEvent( + d, + internal_metadata_dict=internal_metadata, + rejected_reason=rejected_reason, + ) + + if check_redacted and redacted: + ev = prune_event(ev) + + redaction_id = yield self._simple_select_one_onecol( + table="redactions", + keyvalues={"redacts": ev.event_id}, + retcol="event_id", + desc="_get_event_from_row", + ) + + ev.unsigned["redacted_by"] = redaction_id + # Get the redaction event. + + because = yield self.get_event( + redaction_id, + check_redacted=False + ) + + if because: + ev.unsigned["redacted_because"] = because + + if get_prev_content and "replaces_state" in ev.unsigned: + prev = yield self.get_event( + ev.unsigned["replaces_state"], + get_prev_content=False, + ) + if prev: + ev.unsigned["prev_content"] = prev.get_dict()["content"] + + self._get_event_cache.prefill( + ev.event_id, check_redacted, get_prev_content, ev + ) + + defer.returnValue(ev) + + def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, + check_redacted=True, get_prev_content=False, + rejected_reason=None): d = json.loads(js) internal_metadata = json.loads(internal_metadata) From f407cbd2f1f034ee2f7be8a9464aa234266420cc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 May 2015 13:02:01 +0100 Subject: [PATCH 131/175] PEP8 --- synapse/storage/events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index b0d474b8e..3f7f546bd 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -718,8 +718,8 @@ class EventsStore(SQLBaseStore): defer.returnValue(ev) def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, - check_redacted=True, get_prev_content=False, - rejected_reason=None): + check_redacted=True, get_prev_content=False, + rejected_reason=None): d = json.loads(js) internal_metadata = json.loads(internal_metadata) From ab45e12d31bf8238c3cb1887d18d89ae2addbb7a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 May 2015 13:07:19 +0100 Subject: [PATCH 132/175] Make not return a deferred _get_event_from_row_txn --- synapse/storage/events.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 3f7f546bd..4aa4e7ab1 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -723,23 +723,8 @@ class EventsStore(SQLBaseStore): d = json.loads(js) internal_metadata = json.loads(internal_metadata) - def select(txn, *args, **kwargs): - if txn: - return self._simple_select_one_onecol_txn(txn, *args, **kwargs) - else: - return self._simple_select_one_onecol( - *args, - desc="_get_event_from_row", **kwargs - ) - - def get_event(txn, *args, **kwargs): - if txn: - return self._get_event_txn(txn, *args, **kwargs) - else: - return self.get_event(*args, **kwargs) - if rejected_reason: - rejected_reason = yield select( + rejected_reason = self._simple_select_one_onecol_txn( txn, table="rejections", keyvalues={"event_id": rejected_reason}, @@ -755,7 +740,7 @@ class EventsStore(SQLBaseStore): if check_redacted and redacted: ev = prune_event(ev) - redaction_id = yield select( + redaction_id = self._simple_select_one_onecol_txn( txn, table="redactions", keyvalues={"redacts": ev.event_id}, @@ -765,7 +750,7 @@ class EventsStore(SQLBaseStore): ev.unsigned["redacted_by"] = redaction_id # Get the redaction event. - because = yield get_event( + because = self._get_event_txn( txn, redaction_id, check_redacted=False @@ -775,7 +760,7 @@ class EventsStore(SQLBaseStore): ev.unsigned["redacted_because"] = because if get_prev_content and "replaces_state" in ev.unsigned: - prev = yield get_event( + prev = self._get_event_txn( txn, ev.unsigned["replaces_state"], get_prev_content=False, @@ -787,7 +772,7 @@ class EventsStore(SQLBaseStore): ev.event_id, check_redacted, get_prev_content, ev ) - defer.returnValue(ev) + return ev def _parse_events(self, rows): return self.runInteraction( From e01b825cc929e16b6a60be0688bbe6d8d9b3866e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 20 May 2015 13:21:59 +0100 Subject: [PATCH 133/175] Clean up the presence_list checking logic a bit --- synapse/handlers/presence.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6537a3738..226d6a0f5 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1084,14 +1084,14 @@ class PresenceEventSource(object): clock = self.clock latest_serial = 0 + user_ids_to_check = {user} presence_list = yield presence.store.get_presence_list( user.localpart, accepted=True ) - if presence_list is None: - presence_list = () - user_ids_to_check = set( - UserID.from_string(p["observed_user_id"]) for p in presence_list - ) + if presence_list is not None: + user_ids_to_check |= set( + UserID.from_string(p["observed_user_id"]) for p in presence_list + ) room_ids = yield presence.get_joined_rooms_for_user(user) for room_id in set(room_ids) & set(presence._room_serials): if presence._room_serials[room_id] > from_key: @@ -1142,8 +1142,6 @@ class PresenceEventSource(object): def get_pagination_rows(self, user, pagination_config, key): # TODO (erikj): Does this make sense? Ordering? - observer_user = user - from_key = int(pagination_config.from_key) if pagination_config.to_key: @@ -1158,11 +1156,10 @@ class PresenceEventSource(object): presence_list = yield presence.store.get_presence_list( user.localpart, accepted=True ) - if presence_list is None: - presence_list = () - user_ids_to_check |= set( - UserID.from_string(p["observed_user_id"]) for p in presence_list - ) + if presence_list is not None: + user_ids_to_check |= set( + UserID.from_string(p["observed_user_id"]) for p in presence_list + ) room_ids = yield presence.get_joined_rooms_for_user(user) for room_id in set(room_ids) & set(presence._room_serials): if presence._room_serials[room_id] >= from_key: From 8eca5bd50abc9deb3cd428f3f6b3b8fbeb8bdee1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 20 May 2015 13:22:18 +0100 Subject: [PATCH 134/175] Fix the presence tests --- tests/handlers/test_presence.py | 13 +++---------- tests/rest/client/v1/test_presence.py | 3 +++ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index ee773797e..12cf5747a 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -624,6 +624,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): """ PRESENCE_LIST = { 'apple': [ "@banana:test", "@clementine:test" ], + 'banana': [ "@apple:test" ], } @defer.inlineCallbacks @@ -836,12 +837,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): @defer.inlineCallbacks def test_recv_remote(self): - # TODO(paul): Gut-wrenching - potato_set = self.handler._remote_recvmap.setdefault(self.u_potato, - set()) - potato_set.add(self.u_apple) - - self.room_members = [self.u_banana, self.u_potato] + self.room_members = [self.u_apple, self.u_banana, self.u_potato] self.assertEquals(self.event_source.get_current_key(), 0) @@ -886,11 +882,8 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): @defer.inlineCallbacks def test_recv_remote_offline(self): """ Various tests relating to SYN-261 """ - potato_set = self.handler._remote_recvmap.setdefault(self.u_potato, - set()) - potato_set.add(self.u_apple) - self.room_members = [self.u_banana, self.u_potato] + self.room_members = [self.u_apple, self.u_banana, self.u_potato] self.assertEquals(self.event_source.get_current_key(), 0) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 29c0038f0..8f3df9241 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -295,6 +295,9 @@ class PresenceEventStreamTestCase(unittest.TestCase): else: return [] hs.handlers.room_member_handler.get_joined_rooms_for_user = get_rooms_for_user + hs.handlers.room_member_handler.get_room_members = ( + lambda r: self.room_members if r == "a-room" else [] + ) self.mock_datastore = hs.get_datastore() self.mock_datastore.get_app_service_by_token = Mock(return_value=None) From 9118a928621ae31a951412b867179f5e0a627976 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 May 2015 13:27:16 +0100 Subject: [PATCH 135/175] Split up _get_events into defer and txn versions --- synapse/storage/events.py | 59 +++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 4aa4e7ab1..656e57b5c 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -19,7 +19,6 @@ from twisted.internet import defer, reactor from synapse.events import FrozenEvent from synapse.events.utils import prune_event -from synapse.util import unwrap_deferred from synapse.util.logcontext import preserve_context_over_deferred from synapse.util.logutils import log_function @@ -401,11 +400,7 @@ class EventsStore(SQLBaseStore): @defer.inlineCallbacks def _get_events(self, event_ids, check_redacted=True, - get_prev_content=False, allow_rejected=False, txn=None): - """Gets a collection of events. If `txn` is not None the we use the - current transaction to fetch events and we return a deferred that is - guarenteed to have resolved. - """ + get_prev_content=False, allow_rejected=False): if not event_ids: defer.returnValue([]) @@ -424,21 +419,12 @@ class EventsStore(SQLBaseStore): if e_id in event_map and event_map[e_id] ]) - if not txn: - missing_events = yield self._enqueue_events( - missing_events_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) - else: - missing_events = self._fetch_events_txn( - txn, - missing_events_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) + missing_events = yield self._enqueue_events( + missing_events_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) event_map.update(missing_events) @@ -449,13 +435,38 @@ class EventsStore(SQLBaseStore): def _get_events_txn(self, txn, event_ids, check_redacted=True, get_prev_content=False, allow_rejected=False): - return unwrap_deferred(self._get_events( + if not event_ids: + return [] + + event_map = self._get_events_from_cache( event_ids, check_redacted=check_redacted, get_prev_content=get_prev_content, allow_rejected=allow_rejected, - txn=txn, - )) + ) + + missing_events_ids = [e for e in event_ids if e not in event_map] + + if not missing_events_ids: + return [ + event_map[e_id] for e_id in event_ids + if e_id in event_map and event_map[e_id] + ] + + missing_events = self._fetch_events_txn( + txn, + missing_events_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + event_map.update(missing_events) + + return [ + event_map[e_id] for e_id in event_ids + if e_id in event_map and event_map[e_id] + ] def _invalidate_get_event_cache(self, event_id): for check_redacted in (False, True): From 7ae8afb7ef5a0fb3162339737682e9248980600d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 20 May 2015 14:48:11 +0100 Subject: [PATCH 136/175] Removed unused 'is_visible' method --- synapse/handlers/presence.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 226d6a0f5..6c48b1d20 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1045,32 +1045,6 @@ class PresenceEventSource(object): self.hs = hs self.clock = hs.get_clock() - @defer.inlineCallbacks - def is_visible(self, observer_user, observed_user): - if observer_user == observed_user: - defer.returnValue(True) - - presence = self.hs.get_handlers().presence_handler - - if (yield presence.store.user_rooms_intersect( - [u.to_string() for u in observer_user, observed_user])): - defer.returnValue(True) - - if self.hs.is_mine(observed_user): - pushmap = presence._local_pushmap - - defer.returnValue( - observed_user.localpart in pushmap and - observer_user in pushmap[observed_user.localpart] - ) - else: - recvmap = presence._remote_recvmap - - defer.returnValue( - observed_user in recvmap and - observer_user in recvmap[observed_user] - ) - @defer.inlineCallbacks @log_function def get_new_events_for_user(self, user, from_key, limit): @@ -1099,7 +1073,6 @@ class PresenceEventSource(object): user_ids_to_check |= set(joined) updates = [] - # TODO(paul): use a DeferredList ? How to limit concurrency. for observed_user in user_ids_to_check & set(cachemap): cached = cachemap[observed_user] From 80a167b1f032ae9bcb75bdb20c2112a779dda516 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 11:18:22 +0100 Subject: [PATCH 137/175] Add comments --- synapse/storage/events.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 656e57b5c..3d798f5fa 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -35,6 +35,16 @@ import simplejson as json logger = logging.getLogger(__name__) +# These values are used in the `enqueus_event` and `_do_fetch` methods to +# control how we batch/bulk fetch events from the database. +# The values are plucked out of thing air to make initial sync run faster +# on jki.re +# TODO: Make these configurable. +EVENT_QUEUE_THREADS = 3 # Max number of threads that will fetch events +EVENT_QUEUE_ITERATIONS = 3 # No. times we block waiting for events +EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for events + + class EventsStore(SQLBaseStore): @defer.inlineCallbacks @log_function @@ -518,11 +528,12 @@ class EventsStore(SQLBaseStore): self._event_fetch_list = [] if not event_list: - if self.database_engine.single_threaded or i > 3: + single_threaded = self.database_engine.single_threaded + if single_threaded or i > EVENT_QUEUE_ITERATIONS: self._event_fetch_ongoing -= 1 return else: - self._event_fetch_lock.wait(0.1) + self._event_fetch_lock.wait(EVENT_QUEUE_TIMEOUT_S) i += 1 continue i = 0 @@ -584,7 +595,7 @@ class EventsStore(SQLBaseStore): self._event_fetch_lock.notify() - if self._event_fetch_ongoing < 3: + if self._event_fetch_ongoing < EVENT_QUEUE_THREADS: self._event_fetch_ongoing += 1 should_start = True else: From ac5f2bf9db9f37d2cb076cfa3a0912ecb948f508 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 14:50:57 +0100 Subject: [PATCH 138/175] s/for events/for requests for events/ --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 3d798f5fa..84a799bcf 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -42,7 +42,7 @@ logger = logging.getLogger(__name__) # TODO: Make these configurable. EVENT_QUEUE_THREADS = 3 # Max number of threads that will fetch events EVENT_QUEUE_ITERATIONS = 3 # No. times we block waiting for events -EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for events +EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for requests for events class EventsStore(SQLBaseStore): From 27e4b45c0692489a11e4be2b05f3fcb99b3c7489 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 14:52:23 +0100 Subject: [PATCH 139/175] s/for events/for requests for events/ --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 84a799bcf..74946eac3 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) # on jki.re # TODO: Make these configurable. EVENT_QUEUE_THREADS = 3 # Max number of threads that will fetch events -EVENT_QUEUE_ITERATIONS = 3 # No. times we block waiting for events +EVENT_QUEUE_ITERATIONS = 3 # No. times we block waiting for requests for events EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for requests for events From 88f1ea36cedff158e7a595c84bf949286fa38532 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 21 May 2015 15:23:40 +0100 Subject: [PATCH 140/175] Oops, get_rooms_for_user returns a namedtuple, not a room_id --- synapse/notifier.py | 1 + tests/rest/client/v1/test_presence.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 1e73d52c4..4f47f88df 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -296,6 +296,7 @@ class Notifier(object): appservice = yield self.store.get_app_service_by_user_id(user) current_token = yield self.event_sources.get_current_token() rooms = yield self.store.get_rooms_for_user(user) + rooms = [room.room_id for room in rooms] user_stream = _NotifierUserStream( user=user, rooms=rooms, diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 29c0038f0..21f42b3d3 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -29,6 +29,8 @@ from synapse.rest.client.v1 import events from synapse.types import UserID from synapse.util.async import run_on_reactor +from collections import namedtuple + OFFLINE = PresenceState.OFFLINE UNAVAILABLE = PresenceState.UNAVAILABLE @@ -302,7 +304,10 @@ class PresenceEventStreamTestCase(unittest.TestCase): return_value=defer.succeed(None) ) self.mock_datastore.get_rooms_for_user = ( - lambda u: get_rooms_for_user(UserID.from_string(u)) + lambda u: [ + namedtuple("Room", "room_id")(r) + for r in get_rooms_for_user(UserID.from_string(u)) + ] ) def get_profile_displayname(user_id): From 115ef3ddac9dbb1c49c31257190e77062b5a10a8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:37:43 +0100 Subject: [PATCH 141/175] Correctly capture Queue.Empty exception --- synapse/storage/event_federation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 80eff8e6f..e171cbcdb 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -19,7 +19,7 @@ from ._base import SQLBaseStore, cached from syutil.base64util import encode_base64 import logging -from Queue import PriorityQueue +from Queue import PriorityQueue, Empty logger = logging.getLogger(__name__) @@ -398,7 +398,10 @@ class EventFederationStore(SQLBaseStore): queue.put(row) while not queue.empty() and len(event_results) < limit: - _, event_id = queue.get_nowait() + try: + _, event_id = queue.get_nowait() + except Empty: + break event_results.add(event_id) From 6189d8e54d7f7d55cd6cd2e9d7f866e895c6fe44 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:38:08 +0100 Subject: [PATCH 142/175] PriorityQueue gives lowest first --- synapse/storage/event_federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index e171cbcdb..03942fec7 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -395,7 +395,7 @@ class EventFederationStore(SQLBaseStore): ) for row in txn.fetchall(): - queue.put(row) + queue.put((-row[0], row[1])) while not queue.empty() and len(event_results) < limit: try: @@ -411,7 +411,7 @@ class EventFederationStore(SQLBaseStore): ) for row in txn.fetchall(): - queue.put(row) + queue.put((-row[0], row[1])) return event_results From 73d23c6ae85b180dbeca070fc9149692ee2fbcfe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:40:22 +0100 Subject: [PATCH 143/175] Don't readd things that are already in event_results --- synapse/storage/event_federation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 03942fec7..a3da11779 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -411,7 +411,8 @@ class EventFederationStore(SQLBaseStore): ) for row in txn.fetchall(): - queue.put((-row[0], row[1])) + if row[1] not in event_results: + queue.put((-row[0], row[1])) return event_results From dc085ddf8cfa887672f5c505b3bf9c2ce7fc0d58 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:44:05 +0100 Subject: [PATCH 144/175] Don't prepopulate event_results --- synapse/storage/event_federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index a3da11779..26d570cf2 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -372,7 +372,7 @@ class EventFederationStore(SQLBaseStore): room_id, repr(event_list), limit ) - event_results = set(event_list) + event_results = set() # We want to make sure that we do a breadth-first, "depth" ordered # search. From ae3bff349151d8f309bdf29fd258b215cb792e90 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:46:07 +0100 Subject: [PATCH 145/175] Correctly prepopulate queue --- synapse/storage/event_federation.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 26d570cf2..91d19857b 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -394,8 +394,16 @@ class EventFederationStore(SQLBaseStore): (room_id, event_id, limit - len(event_results)) ) - for row in txn.fetchall(): - queue.put((-row[0], row[1])) + depth = self._simple_select_one_onecol_txn( + txn, + table="events", + keyvalues={ + "event_id": event_id, + }, + retcol="depth" + ) + + queue.put((-depth, event_id)) while not queue.empty() and len(event_results) < limit: try: From 39a3340f738f37868b7034cb019fb410f6b1d48b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:48:56 +0100 Subject: [PATCH 146/175] Skip events we've already seen --- synapse/storage/event_federation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 91d19857b..823a4998c 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -411,6 +411,9 @@ class EventFederationStore(SQLBaseStore): except Empty: break + if event_id in event_results: + continue + event_results.add(event_id) txn.execute( From 1f3d1d85a9a12cab1423d39f82c8dbe339e69269 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:52:29 +0100 Subject: [PATCH 147/175] Only get non-state --- synapse/storage/event_federation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 823a4998c..8188b7cbc 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -383,6 +383,7 @@ class EventFederationStore(SQLBaseStore): " ON prev_event_id = events.event_id" " AND event_edges.room_id = events.room_id" " WHERE event_edges.room_id = ? AND event_edges.event_id = ?" + " AND event_edges.is_state = ?" " LIMIT ?" ) @@ -418,7 +419,7 @@ class EventFederationStore(SQLBaseStore): txn.execute( query, - (room_id, event_id, limit - len(event_results)) + (room_id, event_id, False, limit - len(event_results)) ) for row in txn.fetchall(): From 0180bfe4aa97464bd86efd702348fcd412f3006f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:53:41 +0100 Subject: [PATCH 148/175] Remove dead code --- synapse/storage/event_federation.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 8188b7cbc..8a56476f5 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -390,11 +390,6 @@ class EventFederationStore(SQLBaseStore): queue = PriorityQueue() for event_id in event_list: - txn.execute( - query, - (room_id, event_id, limit - len(event_results)) - ) - depth = self._simple_select_one_onecol_txn( txn, table="events", From e309b1045db036174b66364740b645466e459454 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:57:35 +0100 Subject: [PATCH 149/175] Sort backfill events --- synapse/storage/event_federation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 8a56476f5..5fd126cdb 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -364,7 +364,11 @@ class EventFederationStore(SQLBaseStore): return self.runInteraction( "get_backfill_events", self._get_backfill_events, room_id, event_list, limit - ).addCallback(self._get_events) + ).addCallback( + self._get_events + ).addCallback( + lambda l: l.sort(key=lambda e: -e.depth) + ) def _get_backfill_events(self, txn, room_id, event_list, limit): logger.debug( From a910984b58b1653394d84402bb61fb75a88e9bc7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 May 2015 15:58:41 +0100 Subject: [PATCH 150/175] Actually return something from lambda --- synapse/storage/event_federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 5fd126cdb..4655c8e54 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -367,7 +367,7 @@ class EventFederationStore(SQLBaseStore): ).addCallback( self._get_events ).addCallback( - lambda l: l.sort(key=lambda e: -e.depth) + lambda l: sorted(l, key=lambda e: -e.depth) ) def _get_backfill_events(self, txn, room_id, event_list, limit): From a04cde613e7b271b1c34409e614da12a1da52432 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 22 May 2015 10:39:45 +0100 Subject: [PATCH 151/175] Add a cache for get_push rules for user, fix cache invalidation --- synapse/storage/push_rule.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 80d0ac4ea..4cac118d1 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -23,6 +23,7 @@ logger = logging.getLogger(__name__) class PushRuleStore(SQLBaseStore): + @cached() @defer.inlineCallbacks def get_push_rules_for_user(self, user_name): rows = yield self._simple_select_list( @@ -31,6 +32,7 @@ class PushRuleStore(SQLBaseStore): "user_name": user_name, }, retcols=PushRuleTable.fields, + desc="get_push_rules_enabled_for_user", ) rows.sort( @@ -150,6 +152,10 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, priority_class, new_rule_priority)) + txn.call_after( + self.get_push_rules_for_user.invalidate, user_name + ) + txn.call_after( self.get_push_rules_enabled_for_user.invalidate, user_name ) @@ -182,6 +188,9 @@ class PushRuleStore(SQLBaseStore): new_rule['priority_class'] = priority_class new_rule['priority'] = new_prio + txn.call_after( + self.get_push_rules_for_user.invalidate, user_name + ) txn.call_after( self.get_push_rules_enabled_for_user.invalidate, user_name ) @@ -208,6 +217,8 @@ class PushRuleStore(SQLBaseStore): {'user_name': user_name, 'rule_id': rule_id}, desc="delete_push_rule", ) + + self.get_push_rules_for_user.invalidate(user_name) self.get_push_rules_enabled_for_user.invalidate(user_name) @defer.inlineCallbacks @@ -228,7 +239,12 @@ class PushRuleStore(SQLBaseStore): {'enabled': 1 if enabled else 0}, {'id': new_id}, ) - self.get_push_rules_enabled_for_user.invalidate(user_name) + txn.call_after( + self.get_push_rules_for_user.invalidate, user_name + ) + txn.call_after( + self.get_push_rules_enabled_for_user.invalidate, user_name + ) class RuleNotFoundException(Exception): From f43544eecc362943f9d64a004d40984739a68d98 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 22 May 2015 11:01:28 +0100 Subject: [PATCH 152/175] Make the appservice use 'users_in_room' rather than get_room_members since it is cached --- synapse/appservice/__init__.py | 6 +++--- synapse/handlers/appservice.py | 5 +---- tests/appservice/test_appservice.py | 15 +++------------ 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 63a18b802..e3ca45de8 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -148,8 +148,8 @@ class ApplicationService(object): and self.is_interested_in_user(event.state_key)): return True # check joined member events - for member in member_list: - if self.is_interested_in_user(member.state_key): + for user_id in member_list: + if self.is_interested_in_user(user_id): return True return False @@ -173,7 +173,7 @@ class ApplicationService(object): restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. - member_list(list): A list of all joined room members in this room. + member_list(list): A list of all joined user_ids in this room. Returns: bool: True if this service would like to know about this event. """ diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 355ab317d..05735137d 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -147,10 +147,7 @@ class ApplicationServicesHandler(object): ) # We need to know the members associated with this event.room_id, # if any. - member_list = yield self.store.get_room_members( - room_id=event.room_id, - membership=Membership.JOIN - ) + member_list = yield self.store.get_users_in_room(event.room_id) services = yield self.store.get_app_services() interested_list = [ diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 62149d690..8ce8dc0a8 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -217,18 +217,9 @@ class ApplicationServiceTestCase(unittest.TestCase): _regex("@irc_.*") ) join_list = [ - Mock( - type="m.room.member", room_id="!foo:bar", sender="@alice:here", - state_key="@alice:here" - ), - Mock( - type="m.room.member", room_id="!foo:bar", sender="@irc_fo:here", - state_key="@irc_fo:here" # AS user - ), - Mock( - type="m.room.member", room_id="!foo:bar", sender="@bob:here", - state_key="@bob:here" - ) + "@alice:here", + "@irc_fo:here", # AS user + "@bob:here", ] self.event.sender = "@xmpp_foobar:matrix.org" From 254aa3c986256de0c71d36078682c7904e27145b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 11:59:48 +0100 Subject: [PATCH 153/175] Revert register_new_matrix_user to use v1 api --- scripts/register_new_matrix_user | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/register_new_matrix_user b/scripts/register_new_matrix_user index 0ca83795a..4a520bdb5 100755 --- a/scripts/register_new_matrix_user +++ b/scripts/register_new_matrix_user @@ -33,9 +33,10 @@ def request_registration(user, password, server_location, shared_secret): ).hexdigest() data = { - "username": user, + "user": user, "password": password, "mac": mac, + "type": "org.matrix.login.shared_secret", } server_location = server_location.rstrip("/") @@ -43,7 +44,7 @@ def request_registration(user, password, server_location, shared_secret): print "Sending registration request..." req = urllib2.Request( - "%s/_matrix/client/v2_alpha/register" % (server_location,), + "%s/_matrix/client/api/v1/register" % (server_location,), data=json.dumps(data), headers={'Content-Type': 'application/json'} ) From b6adfc59f522cfc897616c153f055d669da71a9a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 22 May 2015 13:00:50 +0100 Subject: [PATCH 154/175] Invalidate the get_latest_event_ids_in_room cache when deleting from event_forward_extremities --- synapse/storage/event_federation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 5d4b7843f..23573e8b2 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -472,3 +472,4 @@ class EventFederationStore(SQLBaseStore): query = "DELETE FROM event_forward_extremities WHERE room_id = ?" txn.execute(query, (room_id,)) + txn.call_after(self.get_latest_event_ids_in_room.invalidate, room_id) From 59a0682f3e9fe64c7ab35c011b0af7b87ee54f71 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 13:13:07 +0100 Subject: [PATCH 155/175] Enable changing the interface the metrics listener binds to --- synapse/app/homeserver.py | 2 +- synapse/config/metrics.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fa4321141..70a7be60b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -277,7 +277,7 @@ class SynapseHomeServer(HomeServer): config, metrics_resource, ), - interface="127.0.0.1", + interface=config.metrics_interface, ) logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 71a1b1d18..c843c079c 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -20,6 +20,7 @@ class MetricsConfig(Config): def read_config(self, config): self.enable_metrics = config["enable_metrics"] self.metrics_port = config.get("metrics_port") + self.metrics_interface = config.get("metrics_interface", "127.0.0.1") def default_config(self, config_dir_path, server_name): return """\ @@ -28,6 +29,9 @@ class MetricsConfig(Config): # Enable collection and rendering of performance metrics enable_metrics: False - # Separate port to accept metrics requests on (on localhost) + # Separate port to accept metrics requests on # metrics_port: 8081 + + # Which interface to bind the metric listener to + # metrics_interface: 127.0.0.1 """ From 1b446a5d85f35d0af016496d9b12733ce667adb1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 14:26:08 +0100 Subject: [PATCH 156/175] Log less lines at INFO level, but include more helpful information --- synapse/federation/transaction_queue.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index ca04822fb..afe71897f 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -207,13 +207,13 @@ class TransactionQueue(object): # request at which point pending_pdus_by_dest just keeps growing. # we need application-layer timeouts of some flavour of these # requests - logger.info( + logger.debug( "TX [%s] Transaction already in progress", destination ) return - logger.info("TX [%s] _attempt_new_transaction", destination) + logger.debug("TX [%s] _attempt_new_transaction", destination) # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) @@ -221,11 +221,11 @@ class TransactionQueue(object): pending_failures = self.pending_failures_by_dest.pop(destination, []) if pending_pdus: - logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", - destination, len(pending_pdus)) + logger.debug("TX [%s] len(pending_pdus_by_dest[dest]) = %d", + destination, len(pending_pdus)) if not pending_pdus and not pending_edus and not pending_failures: - logger.info("TX [%s] Nothing to send", destination) + logger.debug("TX [%s] Nothing to send", destination) return # Sort based on the order field @@ -275,9 +275,13 @@ class TransactionQueue(object): logger.debug("TX [%s] Persisted transaction", destination) logger.info( - "TX [%s] Sending transaction [%s]", + "TX [%s] Sending transaction [%s]," + " (PDUs: %d, EDUs: %d, failures: %d)", destination, transaction.transaction_id, + len(pending_pdus), + len(pending_edus), + len(pending_failures), ) with limiter: From e70e8e053e2d44d0f7cebe3bee7b4afbe103f0a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 14:33:11 +0100 Subject: [PATCH 157/175] Add txn_id to some log lines --- synapse/federation/transaction_queue.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index afe71897f..32fa5e8c1 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -242,6 +242,8 @@ class TransactionQueue(object): try: self.pending_transactions[destination] = 1 + txn_id = str(self._next_txn_id) + limiter = yield get_retry_limiter( destination, self._clock, @@ -249,9 +251,9 @@ class TransactionQueue(object): ) logger.debug( - "TX [%s] Attempting new transaction" + "TX [%s] {%s} Attempting new transaction" " (pdus: %d, edus: %d, failures: %d)", - destination, + destination, txn_id, len(pending_pdus), len(pending_edus), len(pending_failures) @@ -261,7 +263,7 @@ class TransactionQueue(object): transaction = Transaction.create_new( origin_server_ts=int(self._clock.time_msec()), - transaction_id=str(self._next_txn_id), + transaction_id=txn_id, origin=self.server_name, destination=destination, pdus=pdus, @@ -275,9 +277,9 @@ class TransactionQueue(object): logger.debug("TX [%s] Persisted transaction", destination) logger.info( - "TX [%s] Sending transaction [%s]," + "TX [%s] {%s} Sending transaction [%s]," " (PDUs: %d, EDUs: %d, failures: %d)", - destination, + destination, txn_id, transaction.transaction_id, len(pending_pdus), len(pending_edus), @@ -317,7 +319,10 @@ class TransactionQueue(object): code = e.code response = e.response - logger.info("TX [%s] got %d response", destination, code) + logger.info( + "TX [%s] {%s} got %d response", + destination, txn_id, code + ) logger.debug("TX [%s] Sent transaction", destination) logger.debug("TX [%s] Marking as delivered...", destination) From b21d015c55dde5255c7a63f8d17e77caf535030b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 14:44:25 +0100 Subject: [PATCH 158/175] Log origin and stats of incoming transactions --- synapse/federation/transport/server.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 2bfe0f3c9..af87805f3 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -196,6 +196,14 @@ class FederationSendServlet(BaseFederationServlet): transaction_id, str(transaction_data) ) + logger.info( + "Received txn %s from %s. (PDUs: %d, EDUs: %d, failures: %d)", + transaction_id, origin, + len(transaction_data.get("pdus", [])), + len(transaction_data.get("edus", [])), + len(transaction_data.get("failures", [])), + ) + # We should ideally be getting this from the security layer. # origin = body["origin"] From c8135f808b4e0a344e6d5d592049d27cc43af9b1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 22 May 2015 14:45:46 +0100 Subject: [PATCH 159/175] Remove unused import --- synapse/handlers/appservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 05735137d..8269482e4 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes from synapse.appservice import ApplicationService from synapse.types import UserID From 8bb85c8c5a24f5a937fbd66eace23bd680ca728f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 14:48:06 +0100 Subject: [PATCH 160/175] Update log line --- synapse/app/homeserver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 70a7be60b..b887562f9 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -279,7 +279,10 @@ class SynapseHomeServer(HomeServer): ), interface=config.metrics_interface, ) - logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) + logger.info( + "Metrics now running on %s port %d", + config.metrics_interface, config.metrics_port, + ) def run_startup_checks(self, db_conn, database_engine): all_users_native = are_all_users_on_domain( From 1ce1509989ae5eba67acbe0824d82177ab10917f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 14:51:22 +0100 Subject: [PATCH 161/175] s/metric_interface/metric_bind_host/ --- synapse/app/homeserver.py | 4 ++-- synapse/config/metrics.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b887562f9..f3513abb5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -277,11 +277,11 @@ class SynapseHomeServer(HomeServer): config, metrics_resource, ), - interface=config.metrics_interface, + interface=config.metrics_bind_host, ) logger.info( "Metrics now running on %s port %d", - config.metrics_interface, config.metrics_port, + config.metrics_bind_host, config.metrics_port, ) def run_startup_checks(self, db_conn, database_engine): diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index c843c079c..0cfb30ce7 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -20,7 +20,7 @@ class MetricsConfig(Config): def read_config(self, config): self.enable_metrics = config["enable_metrics"] self.metrics_port = config.get("metrics_port") - self.metrics_interface = config.get("metrics_interface", "127.0.0.1") + self.metrics_bind_host = config.get("metrics_bind_host", "127.0.0.1") def default_config(self, config_dir_path, server_name): return """\ @@ -32,6 +32,6 @@ class MetricsConfig(Config): # Separate port to accept metrics requests on # metrics_port: 8081 - # Which interface to bind the metric listener to - # metrics_interface: 127.0.0.1 + # Which host to bind the metric listener to + # metrics_bind_host: 127.0.0.1 """ From 284f55a7fbf597e32508301fe9571cd1b8523625 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 15:18:04 +0100 Subject: [PATCH 162/175] Add doc strings --- synapse/federation/federation_client.py | 2 ++ synapse/federation/transport/client.py | 2 ++ synapse/http/matrixfederationclient.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index ecb6dbd77..3249060bc 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -190,6 +190,8 @@ class FederationClient(FederationBase): outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if it's from an arbitary point in the context as opposed to part of the current block of PDUs. Defaults to `False` + timeout (int): How long to try (in ms) each destination for before + moving to the next destination. None indicates no timeout. Returns: Deferred: Results in the requested PDU. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index c2b53b78b..610a4c316 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -57,6 +57,8 @@ class TransportLayerClient(object): destination (str): The host name of the remote home server we want to get the state from. event_id (str): The id of the event being requested. + timeout (int): How long to try (in ms) the destination for before + giving up. None indicates no timeout. Returns: Deferred: Results in a dict received from the remote homeserver. diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 312bbcc6b..6f976d5ce 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -345,6 +345,9 @@ class MatrixFederationHttpClient(object): path (str): The HTTP path. args (dict): A dictionary used to create query strings, defaults to None. + timeout (int): How long to try (in ms) the destination for before + giving up. None indicates no timeout and that the request will + be retried. Returns: Deferred: Succeeds when we get *any* HTTP response. From 106a3051b88be742d24ace05f72d9ab6bff29dd2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 22 May 2015 15:53:03 +0100 Subject: [PATCH 163/175] Remove spurious TODO comment --- synapse/handlers/presence.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6c48b1d20..670c1d353 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1140,7 +1140,6 @@ class PresenceEventSource(object): user_ids_to_check |= set(joined) updates = [] - # TODO(paul): use a DeferredList ? How to limit concurrency. for observed_user in user_ids_to_check & set(cachemap): if not (to_key < cachemap[observed_user].serial <= from_key): continue From 1a9a9abcc73ebbd14fce0f45689e4648a71d55bc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 22 May 2015 16:11:17 +0100 Subject: [PATCH 164/175] Add a cache for getting the presence list for a user --- synapse/handlers/presence.py | 24 +++++++++++++++--------- synapse/storage/presence.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 670c1d353..023ad33ab 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -521,20 +521,26 @@ class PresenceHandler(BaseHandler): if not self.hs.is_mine(observer_user): raise SynapseError(400, "User is not hosted on this Home Server") - presence = yield self.store.get_presence_list( + presence_list = yield self.store.get_presence_list( observer_user.localpart, accepted=accepted ) - for p in presence: - observed_user = UserID.from_string(p.pop("observed_user_id")) - p["observed_user"] = observed_user - p.update(self._get_or_offline_usercache(observed_user).get_state()) - if "last_active" in p: - p["last_active_ago"] = int( - self.clock.time_msec() - p.pop("last_active") + results = [] + for row in presence_list: + observed_user = UserID.from_string(row["observed_user_id"]) + result = { + "observed_user": observed_user, "accepted": row["accepted"] + } + result.update( + self._get_or_offline_usercache(observed_user).get_state() + ) + if "last_active" in result: + result["last_active_ago"] = int( + self.clock.time_msec() - result.pop("last_active") ) + results.append(result) - defer.returnValue(presence) + defer.returnValue(results) @defer.inlineCallbacks @log_function diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 22ec94bc1..fefcf6bce 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore +from ._base import SQLBaseStore, cached + +from twisted.internet import defer class PresenceStore(SQLBaseStore): @@ -87,31 +89,48 @@ class PresenceStore(SQLBaseStore): desc="add_presence_list_pending", ) + @defer.inlineCallbacks def set_presence_list_accepted(self, observer_localpart, observed_userid): - return self._simple_update_one( + result = yield self._simple_update_one( table="presence_list", keyvalues={"user_id": observer_localpart, "observed_user_id": observed_userid}, updatevalues={"accepted": True}, desc="set_presence_list_accepted", ) + self.get_presence_list_accepted.invalidate(observer_localpart) + defer.returnValue(result) def get_presence_list(self, observer_localpart, accepted=None): - keyvalues = {"user_id": observer_localpart} - if accepted is not None: - keyvalues["accepted"] = accepted + if accepted: + return self.get_presence_list_accepted(observer_localpart) + else: + keyvalues = {"user_id": observer_localpart} + if accepted is not None: + keyvalues["accepted"] = accepted + return self._simple_select_list( + table="presence_list", + keyvalues=keyvalues, + retcols=["observed_user_id", "accepted"], + desc="get_presence_list", + ) + + @cached() + def get_presence_list_accepted(self, observer_localpart): return self._simple_select_list( table="presence_list", - keyvalues=keyvalues, + keyvalues={"user_id": observer_localpart, "accepted": True}, retcols=["observed_user_id", "accepted"], - desc="get_presence_list", + desc="get_presence_list_accepted", ) + @defer.inlineCallbacks def del_presence_list(self, observer_localpart, observed_userid): - return self._simple_delete_one( + yield self._simple_delete_one( table="presence_list", keyvalues={"user_id": observer_localpart, "observed_user_id": observed_userid}, desc="del_presence_list", ) + self.get_presence_list_accepted.invalidate(observer_localpart) From 6eadbfbea0f8eb742f94d73e262631b0877e3dee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 16:12:20 +0100 Subject: [PATCH 165/175] Remove redundant for loop --- synapse/federation/federation_client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index cbb9d354b..d3b46b24c 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -165,9 +165,6 @@ class FederationClient(FederationBase): for p in transaction_data["pdus"] ] - for i, pdu in enumerate(pdus): - pdus[i] = yield self._check_sigs_and_hash(pdu) - # FIXME: We should handle signature failures more gracefully. pdus[:] = yield defer.gatherResults( [self._check_sigs_and_hash(pdu) for pdu in pdus], From 17167898c816c95db8a3f4661d955f43ad6a87ce Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 22 May 2015 16:22:54 +0100 Subject: [PATCH 166/175] Fix the presence tests --- tests/handlers/test_presence.py | 16 ++++++++++------ tests/handlers/test_presencelike.py | 20 +++++++++++--------- tests/rest/client/v1/test_presence.py | 4 ++-- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 12cf5747a..29372d488 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -233,7 +233,7 @@ class MockedDatastorePresenceTestCase(PresenceTestCase): if not user_localpart in self.PRESENCE_LIST: return defer.succeed([]) return defer.succeed([ - {"observed_user_id": u} for u in + {"observed_user_id": u, "accepted": accepted} for u in self.PRESENCE_LIST[user_localpart]]) datastore.get_presence_list = get_presence_list @@ -734,10 +734,12 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): self.assertEquals( [ - {"observed_user": self.u_banana, - "presence": OFFLINE}, + {"observed_user": self.u_banana, + "presence": OFFLINE, + "accepted": True}, {"observed_user": self.u_clementine, - "presence": OFFLINE}, + "presence": OFFLINE, + "accepted": True}, ], presence ) @@ -758,9 +760,11 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): self.assertEquals([ {"observed_user": self.u_banana, "presence": ONLINE, - "last_active_ago": 2000}, + "last_active_ago": 2000, + "accepted": True}, {"observed_user": self.u_clementine, - "presence": OFFLINE}, + "presence": OFFLINE, + "accepted": True}, ], presence) (events, _) = yield self.event_source.get_new_events_for_user( diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 1f2e66ac1..19107caee 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -101,8 +101,8 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): self.datastore.get_profile_avatar_url = get_profile_avatar_url self.presence_list = [ - {"observed_user_id": "@banana:test"}, - {"observed_user_id": "@clementine:test"}, + {"observed_user_id": "@banana:test", "accepted": True}, + {"observed_user_id": "@clementine:test", "accepted": True}, ] def get_presence_list(user_localpart, accepted=None): return defer.succeed(self.presence_list) @@ -144,8 +144,8 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): @defer.inlineCallbacks def test_set_my_state(self): self.presence_list = [ - {"observed_user_id": "@banana:test"}, - {"observed_user_id": "@clementine:test"}, + {"observed_user_id": "@banana:test", "accepted": True}, + {"observed_user_id": "@clementine:test", "accepted": True}, ] mocked_set = self.datastore.set_presence_state @@ -167,8 +167,8 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): self.mock_get_joined.side_effect = get_joined self.presence_list = [ - {"observed_user_id": "@banana:test"}, - {"observed_user_id": "@clementine:test"}, + {"observed_user_id": "@banana:test", "accepted": True}, + {"observed_user_id": "@clementine:test", "accepted": True}, ] self.datastore.set_presence_state.return_value = defer.succeed( @@ -203,9 +203,11 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): "presence": ONLINE, "last_active_ago": 0, "displayname": "Frank", - "avatar_url": "http://foo"}, + "avatar_url": "http://foo", + "accepted": True}, {"observed_user": self.u_clementine, - "presence": OFFLINE} + "presence": OFFLINE, + "accepted": True} ], presence) self.mock_update_client.assert_has_calls([ @@ -233,7 +235,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): @defer.inlineCallbacks def test_push_remote(self): self.presence_list = [ - {"observed_user_id": "@potato:remote"}, + {"observed_user_id": "@potato:remote", "accepted": True}, ] self.datastore.set_presence_state.return_value = defer.succeed( diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 523b30cf8..4b32c7a20 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -183,7 +183,7 @@ class PresenceListTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_my_list(self): self.datastore.get_presence_list.return_value = defer.succeed( - [{"observed_user_id": "@banana:test"}], + [{"observed_user_id": "@banana:test", "accepted": True}], ) (code, response) = yield self.mock_resource.trigger("GET", @@ -191,7 +191,7 @@ class PresenceListTestCase(unittest.TestCase): self.assertEquals(200, code) self.assertEquals([ - {"user_id": "@banana:test", "presence": OFFLINE}, + {"user_id": "@banana:test", "presence": OFFLINE, "accepted": True}, ], response) self.datastore.get_presence_list.assert_called_with( From 27e093cbc1bc50f597119c132596ccb12c219ee4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 May 2015 17:03:37 +0100 Subject: [PATCH 167/175] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 68f86138a..4720d9984 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.0-r5" +__version__ = "0.9.1" From a0bebeda8b5fa4130cc634b5bd9d0b1bd454713e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 26 May 2015 10:14:15 +0100 Subject: [PATCH 168/175] SYN-390: Don't modify the dictionary returned from the data store --- synapse/rest/client/v1/push_rule.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index d4e7ab220..bd759a258 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -118,11 +118,14 @@ class PushRuleRestServlet(ClientV1RestServlet): user.to_string() ) - for r in rawrules: - r["conditions"] = json.loads(r["conditions"]) - r["actions"] = json.loads(r["actions"]) + ruleslist = [] + for rawrule in rawrules: + rule = dict(rawrule) + rule["conditions"] = json.loads(rawrule["conditions"]) + rule["actions"] = json.loads(rawrule["actions"]) + ruleslist.append(rule) - ruleslist = baserules.list_with_base_rules(rawrules, user) + ruleslist = baserules.list_with_base_rules(ruleslist, user) rules = {'global': {}, 'device': {}} From 804b732aab3c7d285c6f976f1fd3e6088d0bdf28 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 26 May 2015 10:35:08 +0100 Subject: [PATCH 169/175] SYN-390: Don't modify the dictionary returned from the database here either --- synapse/push/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index e3dd4ce76..167b973b2 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -74,15 +74,18 @@ class Pusher(object): rawrules = yield self.store.get_push_rules_for_user(self.user_name) - for r in rawrules: - r['conditions'] = json.loads(r['conditions']) - r['actions'] = json.loads(r['actions']) + rules = [] + for rawrule in rawrules: + rule = dict(rawrules) + rule['conditions'] = json.loads(rawrule['conditions']) + rule['actions'] = json.loads(rawrule['actions']) + rules.append(rule) enabled_map = yield self.store.get_push_rules_enabled_for_user(self.user_name) user = UserID.from_string(self.user_name) - rules = baserules.list_with_base_rules(rawrules, user) + rules = baserules.list_with_base_rules(rules, user) room_id = ev['room_id'] From 764856777c0cafef9104e3249911123096f24081 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 May 2015 11:05:44 +0100 Subject: [PATCH 170/175] changelog --- CHANGES.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e1420d7a3..529ab3ee9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,28 @@ +Changes in synapse v0.9.1 (2015-05-26) +====================================== + +General: + +* Add support for backfilling when a client paginates. This allows servers to + request history for a room from remote servers when a client tries to + paginate history the server does not have - SYN-36 +* Fix bug where we couldn't disable non-default pushrules - SYN-378 +* Fix bug with ``register_new_user`` script - SYN-359 +* Improve performance of fetching events from the database. +* Improve performance of event streams. + +Federation: + +* Fix bug with existing backfill implementation where it returned the wrong + selection of events in some circumstances. +* Improve performance of joining remote rooms. + +Configuration: + +* Add support for changing the bind host of the metrics listener via the + ``metrics_bind_host`` option. + + Changes in synapse v0.9.0-r5 (2015-05-21) ========================================= From cb7dac3a5d64eb37cdc39deeb1ea4c959023c8f5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 May 2015 11:08:09 +0100 Subject: [PATCH 171/175] changelog --- CHANGES.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 529ab3ee9..0e5f28c2d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,8 +8,10 @@ General: paginate history the server does not have - SYN-36 * Fix bug where we couldn't disable non-default pushrules - SYN-378 * Fix bug with ``register_new_user`` script - SYN-359 -* Improve performance of fetching events from the database. -* Improve performance of event streams. +* Improve performance of fetching events from the database. This improves both + initialSync and sending of events. +* Improve performance of event streams, allowing synapse to handle more + simultaneous connected clients. Federation: From e417469af2d2fb9c57e68d30cf590e6c20319859 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 May 2015 11:08:46 +0100 Subject: [PATCH 172/175] changelog --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0e5f28c2d..3d4178fad 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,7 +8,7 @@ General: paginate history the server does not have - SYN-36 * Fix bug where we couldn't disable non-default pushrules - SYN-378 * Fix bug with ``register_new_user`` script - SYN-359 -* Improve performance of fetching events from the database. This improves both +* Improve performance of fetching events from the database, this improves both initialSync and sending of events. * Improve performance of event streams, allowing synapse to handle more simultaneous connected clients. From 00dd207f603597f016af11729d1fd31f6391ff9a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 26 May 2015 14:57:48 +0100 Subject: [PATCH 173/175] Take a dict of the rule, not the rule list --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 167b973b2..8059fff1b 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -76,7 +76,7 @@ class Pusher(object): rules = [] for rawrule in rawrules: - rule = dict(rawrules) + rule = dict(rawrule) rule['conditions'] = json.loads(rawrule['conditions']) rule['actions'] = json.loads(rawrule['actions']) rules.append(rule) From 554c63ca604f3b8e9a3e8cde143eb319bf750c00 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 26 May 2015 15:03:49 +0100 Subject: [PATCH 174/175] Iterate over the user_streams not the user_ids --- synapse/notifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 4f47f88df..078abfc56 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -153,7 +153,7 @@ class Notifier(object): for x in self.room_to_user_streams.values(): all_user_streams |= x - for x in self.user_to_user_stream: + for x in self.user_to_user_stream.values(): all_user_streams.add(x) for x in self.appservice_to_user_streams.values(): all_user_streams |= x From 6cb3212fc22b00459c6ffa667e86092da621f93d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 May 2015 16:00:45 +0100 Subject: [PATCH 175/175] changelog --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3d4178fad..1ca2407a7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,8 +6,8 @@ General: * Add support for backfilling when a client paginates. This allows servers to request history for a room from remote servers when a client tries to paginate history the server does not have - SYN-36 -* Fix bug where we couldn't disable non-default pushrules - SYN-378 -* Fix bug with ``register_new_user`` script - SYN-359 +* Fix bug where you couldn't disable non-default pushrules - SYN-378 +* Fix ``register_new_user`` script - SYN-359 * Improve performance of fetching events from the database, this improves both initialSync and sending of events. * Improve performance of event streams, allowing synapse to handle more