From 74c38797601f6d7d1a02d21fc54ceb1a54629c64 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 19 Nov 2014 18:20:59 +0000 Subject: [PATCH 001/630] Start creating a module to do generic notifications (just prints them to stdout currently!) --- synapse/api/errors.py | 1 + synapse/app/homeserver.py | 2 + synapse/push/__init__.py | 76 ++++++++++++++++++++++ synapse/push/httppusher.py | 40 ++++++++++++ synapse/push/pusherpool.py | 94 +++++++++++++++++++++++++++ synapse/rest/__init__.py | 3 +- synapse/rest/pusher.py | 71 +++++++++++++++++++++ synapse/server.py | 5 ++ synapse/storage/__init__.py | 6 +- synapse/storage/pusher.py | 98 +++++++++++++++++++++++++++++ synapse/storage/schema/delta/v7.sql | 28 +++++++++ synapse/storage/schema/pusher.sql | 28 +++++++++ 12 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 synapse/push/__init__.py create mode 100644 synapse/push/httppusher.py create mode 100644 synapse/push/pusherpool.py create mode 100644 synapse/rest/pusher.py create mode 100644 synapse/storage/pusher.py create mode 100644 synapse/storage/schema/delta/v7.sql create mode 100644 synapse/storage/schema/pusher.sql diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 33d15072a..97750ca2b 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -32,6 +32,7 @@ class Codes(object): LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" + MISSING_PARAM = "M_MISSING_PARAM" class CodeMessageException(Exception): diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 85284a491..de16d8a2e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -242,6 +242,8 @@ def setup(): bind_port = None hs.start_listening(bind_port, config.unsecure_port) + hs.get_pusherpool().start() + if config.daemonize: print config.pid_file daemon = Daemonize( diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py new file mode 100644 index 000000000..df0b91a8e --- /dev/null +++ b/synapse/push/__init__.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.streams.config import PaginationConfig +from synapse.types import StreamToken + +import synapse.util.async + +import logging + +logger = logging.getLogger(__name__) + +class Pusher(object): + INITIAL_BACKOFF = 1000 + MAX_BACKOFF = 10 * 60 * 1000 + + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + self.hs = _hs + self.evStreamHandler = self.hs.get_handlers().event_stream_handler + self.store = self.hs.get_datastore() + self.user_name = user_name + self.app = app + self.app_display_name = app_display_name + self.device_display_name = device_display_name + self.pushkey = pushkey + self.data = data + self.last_token = last_token + self.backoff_delay = Pusher.INITIAL_BACKOFF + + @defer.inlineCallbacks + def start(self): + if not self.last_token: + # First-time setup: get a token to start from (we can't just start from no token, ie. 'now' + # because we need the result to be reproduceable in case we fail to dispatch the push) + config = PaginationConfig(from_token=None, limit='1') + chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=0) + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + logger.info("Pusher %s for user %s starting from token %s", + self.pushkey, self.user_name, self.last_token) + + while True: + from_tok = StreamToken.from_string(self.last_token) + config = PaginationConfig(from_token=from_tok, limit='1') + chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) + + if (self.dispatchPush(chunk['chunk'][0])): + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + else: + logger.warn("Failed to dispatch push for user %s. Trying again in %dms", + self.user_name, self.backoff_delay) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *=2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF + + +class PusherConfigException(Exception): + def __init__(self, msg): + super(PusherConfigException, self).__init__(msg) \ No newline at end of file diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py new file mode 100644 index 000000000..988c4e32f --- /dev/null +++ b/synapse/push/httppusher.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.push import Pusher, PusherConfigException +from synapse.http.client import + +import logging + +logger = logging.getLogger(__name__) + +class HttpPusher(Pusher): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + super(HttpPusher, self).__init__(_hs, + user_name, + app, + app_display_name, + device_display_name, + pushkey, + data, + last_token) + if 'url' not in data: + raise PusherConfigException("'url' required in data for HTTP pusher") + self.url = data['url'] + + def dispatchPush(self, event): + print event + return True + diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py new file mode 100644 index 000000000..436040f12 --- /dev/null +++ b/synapse/push/pusherpool.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from httppusher import HttpPusher +from synapse.push import PusherConfigException + +import logging +import json + +logger = logging.getLogger(__name__) + +class PusherPool: + def __init__(self, _hs): + self.hs = _hs + self.store = self.hs.get_datastore() + self.pushers = [] + self.last_pusher_started = -1 + + def start(self): + self._pushers_added() + + def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + # we try to create the pusher just to validate the config: it will then get pulled out of the database, + # recreated, added and started: this means we have only one code path adding pushers. + self._create_pusher({ + "user_name": user_name, + "kind": kind, + "app": app, + "app_display_name": app_display_name, + "device_display_name": device_display_name, + "pushkey": pushkey, + "data": data, + "last_token": None + }) + self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) + + @defer.inlineCallbacks + def _add_pusher_to_store(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + yield self.store.add_pusher(user_name=user_name, + kind=kind, + app=app, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=json.dumps(data)) + self._pushers_added() + + def _create_pusher(self, pusherdict): + if pusherdict['kind'] == 'http': + return HttpPusher(self.hs, + user_name=pusherdict['user_name'], + app=pusherdict['app'], + app_display_name=pusherdict['app_display_name'], + device_display_name=pusherdict['device_display_name'], + pushkey=pusherdict['pushkey'], + data=pusherdict['data'], + last_token=pusherdict['last_token'] + ) + else: + raise PusherConfigException("Unknown pusher type '%s' for user %s" % + (pusherdict['kind'], pusherdict['user_name'])) + + @defer.inlineCallbacks + def _pushers_added(self): + pushers = yield self.store.get_all_pushers_after_id(self.last_pusher_started) + for p in pushers: + p['data'] = json.loads(p['data']) + if (len(pushers)): + self.last_pusher_started = pushers[-1]['id'] + + self._start_pushers(pushers) + + def _start_pushers(self, pushers): + logger.info("Starting %d pushers", (len(pushers))) + for pusherdict in pushers: + p = self._create_pusher(pusherdict) + if p: + self.pushers.append(p) + p.start() \ No newline at end of file diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index e391e5678..c38cf2769 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -16,7 +16,7 @@ from . import ( room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, + voip, admin, pusher, ) @@ -45,3 +45,4 @@ class RestServletFactory(object): directory.register_servlets(hs, client_resource) voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) + pusher.register_servlets(hs, client_resource) diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py new file mode 100644 index 000000000..85d0d1c8c --- /dev/null +++ b/synapse/rest/pusher.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError, Codes +from synapse.push import PusherConfigException +from base import RestServlet, client_path_pattern + +import json + + +class PusherRestServlet(RestServlet): + PATTERN = client_path_pattern("/pushers/(?P[\w]*)$") + + @defer.inlineCallbacks + def on_PUT(self, request, pushkey): + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + reqd = ['kind', 'app', 'app_display_name', 'device_display_name', 'data'] + missing = [] + for i in reqd: + if i not in content: + missing.append(i) + if len(missing): + raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) + + pusher_pool = self.hs.get_pusherpool() + try: + pusher_pool.add_pusher(user_name=user.to_string(), + kind=content['kind'], + app=content['app'], + app_display_name=content['app_display_name'], + device_display_name=content['device_display_name'], + pushkey=pushkey, + data=content['data']) + except PusherConfigException as pce: + raise SynapseError(400, "Config Error: "+pce.message, errcode=Codes.MISSING_PARAM) + + defer.returnValue((200, {})) + + def on_OPTIONS(self, request): + return (200, {}) + +# XXX: C+ped from rest/room.py - surely this should be common? +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + +def register_servlets(hs, http_server): + PusherRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index da0a44433..cfbe7d5e3 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -36,6 +36,7 @@ from synapse.util.lockutils import LockManager from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter from synapse.crypto.keyring import Keyring +from synapse.push.pusherpool import PusherPool class BaseHomeServer(object): @@ -82,6 +83,7 @@ class BaseHomeServer(object): 'ratelimiter', 'keyring', 'event_validator', + 'pusherpool' ] def __init__(self, hostname, **kwargs): @@ -228,6 +230,9 @@ class HomeServer(BaseHomeServer): def build_event_validator(self): return EventValidator(self) + def build_pusherpool(self): + return PusherPool(self) + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c36d938d9..5957f938a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -33,6 +33,7 @@ from .stream import StreamStore from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore +from .pusher import PusherStore from .state import StateStore from .signatures import SignatureStore @@ -62,12 +63,13 @@ SCHEMAS = [ "state", "event_edges", "event_signatures", + "pusher" ] # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 6 +SCHEMA_VERSION = 7 class _RollbackButIsFineException(Exception): @@ -81,7 +83,7 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, - EventFederationStore, ): + EventFederationStore, PusherStore, ): def __init__(self, hs): super(DataStore, self).__init__(hs) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py new file mode 100644 index 000000000..047a5f42d --- /dev/null +++ b/synapse/storage/pusher.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from ._base import SQLBaseStore, Table +from twisted.internet import defer + +from sqlite3 import IntegrityError +from synapse.api.errors import StoreError + +import logging + +logger = logging.getLogger(__name__) + +class PusherStore(SQLBaseStore): + @defer.inlineCallbacks + def get_all_pushers_after_id(self, min_id): + sql = ( + "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, last_token " + "FROM pushers " + "WHERE id > ?" + ) + + rows = yield self._execute(None, sql, min_id) + + ret = [ + { + "id": r[0], + "user_name": r[1], + "kind": r[2], + "app": r[3], + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8] + + } + for r in rows + ] + + defer.returnValue(ret) + + @defer.inlineCallbacks + def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + try: + yield self._simple_insert(PushersTable.table_name, dict( + user_name=user_name, + kind=kind, + app=app, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=data + )) + except IntegrityError: + raise StoreError(409, "Pushkey in use.") + except Exception as e: + logger.error("create_pusher with failed: %s", e) + raise StoreError(500, "Problem creating pusher.") + + @defer.inlineCallbacks + def update_pusher_last_token(self, user_name, pushkey, last_token): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token} + ) + + +class PushersTable(Table): + table_name = "pushers" + + fields = [ + "id", + "user_name", + "kind", + "app" + "app_display_name", + "device_display_name", + "pushkey", + "data", + "last_token" + ] + + EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql new file mode 100644 index 000000000..7f6852485 --- /dev/null +++ b/synapse/storage/schema/delta/v7.sql @@ -0,0 +1,28 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + kind varchar(8) NOT NULL, + app varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + data text, + last_token TEXT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (user_name, pushkey) +); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql new file mode 100644 index 000000000..7f6852485 --- /dev/null +++ b/synapse/storage/schema/pusher.sql @@ -0,0 +1,28 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + kind varchar(8) NOT NULL, + app varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + data text, + last_token TEXT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (user_name, pushkey) +); From 051b18581109022d79d9f270d5f5e565688d89d0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 19 Nov 2014 18:37:00 +0000 Subject: [PATCH 002/630] remove random half-line --- synapse/push/httppusher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 988c4e32f..f3c3ca819 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -14,7 +14,6 @@ # limitations under the License. from synapse.push import Pusher, PusherConfigException -from synapse.http.client import import logging From eb6aedf92c0fe467fd4724623262907ad78573bb Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Nov 2014 12:21:00 +0000 Subject: [PATCH 003/630] More work on pushers. Attempt to do HTTP pokes. Not sure if the actual HTTP pokes work or not yet but the retry semantics are pretty good. --- synapse/http/client.py | 19 ++++++++++ synapse/push/__init__.py | 58 ++++++++++++++++++++++++----- synapse/push/httppusher.py | 55 +++++++++++++++++++++++++-- synapse/push/pusherpool.py | 8 +++- synapse/storage/pusher.py | 26 +++++++++++-- synapse/storage/schema/delta/v7.sql | 2 + synapse/storage/schema/pusher.sql | 2 + 7 files changed, 150 insertions(+), 20 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 048a42890..82e80385c 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -60,6 +60,25 @@ class SimpleHttpClient(object): defer.returnValue(json.loads(body)) + @defer.inlineCallbacks + def post_json_get_json(self, uri, post_json): + json_str = json.dumps(post_json) + + logger.info("HTTP POST %s -> %s", json_str, uri) + + response = yield self.agent.request( + "POST", + uri.encode("ascii"), + headers=Headers({ + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + @defer.inlineCallbacks def get_json(self, uri, args={}): """ Get's some json from the given host and path diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index df0b91a8e..a96f0f018 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -26,12 +26,15 @@ logger = logging.getLogger(__name__) class Pusher(object): INITIAL_BACKOFF = 1000 - MAX_BACKOFF = 10 * 60 * 1000 + MAX_BACKOFF = 60 * 60 * 1000 + GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() + self.clock = self.hs.get_clock() self.user_name = user_name self.app = app self.app_display_name = app_display_name @@ -40,6 +43,7 @@ class Pusher(object): self.data = data self.last_token = last_token self.backoff_delay = Pusher.INITIAL_BACKOFF + self.failing_since = None @defer.inlineCallbacks def start(self): @@ -58,17 +62,51 @@ class Pusher(object): config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) - if (self.dispatchPush(chunk['chunk'][0])): + # limiting to 1 may get 1 event plus 1 presence event, so pick out the actual event + singleEvent = None + for c in chunk['chunk']: + if 'event_id' in c: # Hmmm... + singleEvent = c + break + if not singleEvent: + continue + + ret = yield self.dispatchPush(singleEvent) + if (ret): self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token_and_success(self.user_name, self.pushkey, + self.last_token, self.clock.time_msec()) + if self.failing_since: + self.failing_since = None + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) else: - logger.warn("Failed to dispatch push for user %s. Trying again in %dms", - self.user_name, self.backoff_delay) - yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *=2 - if self.backoff_delay > Pusher.MAX_BACKOFF: - self.backoff_delay = Pusher.MAX_BACKOFF + if not self.failing_since: + self.failing_since = self.clock.time_msec() + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + + if self.failing_since and self.failing_since < self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + # we really only give up so that if the URL gets fixed, we don't suddenly deliver a load + # of old notifications. + logger.warn("Giving up on a notification to user %s, pushkey %s", + self.user_name, self.pushkey) + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + + self.failing_since = None + self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + else: + logger.warn("Failed to dispatch push for user %s (failing for %dms)." + "Trying again in %dms", + self.user_name, + self.clock.time_msec() - self.failing_since, + self.backoff_delay + ) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *=2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF class PusherConfigException(Exception): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index f3c3ca819..33d735b97 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -14,13 +14,17 @@ # limitations under the License. from synapse.push import Pusher, PusherConfigException +from synapse.http.client import SimpleHttpClient + +from twisted.internet import defer import logging logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, last_token): + def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + last_token, last_success, failing_since): super(HttpPusher, self).__init__(_hs, user_name, app, @@ -28,12 +32,55 @@ class HttpPusher(Pusher): device_display_name, pushkey, data, - last_token) + last_token, + last_success, + failing_since) if 'url' not in data: raise PusherConfigException("'url' required in data for HTTP pusher") self.url = data['url'] + self.httpCli = SimpleHttpClient(self.hs) + self.data_minus_url = {} + self.data_minus_url.update(self.data) + del self.data_minus_url['url'] + def _build_notification_dict(self, event): + # we probably do not want to push for every presence update + # (we may want to be able to set up notifications when specific + # people sign in, but we'd want to only deliver the pertinent ones) + # Actually, presence events will not get this far now because we + # need to filter them out in the main Pusher code. + if 'event_id' not in event: + return None + + return { + 'notification': { + 'transition' : 'new', # everything is new for now: we don't have read receipts + 'id': event['event_id'], + 'type': event['type'], + 'from': event['user_id'], + # we may have to fetch this over federation and we can't trust it anyway: is it worth it? + #'fromDisplayName': 'Steve Stevington' + }, + #'counts': { -- we don't mark messages as read yet so we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': { + self.pushkey: { + 'data' : self.data_minus_url + } + } + } + + @defer.inlineCallbacks def dispatchPush(self, event): - print event - return True + notificationDict = self._build_notification_dict(event) + if not notificationDict: + defer.returnValue(True) + try: + yield self.httpCli.post_json_get_json(self.url, notificationDict) + except: + logger.exception("Failed to push %s ", self.url) + defer.returnValue(False) + defer.returnValue(True) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 436040f12..3fa5a4c4f 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -45,7 +45,9 @@ class PusherPool: "device_display_name": device_display_name, "pushkey": pushkey, "data": data, - "last_token": None + "last_token": None, + "last_success": None, + "failing_since": None }) self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) @@ -69,7 +71,9 @@ class PusherPool: device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], data=pusherdict['data'], - last_token=pusherdict['last_token'] + last_token=pusherdict['last_token'], + last_success=pusherdict['last_success'], + failing_since=pusherdict['failing_since'] ) else: raise PusherConfigException("Unknown pusher type '%s' for user %s" % diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 047a5f42d..ce158c4b1 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,7 +29,8 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers_after_id(self, min_id): sql = ( - "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, last_token " + "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, " + "last_token, last_success, failing_since " "FROM pushers " "WHERE id > ?" ) @@ -46,8 +47,9 @@ class PusherStore(SQLBaseStore): "device_display_name": r[5], "pushkey": r[6], "data": r[7], - "last_token": r[8] - + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] } for r in rows ] @@ -79,6 +81,20 @@ class PusherStore(SQLBaseStore): {'last_token': last_token} ) + @defer.inlineCallbacks + def update_pusher_last_token_and_success(self, user_name, pushkey, last_token, last_success): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token, 'last_success': last_success} + ) + + @defer.inlineCallbacks + def update_pusher_failing_since(self, user_name, pushkey, failing_since): + yield self._simple_update_one(PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'failing_since': failing_since} + ) + class PushersTable(Table): table_name = "pushers" @@ -92,7 +108,9 @@ class PushersTable(Table): "device_display_name", "pushkey", "data", - "last_token" + "last_token", + "last_success", + "failing_since" ] EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index 7f6852485..e83f7e743 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -23,6 +23,8 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey blob NOT NULL, data text, last_token TEXT, + last_success BIGINT, + failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (user_name, pushkey) ); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 7f6852485..e83f7e743 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -23,6 +23,8 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey blob NOT NULL, data text, last_token TEXT, + last_success BIGINT, + failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (user_name, pushkey) ); From bdc21e72820e148941bbecb36200d51ca340748d Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 2 Dec 2014 14:10:24 +0000 Subject: [PATCH 004/630] convert to spaces before I start a holy war --- contrib/jitsimeetbridge/jitsimeetbridge.py | 372 ++++++++++----------- 1 file changed, 186 insertions(+), 186 deletions(-) diff --git a/contrib/jitsimeetbridge/jitsimeetbridge.py b/contrib/jitsimeetbridge/jitsimeetbridge.py index dbc6f6ffa..15f8e1c48 100644 --- a/contrib/jitsimeetbridge/jitsimeetbridge.py +++ b/contrib/jitsimeetbridge/jitsimeetbridge.py @@ -39,43 +39,43 @@ ROOMDOMAIN="meet.jit.si" #ROOMDOMAIN="conference.jitsi.vuc.me" class TrivialMatrixClient: - def __init__(self, access_token): - self.token = None - self.access_token = access_token + def __init__(self, access_token): + self.token = None + self.access_token = access_token - def getEvent(self): - while True: - url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000" - if self.token: - url += "&from="+self.token - req = grequests.get(url) - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "incoming from matrix",obj - if 'end' not in obj: - continue - self.token = obj['end'] - if len(obj['chunk']): - return obj['chunk'][0] + def getEvent(self): + while True: + url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000" + if self.token: + url += "&from="+self.token + req = grequests.get(url) + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "incoming from matrix",obj + if 'end' not in obj: + continue + self.token = obj['end'] + if len(obj['chunk']): + return obj['chunk'][0] - def joinRoom(self, roomId): - url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token - print url - headers={ 'Content-Type': 'application/json' } - req = grequests.post(url, headers=headers, data='{}') - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "response: ",obj + def joinRoom(self, roomId): + url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token + print url + headers={ 'Content-Type': 'application/json' } + req = grequests.post(url, headers=headers, data='{}') + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "response: ",obj - def sendEvent(self, roomId, evType, event): - url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token - print url - print json.dumps(event) - headers={ 'Content-Type': 'application/json' } - req = grequests.post(url, headers=headers, data=json.dumps(event)) - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "response: ",obj + def sendEvent(self, roomId, evType, event): + url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token + print url + print json.dumps(event) + headers={ 'Content-Type': 'application/json' } + req = grequests.post(url, headers=headers, data=json.dumps(event)) + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "response: ",obj @@ -83,178 +83,178 @@ xmppClients = {} def matrixLoop(): - while True: - ev = matrixCli.getEvent() - print ev - if ev['type'] == 'm.room.member': - print 'membership event' - if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME: - roomId = ev['room_id'] - print "joining room %s" % (roomId) - matrixCli.joinRoom(roomId) - elif ev['type'] == 'm.room.message': - if ev['room_id'] in xmppClients: - print "already have a bridge for that user, ignoring" - continue - print "got message, connecting" - xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) - gevent.spawn(xmppClients[ev['room_id']].xmppLoop) - elif ev['type'] == 'm.call.invite': - print "Incoming call" - #sdp = ev['content']['offer']['sdp'] - #print "sdp: %s" % (sdp) - #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) - #gevent.spawn(xmppClients[ev['room_id']].xmppLoop) - elif ev['type'] == 'm.call.answer': - print "Call answered" - sdp = ev['content']['answer']['sdp'] - if ev['room_id'] not in xmppClients: - print "We didn't have a call for that room" - continue - # should probably check call ID too - xmppCli = xmppClients[ev['room_id']] - xmppCli.sendAnswer(sdp) - elif ev['type'] == 'm.call.hangup': - if ev['room_id'] in xmppClients: - xmppClients[ev['room_id']].stop() - del xmppClients[ev['room_id']] - + while True: + ev = matrixCli.getEvent() + print ev + if ev['type'] == 'm.room.member': + print 'membership event' + if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME: + roomId = ev['room_id'] + print "joining room %s" % (roomId) + matrixCli.joinRoom(roomId) + elif ev['type'] == 'm.room.message': + if ev['room_id'] in xmppClients: + print "already have a bridge for that user, ignoring" + continue + print "got message, connecting" + xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) + gevent.spawn(xmppClients[ev['room_id']].xmppLoop) + elif ev['type'] == 'm.call.invite': + print "Incoming call" + #sdp = ev['content']['offer']['sdp'] + #print "sdp: %s" % (sdp) + #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) + #gevent.spawn(xmppClients[ev['room_id']].xmppLoop) + elif ev['type'] == 'm.call.answer': + print "Call answered" + sdp = ev['content']['answer']['sdp'] + if ev['room_id'] not in xmppClients: + print "We didn't have a call for that room" + continue + # should probably check call ID too + xmppCli = xmppClients[ev['room_id']] + xmppCli.sendAnswer(sdp) + elif ev['type'] == 'm.call.hangup': + if ev['room_id'] in xmppClients: + xmppClients[ev['room_id']].stop() + del xmppClients[ev['room_id']] + class TrivialXmppClient: - def __init__(self, matrixRoom, userId): - self.rid = 0 - self.matrixRoom = matrixRoom - self.userId = userId - self.running = True + def __init__(self, matrixRoom, userId): + self.rid = 0 + self.matrixRoom = matrixRoom + self.userId = userId + self.running = True - def stop(self): - self.running = False + def stop(self): + self.running = False - def nextRid(self): - self.rid += 1 - return '%d' % (self.rid) + def nextRid(self): + self.rid += 1 + return '%d' % (self.rid) - def sendIq(self, xml): - fullXml = "%s" % (self.nextRid(), self.sid, xml) - #print "\t>>>%s" % (fullXml) - return self.xmppPoke(fullXml) - - def xmppPoke(self, xml): - headers = {'Content-Type': 'application/xml'} - req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml) - resps = grequests.map([req]) - obj = BeautifulSoup(resps[0].content) - return obj + def sendIq(self, xml): + fullXml = "%s" % (self.nextRid(), self.sid, xml) + #print "\t>>>%s" % (fullXml) + return self.xmppPoke(fullXml) - def sendAnswer(self, answer): - print "sdp from matrix client",answer - p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - jingle, out_err = p.communicate(answer) - jingle = jingle % { - 'tojid': self.callfrom, - 'action': 'session-accept', - 'initiator': self.callfrom, - 'responder': self.jid, - 'sid': self.callsid - } - print "answer jingle from sdp",jingle - res = self.sendIq(jingle) - print "reply from answer: ",res - - self.ssrcs = {} - jingleSoup = BeautifulSoup(jingle) - for cont in jingleSoup.iq.jingle.findAll('content'): - if cont.description: - self.ssrcs[cont['name']] = cont.description['ssrc'] - print "my ssrcs:",self.ssrcs + def xmppPoke(self, xml): + headers = {'Content-Type': 'application/xml'} + req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml) + resps = grequests.map([req]) + obj = BeautifulSoup(resps[0].content) + return obj - gevent.joinall([ - gevent.spawn(self.advertiseSsrcs) - ]) - - def advertiseSsrcs(self): + def sendAnswer(self, answer): + print "sdp from matrix client",answer + p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + jingle, out_err = p.communicate(answer) + jingle = jingle % { + 'tojid': self.callfrom, + 'action': 'session-accept', + 'initiator': self.callfrom, + 'responder': self.jid, + 'sid': self.callsid + } + print "answer jingle from sdp",jingle + res = self.sendIq(jingle) + print "reply from answer: ",res + + self.ssrcs = {} + jingleSoup = BeautifulSoup(jingle) + for cont in jingleSoup.iq.jingle.findAll('content'): + if cont.description: + self.ssrcs[cont['name']] = cont.description['ssrc'] + print "my ssrcs:",self.ssrcs + + gevent.joinall([ + gevent.spawn(self.advertiseSsrcs) + ]) + + def advertiseSsrcs(self): time.sleep(7) - print "SSRC spammer started" - while self.running: - ssrcMsg = "%(nick)s" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] } - res = self.sendIq(ssrcMsg) - print "reply from ssrc announce: ",res - time.sleep(10) - - + print "SSRC spammer started" + while self.running: + ssrcMsg = "%(nick)s" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] } + res = self.sendIq(ssrcMsg) + print "reply from ssrc announce: ",res + time.sleep(10) - def xmppLoop(self): - self.matrixCallId = time.time() - res = self.xmppPoke("" % (self.nextRid(), HOST)) - print res - self.sid = res.body['sid'] - print "sid %s" % (self.sid) - res = self.sendIq("") + def xmppLoop(self): + self.matrixCallId = time.time() + res = self.xmppPoke("" % (self.nextRid(), HOST)) - res = self.xmppPoke("" % (self.nextRid(), self.sid, HOST)) - - res = self.sendIq("") - print res + print res + self.sid = res.body['sid'] + print "sid %s" % (self.sid) - self.jid = res.body.iq.bind.jid.string - print "jid: %s" % (self.jid) - self.shortJid = self.jid.split('-')[0] + res = self.sendIq("") - res = self.sendIq("") + res = self.xmppPoke("" % (self.nextRid(), self.sid, HOST)) - #randomthing = res.body.iq['to'] - #whatsitpart = randomthing.split('-')[0] + res = self.sendIq("") + print res - #print "other random bind thing: %s" % (randomthing) + self.jid = res.body.iq.bind.jid.string + print "jid: %s" % (self.jid) + self.shortJid = self.jid.split('-')[0] - # advertise preence to the jitsi room, with our nick - res = self.sendIq("%s" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)) - self.muc = {'users': []} - for p in res.body.findAll('presence'): - u = {} - u['shortJid'] = p['from'].split('/')[1] - if p.c and p.c.nick: - u['nick'] = p.c.nick.string - self.muc['users'].append(u) - print "muc: ",self.muc + res = self.sendIq("") - # wait for stuff - while True: - print "waiting..." - res = self.sendIq("") - print "got from stream: ",res - if res.body.iq: - jingles = res.body.iq.findAll('jingle') - if len(jingles): - self.callfrom = res.body.iq['from'] - self.handleInvite(jingles[0]) - elif 'type' in res.body and res.body['type'] == 'terminate': - self.running = False - del xmppClients[self.matrixRoom] - return + #randomthing = res.body.iq['to'] + #whatsitpart = randomthing.split('-')[0] + + #print "other random bind thing: %s" % (randomthing) + + # advertise preence to the jitsi room, with our nick + res = self.sendIq("%s" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)) + self.muc = {'users': []} + for p in res.body.findAll('presence'): + u = {} + u['shortJid'] = p['from'].split('/')[1] + if p.c and p.c.nick: + u['nick'] = p.c.nick.string + self.muc['users'].append(u) + print "muc: ",self.muc + + # wait for stuff + while True: + print "waiting..." + res = self.sendIq("") + print "got from stream: ",res + if res.body.iq: + jingles = res.body.iq.findAll('jingle') + if len(jingles): + self.callfrom = res.body.iq['from'] + self.handleInvite(jingles[0]) + elif 'type' in res.body and res.body['type'] == 'terminate': + self.running = False + del xmppClients[self.matrixRoom] + return + + def handleInvite(self, jingle): + self.initiator = jingle['initiator'] + self.callsid = jingle['sid'] + p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + print "raw jingle invite",str(jingle) + sdp, out_err = p.communicate(str(jingle)) + print "transformed remote offer sdp",sdp + inviteEvent = { + 'offer': { + 'type': 'offer', + 'sdp': sdp + }, + 'call_id': self.matrixCallId, + 'version': 0, + 'lifetime': 30000 + } + matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent) - def handleInvite(self, jingle): - self.initiator = jingle['initiator'] - self.callsid = jingle['sid'] - p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - print "raw jingle invite",str(jingle) - sdp, out_err = p.communicate(str(jingle)) - print "transformed remote offer sdp",sdp - inviteEvent = { - 'offer': { - 'type': 'offer', - 'sdp': sdp - }, - 'call_id': self.matrixCallId, - 'version': 0, - 'lifetime': 30000 - } - matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent) - matrixCli = TrivialMatrixClient(ACCESS_TOKEN) gevent.joinall([ - gevent.spawn(matrixLoop) + gevent.spawn(matrixLoop) ]) From 88af58d41d561f1d9f6bbbfb2a1e8bd00dbbe638 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 3 Dec 2014 13:37:02 +0000 Subject: [PATCH 005/630] Update to app_id / app_instance_id (partially) and mangle to be PEP8 compliant. --- synapse/push/__init__.py | 99 ++++++++++++++++++++--------- synapse/push/httppusher.py | 75 ++++++++++++---------- synapse/push/pusherpool.py | 75 +++++++++++++--------- synapse/rest/pusher.py | 32 ++++++---- synapse/storage/pusher.py | 54 +++++++++------- synapse/storage/schema/delta/v7.sql | 5 +- synapse/storage/schema/pusher.sql | 5 +- 7 files changed, 214 insertions(+), 131 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index a96f0f018..5fca3bd77 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -24,90 +24,127 @@ import logging logger = logging.getLogger(__name__) + class Pusher(object): INITIAL_BACKOFF = 1000 MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + def __init__(self, _hs, user_name, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() self.user_name = user_name - self.app = app + self.app_id = app_id + self.app_instance_id = app_instance_id self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey self.data = data self.last_token = last_token + self.last_success = last_success # not actually used self.backoff_delay = Pusher.INITIAL_BACKOFF - self.failing_since = None + self.failing_since = failing_since @defer.inlineCallbacks def start(self): if not self.last_token: - # First-time setup: get a token to start from (we can't just start from no token, ie. 'now' - # because we need the result to be reproduceable in case we fail to dispatch the push) + # First-time setup: get a token to start from (we can't + # just start from no token, ie. 'now' + # because we need the result to be reproduceable in case + # we fail to dispatch the push) config = PaginationConfig(from_token=None, limit='1') - chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=0) + chunk = yield self.evStreamHandler.get_stream( + self.user_name, config, timeout=0) self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token( + self.user_name, self.pushkey, self.last_token) logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) while True: from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') - chunk = yield self.evStreamHandler.get_stream(self.user_name, config, timeout=100*365*24*60*60*1000) + chunk = yield self.evStreamHandler.get_stream( + self.user_name, config, timeout=100*365*24*60*60*1000) - # limiting to 1 may get 1 event plus 1 presence event, so pick out the actual event - singleEvent = None + # limiting to 1 may get 1 event plus 1 presence event, so + # pick out the actual event + single_event = None for c in chunk['chunk']: - if 'event_id' in c: # Hmmm... - singleEvent = c + if 'event_id' in c: # Hmmm... + single_event = c break - if not singleEvent: + if not single_event: continue - ret = yield self.dispatchPush(singleEvent) - if (ret): + ret = yield self.dispatch_push(single_event) + if ret: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token_and_success(self.user_name, self.pushkey, - self.last_token, self.clock.time_msec()) + self.store.update_pusher_last_token_and_success( + self.user_name, + self.pushkey, + self.last_token, + self.clock.time_msec() + ) if self.failing_since: self.failing_since = None - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since) else: if not self.failing_since: self.failing_since = self.clock.time_msec() - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since + ) - if self.failing_since and self.failing_since < self.clock.time_msec() - Pusher.GIVE_UP_AFTER: - # we really only give up so that if the URL gets fixed, we don't suddenly deliver a load + if self.failing_since and \ + self.failing_since < \ + self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + # we really only give up so that if the URL gets + # fixed, we don't suddenly deliver a load # of old notifications. - logger.warn("Giving up on a notification to user %s, pushkey %s", + logger.warn("Giving up on a notification to user %s, " + "pushkey %s", self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token(self.user_name, self.pushkey, self.last_token) + self.store.update_pusher_last_token( + self.user_name, + self.pushkey, + self.last_token + ) self.failing_since = None - self.store.update_pusher_failing_since(self.user_name, self.pushkey, self.failing_since) - else: - logger.warn("Failed to dispatch push for user %s (failing for %dms)." - "Trying again in %dms", - self.user_name, - self.clock.time_msec() - self.failing_since, - self.backoff_delay + self.store.update_pusher_failing_since( + self.user_name, + self.pushkey, + self.failing_since ) + else: + logger.warn("Failed to dispatch push for user %s " + "(failing for %dms)." + "Trying again in %dms", + self.user_name, + self.clock.time_msec() - self.failing_since, + self.backoff_delay + ) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *=2 + self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: self.backoff_delay = Pusher.MAX_BACKOFF + def dispatch_push(self, p): + pass + class PusherConfigException(Exception): def __init__(self, msg): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 33d735b97..fd7fe4e39 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -22,21 +22,28 @@ import logging logger = logging.getLogger(__name__) + class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app, app_display_name, device_display_name, pushkey, data, + def __init__(self, _hs, user_name, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): - super(HttpPusher, self).__init__(_hs, - user_name, - app, - app_display_name, - device_display_name, - pushkey, - data, - last_token, - last_success, - failing_since) + super(HttpPusher, self).__init__( + _hs, + user_name, + app_id, + app_instance_id, + app_display_name, + device_display_name, + pushkey, + data, + last_token, + last_success, + failing_since + ) if 'url' not in data: - raise PusherConfigException("'url' required in data for HTTP pusher") + raise PusherConfigException( + "'url' required in data for HTTP pusher" + ) self.url = data['url'] self.httpCli = SimpleHttpClient(self.hs) self.data_minus_url = {} @@ -53,34 +60,36 @@ class HttpPusher(Pusher): return None return { - 'notification': { - 'transition' : 'new', # everything is new for now: we don't have read receipts - 'id': event['event_id'], - 'type': event['type'], - 'from': event['user_id'], - # we may have to fetch this over federation and we can't trust it anyway: is it worth it? - #'fromDisplayName': 'Steve Stevington' - }, - #'counts': { -- we don't mark messages as read yet so we have no way of knowing - # 'unread': 1, - # 'missedCalls': 2 - # }, - 'devices': { - self.pushkey: { - 'data' : self.data_minus_url + 'notification': { + 'transition': 'new', + # everything is new for now: we don't have read receipts + 'id': event['event_id'], + 'type': event['type'], + 'from': event['user_id'], + # we may have to fetch this over federation and we + # can't trust it anyway: is it worth it? + #'fromDisplayName': 'Steve Stevington' + }, + #'counts': { -- we don't mark messages as read yet so + # we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': { + self.pushkey: { + 'data': self.data_minus_url } - } + } } @defer.inlineCallbacks - def dispatchPush(self, event): - notificationDict = self._build_notification_dict(event) - if not notificationDict: + def dispatch_push(self, event): + notification_dict = self._build_notification_dict(event) + if not notification_dict: defer.returnValue(True) try: - yield self.httpCli.post_json_get_json(self.url, notificationDict) + yield self.httpCli.post_json_get_json(self.url, notification_dict) except: logger.exception("Failed to push %s ", self.url) defer.returnValue(False) defer.returnValue(True) - diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 3fa5a4c4f..045c36f3b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -34,13 +34,17 @@ class PusherPool: def start(self): self._pushers_added() - def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): - # we try to create the pusher just to validate the config: it will then get pulled out of the database, - # recreated, added and started: this means we have only one code path adding pushers. + def add_pusher(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data): + # we try to create the pusher just to validate the config: it + # will then get pulled out of the database, + # recreated, added and started: this means we have only one + # code path adding pushers. self._create_pusher({ "user_name": user_name, "kind": kind, - "app": app, + "app_id": app_id, + "app_instance_id": app_instance_id, "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, @@ -49,42 +53,55 @@ class PusherPool: "last_success": None, "failing_since": None }) - self._add_pusher_to_store(user_name, kind, app, app_display_name, device_display_name, pushkey, data) + self._add_pusher_to_store(user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, + pushkey, data) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): - yield self.store.add_pusher(user_name=user_name, - kind=kind, - app=app, - app_display_name=app_display_name, - device_display_name=device_display_name, - pushkey=pushkey, - data=json.dumps(data)) + def _add_pusher_to_store(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, + pushkey, data): + yield self.store.add_pusher( + user_name=user_name, + kind=kind, + app_id=app_id, + app_instance_id=app_instance_id, + app_display_name=app_display_name, + device_display_name=device_display_name, + pushkey=pushkey, + data=json.dumps(data) + ) self._pushers_added() def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': - return HttpPusher(self.hs, - user_name=pusherdict['user_name'], - app=pusherdict['app'], - app_display_name=pusherdict['app_display_name'], - device_display_name=pusherdict['device_display_name'], - pushkey=pusherdict['pushkey'], - data=pusherdict['data'], - last_token=pusherdict['last_token'], - last_success=pusherdict['last_success'], - failing_since=pusherdict['failing_since'] - ) + return HttpPusher( + self.hs, + user_name=pusherdict['user_name'], + app_id=pusherdict['app_id'], + app_instance_id=pusherdict['app_instance_id'], + app_display_name=pusherdict['app_display_name'], + device_display_name=pusherdict['device_display_name'], + pushkey=pusherdict['pushkey'], + data=pusherdict['data'], + last_token=pusherdict['last_token'], + last_success=pusherdict['last_success'], + failing_since=pusherdict['failing_since'] + ) else: - raise PusherConfigException("Unknown pusher type '%s' for user %s" % - (pusherdict['kind'], pusherdict['user_name'])) + raise PusherConfigException( + "Unknown pusher type '%s' for user %s" % + (pusherdict['kind'], pusherdict['user_name']) + ) @defer.inlineCallbacks def _pushers_added(self): - pushers = yield self.store.get_all_pushers_after_id(self.last_pusher_started) + pushers = yield self.store.get_all_pushers_after_id( + self.last_pusher_started + ) for p in pushers: p['data'] = json.loads(p['data']) - if (len(pushers)): + if len(pushers): self.last_pusher_started = pushers[-1]['id'] self._start_pushers(pushers) @@ -95,4 +112,4 @@ class PusherPool: p = self._create_pusher(pusherdict) if p: self.pushers.append(p) - p.start() \ No newline at end of file + p.start() diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 85d0d1c8c..a39341cd8 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -31,30 +31,37 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) - reqd = ['kind', 'app', 'app_display_name', 'device_display_name', 'data'] + reqd = ['kind', 'app_id', 'app_instance_id', 'app_display_name', + 'device_display_name', 'data'] missing = [] for i in reqd: if i not in content: missing.append(i) if len(missing): - raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) + raise SynapseError(400, "Missing parameters: "+','.join(missing), + errcode=Codes.MISSING_PARAM) pusher_pool = self.hs.get_pusherpool() try: - pusher_pool.add_pusher(user_name=user.to_string(), - kind=content['kind'], - app=content['app'], - app_display_name=content['app_display_name'], - device_display_name=content['device_display_name'], - pushkey=pushkey, - data=content['data']) + pusher_pool.add_pusher( + user_name=user.to_string(), + kind=content['kind'], + app_id=content['app_id'], + app_instance_id=content['app_instance_id'], + app_display_name=content['app_display_name'], + device_display_name=content['device_display_name'], + pushkey=pushkey, + data=content['data'] + ) except PusherConfigException as pce: - raise SynapseError(400, "Config Error: "+pce.message, errcode=Codes.MISSING_PARAM) + raise SynapseError(400, "Config Error: "+pce.message, + errcode=Codes.MISSING_PARAM) defer.returnValue((200, {})) - def on_OPTIONS(self, request): - return (200, {}) + def on_OPTIONS(self, _): + return 200, {} + # XXX: C+ped from rest/room.py - surely this should be common? def _parse_json(request): @@ -67,5 +74,6 @@ def _parse_json(request): except ValueError: raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + def register_servlets(hs, http_server): PusherRestServlet(hs).register(http_server) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index ce158c4b1..a858e46f3 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -25,11 +25,13 @@ import logging logger = logging.getLogger(__name__) + class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers_after_id(self, min_id): sql = ( - "SELECT id, user_name, kind, app, app_display_name, device_display_name, pushkey, data, " + "SELECT id, user_name, kind, app_id, app_instance_id," + "app_display_name, device_display_name, pushkey, data, " "last_token, last_success, failing_since " "FROM pushers " "WHERE id > ?" @@ -42,14 +44,15 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "app_id": r[3], + "app_instance_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -57,12 +60,14 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app, app_display_name, device_display_name, pushkey, data): + def add_pusher(self, user_name, kind, app_id, app_instance_id, + app_display_name, device_display_name, pushkey, data): try: yield self._simple_insert(PushersTable.table_name, dict( user_name=user_name, kind=kind, - app=app, + app_id=app_id, + app_instance_id=app_instance_id, app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, @@ -76,23 +81,27 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def update_pusher_last_token(self, user_name, pushkey, last_token): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'last_token': last_token} + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token} ) @defer.inlineCallbacks - def update_pusher_last_token_and_success(self, user_name, pushkey, last_token, last_success): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'last_token': last_token, 'last_success': last_success} + def update_pusher_last_token_and_success(self, user_name, pushkey, + last_token, last_success): + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'last_token': last_token, 'last_success': last_success} ) @defer.inlineCallbacks def update_pusher_failing_since(self, user_name, pushkey, failing_since): - yield self._simple_update_one(PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, - {'failing_since': failing_since} + yield self._simple_update_one( + PushersTable.table_name, + {'user_name': user_name, 'pushkey': pushkey}, + {'failing_since': failing_since} ) @@ -103,7 +112,8 @@ class PushersTable(Table): "id", "user_name", "kind", - "app" + "app_id", + "app_instance_id", "app_display_name", "device_display_name", "pushkey", diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index e83f7e743..b60aeda75 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -17,11 +17,12 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, kind varchar(8) NOT NULL, - app varchar(64) NOT NULL, + app_id varchar(64) NOT NULL, + app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, - data text, + data blob, last_token TEXT, last_success BIGINT, failing_since BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index e83f7e743..b60aeda75 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -17,11 +17,12 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, kind varchar(8) NOT NULL, - app varchar(64) NOT NULL, + app_id varchar(64) NOT NULL, + app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, - data text, + data blob, last_token TEXT, last_success BIGINT, failing_since BIGINT, From 9728c305a34a1f9546d2ce0ef4c54352dc55a16d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:49:22 +0000 Subject: [PATCH 006/630] after a few rethinks, a working implementation of pushers. --- synapse/push/__init__.py | 12 +++-- synapse/push/httppusher.py | 25 ++++----- synapse/push/pusherpool.py | 47 ++++++++-------- synapse/rest/pusher.py | 13 +++-- synapse/storage/_base.py | 45 ++++++++++++++++ synapse/storage/pusher.py | 83 ++++++++++++++++++++--------- synapse/storage/schema/delta/v7.sql | 3 +- synapse/storage/schema/pusher.sql | 3 +- 8 files changed, 158 insertions(+), 73 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5fca3bd77..5fe8719fe 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -30,7 +30,7 @@ class Pusher(object): MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app_id, app_instance_id, + def __init__(self, _hs, user_name, app_id, app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): self.hs = _hs @@ -39,7 +39,6 @@ class Pusher(object): self.clock = self.hs.get_clock() self.user_name = user_name self.app_id = app_id - self.app_instance_id = app_instance_id self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey @@ -48,6 +47,7 @@ class Pusher(object): self.last_success = last_success # not actually used self.backoff_delay = Pusher.INITIAL_BACKOFF self.failing_since = failing_since + self.alive = True @defer.inlineCallbacks def start(self): @@ -65,7 +65,7 @@ class Pusher(object): logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) - while True: + while self.alive: from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream( @@ -81,6 +81,9 @@ class Pusher(object): if not single_event: continue + if not self.alive: + continue + ret = yield self.dispatch_push(single_event) if ret: self.backoff_delay = Pusher.INITIAL_BACKOFF @@ -142,6 +145,9 @@ class Pusher(object): if self.backoff_delay > Pusher.MAX_BACKOFF: self.backoff_delay = Pusher.MAX_BACKOFF + def stop(self): + self.alive = False + def dispatch_push(self, p): pass diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index fd7fe4e39..f94f67339 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -24,14 +24,13 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app_id, app_instance_id, + def __init__(self, _hs, user_name, app_id, app_display_name, device_display_name, pushkey, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, user_name, app_id, - app_instance_id, app_display_name, device_display_name, pushkey, @@ -69,16 +68,18 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'fromDisplayName': 'Steve Stevington' - }, - #'counts': { -- we don't mark messages as read yet so - # we have no way of knowing - # 'unread': 1, - # 'missedCalls': 2 - # }, - 'devices': { - self.pushkey: { - 'data': self.data_minus_url - } + #'counts': { -- we don't mark messages as read yet so + # we have no way of knowing + # 'unread': 1, + # 'missedCalls': 2 + # }, + 'devices': [ + { + 'app_id': self.app_id, + 'pushkey': self.pushkey, + 'data': self.data_minus_url + } + ] } } diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 045c36f3b..d34ef3f6c 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -24,17 +24,23 @@ import json logger = logging.getLogger(__name__) + class PusherPool: def __init__(self, _hs): self.hs = _hs self.store = self.hs.get_datastore() - self.pushers = [] + self.pushers = {} self.last_pusher_started = -1 + @defer.inlineCallbacks def start(self): - self._pushers_added() + pushers = yield self.store.get_all_pushers() + for p in pushers: + p['data'] = json.loads(p['data']) + self._start_pushers(pushers) - def add_pusher(self, user_name, kind, app_id, app_instance_id, + @defer.inlineCallbacks + def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -44,7 +50,6 @@ class PusherPool: "user_name": user_name, "kind": kind, "app_id": app_id, - "app_instance_id": app_instance_id, "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, @@ -53,25 +58,26 @@ class PusherPool: "last_success": None, "failing_since": None }) - self._add_pusher_to_store(user_name, kind, app_id, app_instance_id, - app_display_name, device_display_name, - pushkey, data) + yield self._add_pusher_to_store( + user_name, kind, app_id, + app_display_name, device_display_name, + pushkey, data + ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app_id, app_instance_id, + def _add_pusher_to_store(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): yield self.store.add_pusher( user_name=user_name, kind=kind, app_id=app_id, - app_instance_id=app_instance_id, app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, data=json.dumps(data) ) - self._pushers_added() + self._refresh_pusher((app_id, pushkey)) def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': @@ -79,7 +85,6 @@ class PusherPool: self.hs, user_name=pusherdict['user_name'], app_id=pusherdict['app_id'], - app_instance_id=pusherdict['app_instance_id'], app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], @@ -95,21 +100,21 @@ class PusherPool: ) @defer.inlineCallbacks - def _pushers_added(self): - pushers = yield self.store.get_all_pushers_after_id( - self.last_pusher_started + def _refresh_pusher(self, app_id_pushkey): + p = yield self.store.get_pushers_by_app_id_and_pushkey( + app_id_pushkey ) - for p in pushers: - p['data'] = json.loads(p['data']) - if len(pushers): - self.last_pusher_started = pushers[-1]['id'] + p['data'] = json.loads(p['data']) - self._start_pushers(pushers) + self._start_pushers([p]) def _start_pushers(self, pushers): - logger.info("Starting %d pushers", (len(pushers))) + logger.info("Starting %d pushers", len(pushers)) for pusherdict in pushers: p = self._create_pusher(pusherdict) if p: - self.pushers.append(p) + fullid = "%s:%s" % (pusherdict['app_id'], pusherdict['pushkey']) + if fullid in self.pushers: + self.pushers[fullid].stop() + self.pushers[fullid] = p p.start() diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index a39341cd8..5b371318d 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -23,16 +23,16 @@ import json class PusherRestServlet(RestServlet): - PATTERN = client_path_pattern("/pushers/(?P[\w]*)$") + PATTERN = client_path_pattern("/pushers/set$") @defer.inlineCallbacks - def on_PUT(self, request, pushkey): + def on_POST(self, request): user = yield self.auth.get_user_by_req(request) content = _parse_json(request) - reqd = ['kind', 'app_id', 'app_instance_id', 'app_display_name', - 'device_display_name', 'data'] + reqd = ['kind', 'app_id', 'app_display_name', + 'device_display_name', 'pushkey', 'data'] missing = [] for i in reqd: if i not in content: @@ -43,14 +43,13 @@ class PusherRestServlet(RestServlet): pusher_pool = self.hs.get_pusherpool() try: - pusher_pool.add_pusher( + yield pusher_pool.add_pusher( user_name=user.to_string(), kind=content['kind'], app_id=content['app_id'], - app_instance_id=content['app_instance_id'], app_display_name=content['app_display_name'], device_display_name=content['device_display_name'], - pushkey=pushkey, + pushkey=content['pushkey'], data=content['data'] ) except PusherConfigException as pce: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4881f0336..eb8cc4a9f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -195,6 +195,51 @@ class SQLBaseStore(object): txn.execute(sql, values.values()) return txn.lastrowid + def _simple_upsert(self, table, keyvalues, values): + """ + :param table: The table to upsert into + :param keyvalues: Dict of the unique key tables and their new values + :param values: Dict of all the nonunique columns and their new values + :return: A deferred + """ + return self.runInteraction( + "_simple_upsert", + self._simple_upsert_txn, table, keyvalues, values + ) + + def _simple_upsert_txn(self, txn, table, keyvalues, values): + # Try to update + sql = "UPDATE %s SET %s WHERE %s" % ( + table, + ", ".join("%s = ?" % (k) for k in values), + " AND ".join("%s = ?" % (k) for k in keyvalues) + ) + sqlargs = values.values() + keyvalues.values() + logger.debug( + "[SQL] %s Args=%s", + sql, sqlargs, + ) + + txn.execute(sql, sqlargs) + if txn.rowcount == 0: + # We didn't update and rows so insert a new one + allvalues = {} + allvalues.update(keyvalues) + allvalues.update(values) + + sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + table, + ", ".join(k for k in allvalues), + ", ".join("?" for _ in allvalues) + ) + logger.debug( + "[SQL] %s Args=%s", + sql, keyvalues.values(), + ) + txn.execute(sql, allvalues.values()) + + + def _simple_select_one(self, table, keyvalues, retcols, allow_none=False): """Executes a SELECT query on the named table, which is expected to diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index a858e46f3..deabd9cd2 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -28,16 +28,18 @@ logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): @defer.inlineCallbacks - def get_all_pushers_after_id(self, min_id): + def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( - "SELECT id, user_name, kind, app_id, app_instance_id," + "SELECT id, user_name, kind, app_id," "app_display_name, device_display_name, pushkey, data, " "last_token, last_success, failing_since " "FROM pushers " - "WHERE id > ?" + "WHERE app_id = ? AND pushkey = ?" ) - rows = yield self._execute(None, sql, min_id) + rows = yield self._execute( + None, sql, app_id_and_pushkey[0], app_id_and_pushkey[1] + ) ret = [ { @@ -45,14 +47,43 @@ class PusherStore(SQLBaseStore): "user_name": r[1], "kind": r[2], "app_id": r[3], - "app_instance_id": r[4], - "app_display_name": r[5], - "device_display_name": r[6], - "pushkey": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] + } + for r in rows + ] + + defer.returnValue(ret[0]) + + @defer.inlineCallbacks + def get_all_pushers(self): + sql = ( + "SELECT id, user_name, kind, app_id," + "app_display_name, device_display_name, pushkey, data, " + "last_token, last_success, failing_since " + "FROM pushers" + ) + + rows = yield self._execute(None, sql) + + ret = [ + { + "id": r[0], + "user_name": r[1], + "kind": r[2], + "app_id": r[3], + "app_display_name": r[4], + "device_display_name": r[5], + "pushkey": r[6], + "data": r[7], + "last_token": r[8], + "last_success": r[9], + "failing_since": r[10] } for r in rows ] @@ -60,21 +91,22 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, app_instance_id, + def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, pushkey, data): try: - yield self._simple_insert(PushersTable.table_name, dict( - user_name=user_name, - kind=kind, - app_id=app_id, - app_instance_id=app_instance_id, - app_display_name=app_display_name, - device_display_name=device_display_name, - pushkey=pushkey, - data=data - )) - except IntegrityError: - raise StoreError(409, "Pushkey in use.") + yield self._simple_upsert( + PushersTable.table_name, + dict( + app_id=app_id, + pushkey=pushkey, + ), + dict( + user_name=user_name, + kind=kind, + app_display_name=app_display_name, + device_display_name=device_display_name, + data=data + )) except Exception as e: logger.error("create_pusher with failed: %s", e) raise StoreError(500, "Problem creating pusher.") @@ -113,7 +145,6 @@ class PushersTable(Table): "user_name", "kind", "app_id", - "app_instance_id", "app_display_name", "device_display_name", "pushkey", diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v7.sql index b60aeda75..799e48d78 100644 --- a/synapse/storage/schema/delta/v7.sql +++ b/synapse/storage/schema/delta/v7.sql @@ -18,7 +18,6 @@ CREATE TABLE IF NOT EXISTS pushers ( user_name TEXT NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, - app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, @@ -27,5 +26,5 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (user_name, pushkey) + UNIQUE (app_id, pushkey) ); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index b60aeda75..799e48d78 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -18,7 +18,6 @@ CREATE TABLE IF NOT EXISTS pushers ( user_name TEXT NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, - app_instance_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, @@ -27,5 +26,5 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, FOREIGN KEY(user_name) REFERENCES users(name), - UNIQUE (user_name, pushkey) + UNIQUE (app_id, pushkey) ); From fc7c5e9cd7e0b1e29984233249311abe5cf23735 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:51:29 +0000 Subject: [PATCH 007/630] Rename the pusher SQL delta to v9 which the next free one --- synapse/storage/schema/delta/{v7.sql => v9.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename synapse/storage/schema/delta/{v7.sql => v9.sql} (100%) diff --git a/synapse/storage/schema/delta/v7.sql b/synapse/storage/schema/delta/v9.sql similarity index 100% rename from synapse/storage/schema/delta/v7.sql rename to synapse/storage/schema/delta/v9.sql From 173264b656b480a2f3634f49e78fd6093633af56 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:53:10 +0000 Subject: [PATCH 008/630] ...and bump SCHEMA_VERSION --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 642e5e289..348c3b259 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 8 +SCHEMA_VERSION = 9 class _RollbackButIsFineException(Exception): From 4c7ad50f6e50b95dfa9e0961a504e2f0d5b6921a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 14:55:04 +0000 Subject: [PATCH 009/630] Thank you, pyflakes --- synapse/storage/pusher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index deabd9cd2..9b5170a5f 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -18,7 +18,6 @@ import collections from ._base import SQLBaseStore, Table from twisted.internet import defer -from sqlite3 import IntegrityError from synapse.api.errors import StoreError import logging From afa953a29301dcae40606171ed4cdac90eefab63 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 15:11:06 +0000 Subject: [PATCH 010/630] schema version is now 10 --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/{v9.sql => v10.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/delta/{v9.sql => v10.sql} (100%) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 348c3b259..ad1765e04 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v10.sql similarity index 100% rename from synapse/storage/schema/delta/v9.sql rename to synapse/storage/schema/delta/v10.sql From fead431c181918558207f5115bd678a8984ce8d5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Dec 2014 18:44:33 +0000 Subject: [PATCH 011/630] If we didn't get any events, advance the token or we'll just keep not getting the same events again. --- synapse/push/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5fe8719fe..c5586be7b 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -79,6 +79,7 @@ class Pusher(object): single_event = c break if not single_event: + self.last_token = chunk['end'] continue if not self.alive: From 70d0a453f353bf350e02d5306a98126f6b318c88 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 13 Jan 2015 13:14:41 +0000 Subject: [PATCH 012/630] Split out function to decide whether to notify or a given event --- synapse/push/__init__.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index c5586be7b..f4795d559 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -49,6 +49,17 @@ class Pusher(object): self.failing_since = failing_since self.alive = True + def _should_notify_for_event(self, ev): + """ + This should take into account notification settings that the user + has configured both globally and per-room when we have the ability + to do such things. + """ + if ev['user_id'] == self.user_name: + # let's assume you probably know about messages you sent yourself + return False + return True + @defer.inlineCallbacks def start(self): if not self.last_token: @@ -85,8 +96,12 @@ class Pusher(object): if not self.alive: continue - ret = yield self.dispatch_push(single_event) - if ret: + processed = False + if self._should_notify_for_event(single_event): + processed = yield self.dispatch_push(single_event) + else: + processed = True + if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token_and_success( From 2cb30767fa5e428f82c6c3ebced15d568d671c3c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 13 Jan 2015 19:48:37 +0000 Subject: [PATCH 013/630] Honour the 'rejected' return from push gateways Add a timestamp to push tokens so we know the last time they we got them from the device. Send it to the push gateways so it can determine whether its failure is more recent than the token. Stop and remove pushers that have been rejected. --- synapse/push/__init__.py | 37 +++++++++++++++++++++++++--- synapse/push/httppusher.py | 15 +++++++---- synapse/push/pusherpool.py | 12 +++++++++ synapse/storage/pusher.py | 34 ++++++++++++++++--------- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 6 files changed, 81 insertions(+), 19 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index f4795d559..839f66639 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -31,8 +31,8 @@ class Pusher(object): GIVE_UP_AFTER = 24 * 60 * 60 * 1000 def __init__(self, _hs, user_name, app_id, - app_display_name, device_display_name, pushkey, data, - last_token, last_success, failing_since): + app_display_name, device_display_name, pushkey, pushkey_ts, + data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() @@ -42,6 +42,7 @@ class Pusher(object): self.app_display_name = app_display_name self.device_display_name = device_display_name self.pushkey = pushkey + self.pushkey_ts = pushkey_ts self.data = data self.last_token = last_token self.last_success = last_success # not actually used @@ -98,9 +99,31 @@ class Pusher(object): processed = False if self._should_notify_for_event(single_event): - processed = yield self.dispatch_push(single_event) + rejected = yield self.dispatch_push(single_event) + if not rejected == False: + processed = True + for pk in rejected: + if pk != self.pushkey: + # for sanity, we only remove the pushkey if it + # was the one we actually sent... + logger.warn( + ("Ignoring rejected pushkey %s because we" + + "didn't send it"), (pk,) + ) + else: + logger.info( + "Pushkey %s was rejected: removing", + pk + ) + yield self.hs.get_pusherpool().remove_pusher( + self.app_id, pk + ) else: processed = True + + if not self.alive: + continue + if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] @@ -165,6 +188,14 @@ class Pusher(object): self.alive = False def dispatch_push(self, p): + """ + Overridden by implementing classes to actually deliver the notification + :param p: The event to notify for as a single event from the event stream + :return: If the notification was delivered, an array containing any + pushkeys that were rejected by the push gateway. + False if the notification could not be delivered (ie. + should be retried). + """ pass diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index f94f67339..bcfa06e2a 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -25,8 +25,8 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): def __init__(self, _hs, user_name, app_id, - app_display_name, device_display_name, pushkey, data, - last_token, last_success, failing_since): + app_display_name, device_display_name, pushkey, pushkey_ts, + data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, user_name, @@ -34,6 +34,7 @@ class HttpPusher(Pusher): app_display_name, device_display_name, pushkey, + pushkey_ts, data, last_token, last_success, @@ -77,6 +78,7 @@ class HttpPusher(Pusher): { 'app_id': self.app_id, 'pushkey': self.pushkey, + 'pushkeyTs': long(self.pushkey_ts / 1000), 'data': self.data_minus_url } ] @@ -87,10 +89,13 @@ class HttpPusher(Pusher): def dispatch_push(self, event): notification_dict = self._build_notification_dict(event) if not notification_dict: - defer.returnValue(True) + defer.returnValue([]) try: - yield self.httpCli.post_json_get_json(self.url, notification_dict) + resp = yield self.httpCli.post_json_get_json(self.url, notification_dict) except: logger.exception("Failed to push %s ", self.url) defer.returnValue(False) - defer.returnValue(True) + rejected = [] + if 'rejected' in resp: + rejected = resp['rejected'] + defer.returnValue(rejected) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index d34ef3f6c..edddc3003 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -53,6 +53,7 @@ class PusherPool: "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, + "pushkey_ts": self.hs.get_clock().time_msec(), "data": data, "last_token": None, "last_success": None, @@ -75,6 +76,7 @@ class PusherPool: app_display_name=app_display_name, device_display_name=device_display_name, pushkey=pushkey, + pushkey_ts=self.hs.get_clock().time_msec(), data=json.dumps(data) ) self._refresh_pusher((app_id, pushkey)) @@ -88,6 +90,7 @@ class PusherPool: app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], + pushkey_ts=pusherdict['pushkey_ts'], data=pusherdict['data'], last_token=pusherdict['last_token'], last_success=pusherdict['last_success'], @@ -118,3 +121,12 @@ class PusherPool: self.pushers[fullid].stop() self.pushers[fullid] = p p.start() + + @defer.inlineCallbacks + def remove_pusher(self, app_id, pushkey): + fullid = "%s:%s" % (app_id, pushkey) + if fullid in self.pushers: + logger.info("Stopping pusher %s", fullid) + self.pushers[fullid].stop() + del self.pushers[fullid] + yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) \ No newline at end of file diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 9b5170a5f..bfc498025 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -30,7 +30,7 @@ class PusherStore(SQLBaseStore): def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( "SELECT id, user_name, kind, app_id," - "app_display_name, device_display_name, pushkey, data, " + "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers " "WHERE app_id = ? AND pushkey = ?" @@ -49,10 +49,11 @@ class PusherStore(SQLBaseStore): "app_display_name": r[4], "device_display_name": r[5], "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "pushkey_ts": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -63,7 +64,7 @@ class PusherStore(SQLBaseStore): def get_all_pushers(self): sql = ( "SELECT id, user_name, kind, app_id," - "app_display_name, device_display_name, pushkey, data, " + "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers" ) @@ -79,10 +80,11 @@ class PusherStore(SQLBaseStore): "app_display_name": r[4], "device_display_name": r[5], "pushkey": r[6], - "data": r[7], - "last_token": r[8], - "last_success": r[9], - "failing_since": r[10] + "pushkey_ts": r[7], + "data": r[8], + "last_token": r[9], + "last_success": r[10], + "failing_since": r[11] } for r in rows ] @@ -91,7 +93,8 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, - app_display_name, device_display_name, pushkey, data): + app_display_name, device_display_name, + pushkey, pushkey_ts, data): try: yield self._simple_upsert( PushersTable.table_name, @@ -104,12 +107,20 @@ class PusherStore(SQLBaseStore): kind=kind, app_display_name=app_display_name, device_display_name=device_display_name, + ts=pushkey_ts, data=data )) except Exception as e: logger.error("create_pusher with failed: %s", e) raise StoreError(500, "Problem creating pusher.") + @defer.inlineCallbacks + def delete_pusher_by_app_id_pushkey(self, app_id, pushkey): + yield self._simple_delete_one( + PushersTable.table_name, + dict(app_id=app_id, pushkey=pushkey) + ) + @defer.inlineCallbacks def update_pusher_last_token(self, user_name, pushkey, last_token): yield self._simple_update_one( @@ -147,6 +158,7 @@ class PushersTable(Table): "app_display_name", "device_display_name", "pushkey", + "pushkey_ts", "data", "last_token", "last_success", diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index 799e48d78..a991e4eb1 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -21,6 +21,7 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, + ts BIGINT NOT NULL, data blob, last_token TEXT, last_success BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 799e48d78..a991e4eb1 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -21,6 +21,7 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name varchar(64) NOT NULL, device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, + ts BIGINT NOT NULL, data blob, last_token TEXT, last_success BIGINT, From e3e2fc3255665ac888f3e0c0c6e2be39d5fda5f3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 15 Jan 2015 16:17:21 +0000 Subject: [PATCH 014/630] Don't make the pushers' event streams cause people to appear online --- synapse/handlers/events.py | 45 +++++++++++++++++++------------------- synapse/push/__init__.py | 4 +++- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index c9ade253d..54ab27004 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -47,11 +47,11 @@ class EventStreamHandler(BaseHandler): @defer.inlineCallbacks @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, - as_client_event=True): + as_client_event=True, affect_presence=True): auth_user = self.hs.parse_userid(auth_user_id) try: - if auth_user not in self._streams_per_user: + if affect_presence and auth_user not in self._streams_per_user: self._streams_per_user[auth_user] = 0 if auth_user in self._stop_timer_per_user: try: @@ -64,7 +64,7 @@ class EventStreamHandler(BaseHandler): yield self.distributor.fire( "started_user_eventstream", auth_user ) - self._streams_per_user[auth_user] += 1 + self._streams_per_user[auth_user] += 1 if pagin_config.from_token is None: pagin_config.from_token = None @@ -92,28 +92,29 @@ class EventStreamHandler(BaseHandler): defer.returnValue(chunk) finally: - self._streams_per_user[auth_user] -= 1 - if not self._streams_per_user[auth_user]: - del self._streams_per_user[auth_user] + if affect_presence: + self._streams_per_user[auth_user] -= 1 + if not self._streams_per_user[auth_user]: + del self._streams_per_user[auth_user] - # 10 seconds of grace to allow the client to reconnect again - # before we think they're gone - def _later(): - logger.debug( - "_later stopped_user_eventstream %s", auth_user + # 10 seconds of grace to allow the client to reconnect again + # before we think they're gone + def _later(): + logger.debug( + "_later stopped_user_eventstream %s", auth_user + ) + + self._stop_timer_per_user.pop(auth_user, None) + + yield self.distributor.fire( + "stopped_user_eventstream", auth_user + ) + + logger.debug("Scheduling _later: for %s", auth_user) + self._stop_timer_per_user[auth_user] = ( + self.clock.call_later(30, _later) ) - self._stop_timer_per_user.pop(auth_user, None) - - yield self.distributor.fire( - "stopped_user_eventstream", auth_user - ) - - logger.debug("Scheduling _later: for %s", auth_user) - self._stop_timer_per_user[auth_user] = ( - self.clock.call_later(30, _later) - ) - class EventHandler(BaseHandler): diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 839f66639..9cf996fb8 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -81,7 +81,9 @@ class Pusher(object): from_tok = StreamToken.from_string(self.last_token) config = PaginationConfig(from_token=from_tok, limit='1') chunk = yield self.evStreamHandler.get_stream( - self.user_name, config, timeout=100*365*24*60*60*1000) + self.user_name, config, + timeout=100*365*24*60*60*1000, affect_presence=False + ) # limiting to 1 may get 1 event plus 1 presence event, so # pick out the actual event From 2ca2dbc82183f7dbe8c01694bf1c32a8c4c4b9de Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 15 Jan 2015 16:56:18 +0000 Subject: [PATCH 015/630] Send room name and first alias in notification poke. --- synapse/push/__init__.py | 13 +++++++++++++ synapse/push/httppusher.py | 16 +++++++++++++--- synapse/storage/__init__.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 9cf996fb8..5f4e833ad 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -61,6 +61,19 @@ class Pusher(object): return False return True + @defer.inlineCallbacks + def get_context_for_event(self, ev): + name_aliases = yield self.store.get_room_name_and_aliases( + ev['room_id'] + ) + + ctx = {'aliases': name_aliases[1]} + if name_aliases[0] is not None: + ctx['name'] = name_aliases[0] + + defer.returnValue(ctx) + + @defer.inlineCallbacks def start(self): if not self.last_token: diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index bcfa06e2a..7631a741f 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -50,6 +50,7 @@ class HttpPusher(Pusher): self.data_minus_url.update(self.data) del self.data_minus_url['url'] + @defer.inlineCallbacks def _build_notification_dict(self, event): # we probably do not want to push for every presence update # (we may want to be able to set up notifications when specific @@ -57,9 +58,11 @@ class HttpPusher(Pusher): # Actually, presence events will not get this far now because we # need to filter them out in the main Pusher code. if 'event_id' not in event: - return None + defer.returnValue(None) - return { + ctx = yield self.get_context_for_event(event) + + d = { 'notification': { 'transition': 'new', # everything is new for now: we don't have read receipts @@ -85,9 +88,16 @@ class HttpPusher(Pusher): } } + if len(ctx['aliases']): + d['notification']['roomAlias'] = ctx['aliases'][0] + if 'name' in ctx: + d['notification']['roomName'] = ctx['name'] + + defer.returnValue(d) + @defer.inlineCallbacks def dispatch_push(self, event): - notification_dict = self._build_notification_dict(event) + notification_dict = yield self._build_notification_dict(event) if not notification_dict: defer.returnValue([]) try: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index fa7ad0eea..191fe462a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -384,6 +384,41 @@ class DataStore(RoomMemberStore, RoomStore, events = yield self._parse_events(results) defer.returnValue(events) + @defer.inlineCallbacks + def get_room_name_and_aliases(self, room_id): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + sql += " AND (s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases'" + args = (room_id,) + + results = yield self._execute_and_decode(sql, *args) + + events = yield self._parse_events(results) + + name = None + aliases = [] + + for e in events: + if e.type == 'm.room.name': + name = e.content['name'] + elif e.type == 'm.room.aliases': + aliases.extend(e.content['aliases']) + + defer.returnValue((name, aliases)) + @defer.inlineCallbacks def _get_min_token(self): row = yield self._execute( From 2d2953cf5fce26625e56fc1abc230735d007ea1e Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 16 Jan 2015 11:24:10 +0000 Subject: [PATCH 016/630] Require device language when adding a pusher. Because this seems like it might be useful to do sooner rather than later. --- synapse/push/pusherpool.py | 8 +++++--- synapse/rest/pusher.py | 3 ++- synapse/storage/pusher.py | 3 ++- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index edddc3003..8c77f4b66 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -41,7 +41,7 @@ class PusherPool: @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, - app_display_name, device_display_name, pushkey, data): + app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, # recreated, added and started: this means we have only one @@ -54,6 +54,7 @@ class PusherPool: "device_display_name": device_display_name, "pushkey": pushkey, "pushkey_ts": self.hs.get_clock().time_msec(), + "lang": lang, "data": data, "last_token": None, "last_success": None, @@ -62,13 +63,13 @@ class PusherPool: yield self._add_pusher_to_store( user_name, kind, app_id, app_display_name, device_display_name, - pushkey, data + pushkey, lang, data ) @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, kind, app_id, app_display_name, device_display_name, - pushkey, data): + pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, kind=kind, @@ -77,6 +78,7 @@ class PusherPool: device_display_name=device_display_name, pushkey=pushkey, pushkey_ts=self.hs.get_clock().time_msec(), + lang=lang, data=json.dumps(data) ) self._refresh_pusher((app_id, pushkey)) diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 5b371318d..6b9a59adb 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -32,7 +32,7 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) reqd = ['kind', 'app_id', 'app_display_name', - 'device_display_name', 'pushkey', 'data'] + 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] for i in reqd: if i not in content: @@ -50,6 +50,7 @@ class PusherRestServlet(RestServlet): app_display_name=content['app_display_name'], device_display_name=content['device_display_name'], pushkey=content['pushkey'], + lang=content['lang'], data=content['data'] ) except PusherConfigException as pce: diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index bfc498025..4eb30c7bd 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -94,7 +94,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def add_pusher(self, user_name, kind, app_id, app_display_name, device_display_name, - pushkey, pushkey_ts, data): + pushkey, pushkey_ts, lang, data): try: yield self._simple_upsert( PushersTable.table_name, @@ -108,6 +108,7 @@ class PusherStore(SQLBaseStore): app_display_name=app_display_name, device_display_name=device_display_name, ts=pushkey_ts, + lang=lang, data=data )) except Exception as e: diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index a991e4eb1..689d2dff8 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS pushers ( device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, ts BIGINT NOT NULL, + lang varchar(8), data blob, last_token TEXT, last_success BIGINT, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index a991e4eb1..689d2dff8 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS pushers ( device_display_name varchar(128) NOT NULL, pushkey blob NOT NULL, ts BIGINT NOT NULL, + lang varchar(8), data blob, last_token TEXT, last_success BIGINT, From afb714f7bebf88ac27eac018cffa2078e2723310 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 20 Jan 2015 11:49:48 +0000 Subject: [PATCH 017/630] add instance_handles to pushers so we have a way to refer to them even if the push token changes. --- synapse/push/__init__.py | 3 +- synapse/push/httppusher.py | 3 +- synapse/push/pusherpool.py | 9 ++++-- synapse/rest/pusher.py | 3 +- synapse/storage/pusher.py | 46 +++++++++++++++------------- synapse/storage/schema/delta/v10.sql | 1 + synapse/storage/schema/pusher.sql | 1 + 7 files changed, 39 insertions(+), 27 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 5f4e833ad..3ee652f3b 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -30,13 +30,14 @@ class Pusher(object): MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 - def __init__(self, _hs, user_name, app_id, + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() + self.instance_handle = instance_handle, self.user_name = user_name self.app_id = app_id self.app_display_name = app_display_name diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 7631a741f..9a3e0be15 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -24,11 +24,12 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, user_name, app_id, + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, + instance_handle, user_name, app_id, app_display_name, diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 8c77f4b66..2dfecf178 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -40,7 +40,7 @@ class PusherPool: self._start_pushers(pushers) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, + def add_pusher(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -49,6 +49,7 @@ class PusherPool: self._create_pusher({ "user_name": user_name, "kind": kind, + "instance_handle": instance_handle, "app_id": app_id, "app_display_name": app_display_name, "device_display_name": device_display_name, @@ -61,17 +62,18 @@ class PusherPool: "failing_since": None }) yield self._add_pusher_to_store( - user_name, kind, app_id, + user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, kind, app_id, + def _add_pusher_to_store(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, + instance_handle=instance_handle, kind=kind, app_id=app_id, app_display_name=app_display_name, @@ -87,6 +89,7 @@ class PusherPool: if pusherdict['kind'] == 'http': return HttpPusher( self.hs, + instance_handle=pusherdict['instance_handle'], user_name=pusherdict['user_name'], app_id=pusherdict['app_id'], app_display_name=pusherdict['app_display_name'], diff --git a/synapse/rest/pusher.py b/synapse/rest/pusher.py index 6b9a59adb..4659c9b1d 100644 --- a/synapse/rest/pusher.py +++ b/synapse/rest/pusher.py @@ -31,7 +31,7 @@ class PusherRestServlet(RestServlet): content = _parse_json(request) - reqd = ['kind', 'app_id', 'app_display_name', + reqd = ['instance_handle', 'kind', 'app_id', 'app_display_name', 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] for i in reqd: @@ -45,6 +45,7 @@ class PusherRestServlet(RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), + instance_handle=content['instance_handle'], kind=content['kind'], app_id=content['app_id'], app_display_name=content['app_display_name'], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 4eb30c7bd..113cdc8a8 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,7 +29,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( - "SELECT id, user_name, kind, app_id," + "SELECT id, user_name, kind, instance_handle, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers " @@ -45,15 +45,16 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app_id": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "pushkey_ts": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "instance_handle": r[3], + "app_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "pushkey_ts": r[8], + "data": r[9], + "last_token": r[10], + "last_success": r[11], + "failing_since": r[12] } for r in rows ] @@ -63,7 +64,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers(self): sql = ( - "SELECT id, user_name, kind, app_id," + "SELECT id, user_name, kind, instance_handle, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers" @@ -76,15 +77,16 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "app_id": r[3], - "app_display_name": r[4], - "device_display_name": r[5], - "pushkey": r[6], - "pushkey_ts": r[7], - "data": r[8], - "last_token": r[9], - "last_success": r[10], - "failing_since": r[11] + "instance_handle": r[3], + "app_id": r[4], + "app_display_name": r[5], + "device_display_name": r[6], + "pushkey": r[7], + "pushkey_ts": r[8], + "data": r[9], + "last_token": r[10], + "last_success": r[11], + "failing_since": r[12] } for r in rows ] @@ -92,7 +94,7 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, kind, app_id, + def add_pusher(self, user_name, instance_handle, kind, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: @@ -105,6 +107,7 @@ class PusherStore(SQLBaseStore): dict( user_name=user_name, kind=kind, + instance_handle=instance_handle, app_display_name=app_display_name, device_display_name=device_display_name, ts=pushkey_ts, @@ -155,6 +158,7 @@ class PushersTable(Table): "id", "user_name", "kind", + "instance_handle", "app_id", "app_display_name", "device_display_name", diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index 689d2dff8..b84ce20ef 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, + instance_handle varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 689d2dff8..b84ce20ef 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, + instance_handle varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, From 5d5932d493dc769e95472835b438769567ed550e Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 20 Jan 2015 11:52:08 +0000 Subject: [PATCH 018/630] use underscores everywhere, not camelcase. --- synapse/push/httppusher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 9a3e0be15..46433ad4a 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -72,17 +72,17 @@ class HttpPusher(Pusher): 'from': event['user_id'], # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? - #'fromDisplayName': 'Steve Stevington' + #'from_display_name': 'Steve Stevington' #'counts': { -- we don't mark messages as read yet so # we have no way of knowing # 'unread': 1, - # 'missedCalls': 2 + # 'missed_calls': 2 # }, 'devices': [ { 'app_id': self.app_id, 'pushkey': self.pushkey, - 'pushkeyTs': long(self.pushkey_ts / 1000), + 'pushkey_ts': long(self.pushkey_ts / 1000), 'data': self.data_minus_url } ] From 6dcade97be7f1331063fd12ac85e61c6f2cf7dac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jan 2015 16:27:04 +0000 Subject: [PATCH 019/630] Implement new state resolution algorithm --- synapse/state.py | 101 +++++++---- tests/test_state.py | 434 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 430 insertions(+), 105 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 8144fa02b..7d58a76ed 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.api.constants import EventTypes +from synapse.api.errors import AuthError from synapse.events.snapshot import EventContext from collections import namedtuple @@ -42,6 +43,8 @@ class StateHandler(object): def __init__(self, hs): self.store = hs.get_datastore() + # self.auth = hs.get_auth() + self.hs = hs @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): @@ -210,15 +213,22 @@ class StateHandler(object): else: prev_states = [] + auth_events = { + k: e for k, e in unconflicted_state.items() + if k[0] in (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + } + try: - new_state = {} - new_state.update(unconflicted_state) - for key, events in conflicted_state.items(): - new_state[key] = self._resolve_state_events(events) + resolved_state = self._resolve_state_events( + conflicted_state, auth_events + ) except: logger.exception("Failed to resolve state") raise + new_state = unconflicted_state + new_state.update(resolved_state) + defer.returnValue((None, new_state, prev_states)) def _get_power_level_from_event_state(self, event, user_id): @@ -238,36 +248,65 @@ class StateHandler(object): return 0 @log_function - def _resolve_state_events(self, events): - curr_events = events + def _resolve_state_events(self, conflicted_state, auth_events): + resolved_state = {} + power_key = (EventTypes.PowerLevels, "") + if power_key in conflicted_state.items(): + power_levels = conflicted_state[power_key] + resolved_state[power_key] = self._resolve_auth_events(power_levels) - new_powers = [ - self._get_power_level_from_event_state(e, e.user_id) - for e in curr_events - ] + auth_events.update(resolved_state) - new_powers = [ - int(p) if p else 0 for p in new_powers - ] + for key, events in conflicted_state.items(): + if key[0] == EventTypes.Member: + resolved_state[key] = self._resolve_auth_events( + events, + auth_events + ) - max_power = max(new_powers) + auth_events.update(resolved_state) - curr_events = [ - z[0] for z in zip(curr_events, new_powers) - if z[1] == max_power - ] + for key, events in conflicted_state.items(): + if key not in resolved_state: + resolved_state[key] = self._resolve_normal_events( + events, auth_events + ) - if not curr_events: - raise RuntimeError("Max didn't get a max?") - elif len(curr_events) == 1: - return curr_events[0] + return resolved_state - # TODO: For now, just choose the one with the largest event_id. - return ( - sorted( - curr_events, - key=lambda e: hashlib.sha1( - e.event_id + e.user_id + e.room_id + e.type - ).hexdigest() - )[0] - ) + def _resolve_auth_events(self, events, auth_events): + reverse = [i for i in reversed(self._ordered_events(events))] + + auth_events = dict(auth_events) + + prev_event = reverse[0] + for event in reverse[1:]: + auth_events[(prev_event.type, prev_event.state_key)] = prev_event + try: + # FIXME: hs.get_auth() is bad style, but we need to do it to + # get around circular deps. + self.hs.get_auth().check(event, auth_events) + prev_event = event + except AuthError: + return prev_event + + return event + + def _resolve_normal_events(self, events, auth_events): + for event in self._ordered_events(events): + try: + # FIXME: hs.get_auth() is bad style, but we need to do it to + # get around circular deps. + self.hs.get_auth().check(event, auth_events) + return event + except AuthError as e: + pass + + # Oh dear. + return event + + def _ordered_events(self, events): + def key_func(e): + return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() + + return sorted(events, key=key_func) \ No newline at end of file diff --git a/tests/test_state.py b/tests/test_state.py index 98ad9e54c..019e794aa 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -16,11 +16,120 @@ from tests import unittest from twisted.internet import defer +from synapse.events import FrozenEvent +from synapse.api.auth import Auth +from synapse.api.constants import EventTypes, Membership from synapse.state import StateHandler from mock import Mock +_next_event_id = 1000 + + +def create_event(name=None, type=None, state_key=None, depth=2, event_id=None, + prev_events=[], **kwargs): + global _next_event_id + + if not event_id: + _next_event_id += 1 + event_id = str(_next_event_id) + + if not name: + if state_key is not None: + name = "<%s-%s, %s>" % (type, state_key, event_id,) + else: + name = "<%s, %s>" % (type, event_id,) + + d = { + "event_id": event_id, + "type": type, + "sender": "@user_id:example.com", + "room_id": "!room_id:example.com", + "depth": depth, + "prev_events": prev_events, + } + + if state_key is not None: + d["state_key"] = state_key + + d.update(kwargs) + + event = FrozenEvent(d) + + return event + + +class StateGroupStore(object): + def __init__(self): + self._event_to_state_group = {} + self._group_to_state = {} + + self._next_group = 1 + + def get_state_groups(self, event_ids): + groups = {} + for event_id in event_ids: + group = self._event_to_state_group.get(event_id) + if group: + groups[group] = self._group_to_state[group] + + return defer.succeed(groups) + + def store_state_groups(self, event, context): + if context.current_state is None: + return + + state_events = context.current_state + + if event.is_state(): + state_events[(event.type, event.state_key)] = event + + state_group = context.state_group + if not state_group: + state_group = self._next_group + self._next_group += 1 + + self._group_to_state[state_group] = state_events.values() + + self._event_to_state_group[event.event_id] = state_group + + +class DictObj(dict): + def __init__(self, **kwargs): + super(DictObj, self).__init__(kwargs) + self.__dict__ = self + + +class Graph(object): + def __init__(self, nodes, edges): + events = {} + clobbered = set(events.keys()) + + for event_id, fields in nodes.items(): + refs = edges.get(event_id) + if refs: + clobbered.difference_update(refs) + prev_events = [(r, {}) for r in refs] + else: + prev_events = [] + + events[event_id] = create_event( + event_id=event_id, + prev_events=prev_events, + **fields + ) + + self._leaves = clobbered + self._events = sorted(events.values(), key=lambda e: e.depth) + + def walk(self): + return iter(self._events) + + def get_leaves(self): + return (self._events[i] for i in self._leaves) + + class StateTestCase(unittest.TestCase): def setUp(self): self.store = Mock( @@ -29,20 +138,188 @@ class StateTestCase(unittest.TestCase): "add_event_hashes", ] ) - hs = Mock(spec=["get_datastore"]) + hs = Mock(spec=["get_datastore", "get_auth", "get_state_handler"]) hs.get_datastore.return_value = self.store + hs.get_state_handler.return_value = None + hs.get_auth.return_value = Auth(hs) self.state = StateHandler(hs) self.event_id = 0 + @defer.inlineCallbacks + def test_branch_no_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="", + depth=1, + ), + "A": DictObj( + type=EventTypes.Message, + depth=2, + ), + "B": DictObj( + type=EventTypes.Message, + depth=3, + ), + "C": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "D": DictObj( + type=EventTypes.Message, + depth=4, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["A"], + "D": ["B", "C"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertEqual(2, len(context_store["D"].current_state)) + + @defer.inlineCallbacks + def test_branch_basic_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="creator", + content={"membership": "@user_id:example.com"}, + depth=1, + ), + "A": DictObj( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={"membership": Membership.JOIN}, + membership=Membership.JOIN, + depth=2, + ), + "B": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "C": DictObj( + type=EventTypes.Name, + state_key="", + depth=4, + ), + "D": DictObj( + type=EventTypes.Message, + depth=5, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["A"], + "D": ["B", "C"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertSetEqual( + {"START", "A", "C"}, + {e.event_id for e in context_store["D"].current_state.values()} + ) + + @defer.inlineCallbacks + def test_branch_have_banned_conflict(self): + graph = Graph( + nodes={ + "START": DictObj( + type=EventTypes.Create, + state_key="creator", + content={"membership": "@user_id:example.com"}, + depth=1, + ), + "A": DictObj( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={"membership": Membership.JOIN}, + membership=Membership.JOIN, + depth=2, + ), + "B": DictObj( + type=EventTypes.Name, + state_key="", + depth=3, + ), + "C": DictObj( + type=EventTypes.Member, + state_key="@user_id_2:example.com", + content={"membership": Membership.BAN}, + membership=Membership.BAN, + depth=4, + ), + "D": DictObj( + type=EventTypes.Name, + state_key="", + depth=4, + sender="@user_id_2:example.com", + ), + "E": DictObj( + type=EventTypes.Message, + depth=5, + ), + }, + edges={ + "A": ["START"], + "B": ["A"], + "C": ["B"], + "D": ["B"], + "E": ["C", "D"] + } + ) + + store = StateGroupStore() + self.store.get_state_groups.side_effect = store.get_state_groups + + context_store = {} + + for event in graph.walk(): + context = yield self.state.compute_event_context(event) + store.store_state_groups(event, context) + context_store[event.event_id] = context + + self.assertSetEqual( + {"START", "A", "B", "C"}, + {e.event_id for e in context_store["E"].current_state.values()} + ) + @defer.inlineCallbacks def test_annotate_with_old_message(self): - event = self.create_event(type="test_message", name="event") + event = create_event(type="test_message", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] context = yield self.state.compute_event_context( @@ -62,12 +339,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_annotate_with_old_state(self): - event = self.create_event(type="state", state_key="", name="event") + event = create_event(type="state", state_key="", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] context = yield self.state.compute_event_context( @@ -88,13 +365,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_trivial_annotate_message(self): - event = self.create_event(type="test_message", name="event") - event.prev_events = [] + event = create_event(type="test_message", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] group_name = "group_name_1" @@ -119,13 +395,12 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_trivial_annotate_state(self): - event = self.create_event(type="state", state_key="", name="event") - event.prev_events = [] + event = create_event(type="state", state_key="", name="event") old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] group_name = "group_name_1" @@ -150,30 +425,21 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_resolve_message_conflict(self): - event = self.create_event(type="test_message", name="event") - event.prev_events = [] + event = create_event(type="test_message", name="event") old_state_1 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] old_state_2 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test3", state_key="2"), - self.create_event(type="test4", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test3", state_key="2"), + create_event(type="test4", state_key=""), ] - group_name_1 = "group_name_1" - group_name_2 = "group_name_2" - - self.store.get_state_groups.return_value = { - group_name_1: old_state_1, - group_name_2: old_state_2, - } - - context = yield self.state.compute_event_context(event) + context = yield self._get_context(event, old_state_1, old_state_2) self.assertEqual(len(context.current_state), 5) @@ -181,21 +447,70 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_resolve_state_conflict(self): - event = self.create_event(type="test4", state_key="", name="event") - event.prev_events = [] + event = create_event(type="test4", state_key="", name="event") old_state_1 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test1", state_key="2"), + create_event(type="test2", state_key=""), ] old_state_2 = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test3", state_key="2"), - self.create_event(type="test4", state_key=""), + create_event(type="test1", state_key="1"), + create_event(type="test3", state_key="2"), + create_event(type="test4", state_key=""), ] + context = yield self._get_context(event, old_state_1, old_state_2) + + self.assertEqual(len(context.current_state), 5) + + self.assertIsNone(context.state_group) + + @defer.inlineCallbacks + def test_standard_depth_conflict(self): + event = create_event(type="test4", name="event") + + member_event = create_event( + type=EventTypes.Member, + state_key="@user_id:example.com", + content={ + "membership": Membership.JOIN, + } + ) + + old_state_1 = [ + member_event, + create_event(type="test1", state_key="1", depth=1), + ] + + old_state_2 = [ + member_event, + create_event(type="test1", state_key="1", depth=2), + ] + + context = yield self._get_context(event, old_state_1, old_state_2) + + self.assertEqual(old_state_2[1], context.current_state[("test1", "1")]) + + # Reverse the depth to make sure we are actually using the depths + # during state resolution. + + old_state_1 = [ + member_event, + create_event(type="test1", state_key="1", depth=2), + ] + + old_state_2 = [ + member_event, + create_event(type="test1", state_key="1", depth=1), + ] + + context = yield self._get_context(event, old_state_1, old_state_2) + + self.assertEqual(old_state_1[1], context.current_state[("test1", "1")]) + + def _get_context(self, event, old_state_1, old_state_2): group_name_1 = "group_name_1" group_name_2 = "group_name_2" @@ -204,33 +519,4 @@ class StateTestCase(unittest.TestCase): group_name_2: old_state_2, } - context = yield self.state.compute_event_context(event) - - self.assertEqual(len(context.current_state), 5) - - self.assertIsNone(context.state_group) - - def create_event(self, name=None, type=None, state_key=None): - self.event_id += 1 - event_id = str(self.event_id) - - if not name: - if state_key is not None: - name = "<%s-%s>" % (type, state_key) - else: - name = "<%s>" % (type, ) - - event = Mock(name=name, spec=[]) - event.type = type - - if state_key is not None: - event.state_key = state_key - event.event_id = event_id - - event.is_state = lambda: (state_key is not None) - event.unsigned = {} - - event.user_id = "@user_id:example.com" - event.room_id = "!room_id:example.com" - - return event + return self.state.compute_event_context(event) From b390bf39f280c64497da089c647402bc3287f4b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jan 2015 16:44:04 +0000 Subject: [PATCH 020/630] Remove unused function. Add comment. --- synapse/state.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 7d58a76ed..5b622ad3b 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -231,24 +231,20 @@ class StateHandler(object): defer.returnValue((None, new_state, prev_states)) - def _get_power_level_from_event_state(self, event, user_id): - if hasattr(event, "old_state_events") and event.old_state_events: - key = (EventTypes.PowerLevels, "", ) - power_level_event = event.old_state_events.get(key) - level = None - if power_level_event: - level = power_level_event.content.get("users", {}).get( - user_id - ) - if not level: - level = power_level_event.content.get("users_default", 0) - - return level - else: - return 0 - @log_function def _resolve_state_events(self, conflicted_state, auth_events): + """ This is where we actually decide which of the conflicted state to + use. + + We resolve conflicts in the following order: + 1. power levels + 2. memberships + 3. other events. + + :param conflicted_state: + :param auth_events: + :return: + """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") if power_key in conflicted_state.items(): From 58691680b8dd93fab56fed914b0204f2b6751b6a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Jan 2015 14:20:53 +0000 Subject: [PATCH 021/630] update .gitignore, set media-store-path in demo --- .gitignore | 11 +++-------- demo/start.sh | 3 ++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index af90668c8..2ed22b1cd 100644 --- a/.gitignore +++ b/.gitignore @@ -26,17 +26,12 @@ htmlcov demo/*.db demo/*.log +demo/*.log.* demo/*.pid +demo/media_store.* demo/etc -graph/*.svg -graph/*.png -graph/*.dot - -**/webclient/config.js -**/webclient/test/coverage/ -**/webclient/test/environment-protractor.js - uploads .idea/ +media_store/ diff --git a/demo/start.sh b/demo/start.sh index ce3e29248..bb2248770 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -32,7 +32,8 @@ for port in 8080 8081 8082; do -D --pid-file "$DIR/$port.pid" \ --manhole $((port + 1000)) \ --tls-dh-params-path "demo/demo.tls.dh" \ - $PARAMS $SYNAPSE_PARAMS + --media-store-path "demo/media_store.$port" \ + $PARAMS $SYNAPSE_PARAMS \ python -m synapse.app.homeserver \ --config-path "demo/etc/$port.config" \ From d00cca11b961f59c590956691b750eb082160edc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Jan 2015 14:28:07 +0000 Subject: [PATCH 022/630] Add demo and scripts to python manifest --- MANIFEST.in | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index a1a77ff54..7979ec080 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,13 @@ -recursive-include docs * -recursive-include tests *.py +include synctl +include LICENSE +include VERSION +include *.rst + recursive-include synapse/storage/schema *.sql -recursive-include syweb/webclient * + +recursive-include demo *.dh +recursive-include demo *.py +recursive-include demo *.sh +recursive-include docs * +recursive-include scripts * +recursive-include tests *.py From 16bfabb9c51e4721851a5d3220d5b78cff4d7892 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Jan 2015 14:32:51 +0000 Subject: [PATCH 023/630] Fix manifest. Ignore contrib and docs directories when checking manifest against source control. --- MANIFEST.in | 1 + setup.cfg | 8 ++++++++ tests/storage/TESTS_NEEDED_FOR | 5 ----- 3 files changed, 9 insertions(+), 5 deletions(-) delete mode 100644 tests/storage/TESTS_NEEDED_FOR diff --git a/MANIFEST.in b/MANIFEST.in index 7979ec080..8243a942e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include synctl include LICENSE include VERSION include *.rst +include demo/README recursive-include synapse/storage/schema *.sql diff --git a/setup.cfg b/setup.cfg index 2830831f0..888ad6ed4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,3 +8,11 @@ test = trial [trial] test_suite = tests + +[check-manifest] +ignore = + contrib + contrib/* + docs/* + pylint.cfg + tox.ini diff --git a/tests/storage/TESTS_NEEDED_FOR b/tests/storage/TESTS_NEEDED_FOR deleted file mode 100644 index 8e5d0cbdc..000000000 --- a/tests/storage/TESTS_NEEDED_FOR +++ /dev/null @@ -1,5 +0,0 @@ -synapse/storage/feedback.py -synapse/storage/keys.py -synapse/storage/pdu.py -synapse/storage/stream.py -synapse/storage/transactions.py From 1d2016b4a881538aa86f4824f1131dfada186ae0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Jan 2015 14:59:08 +0000 Subject: [PATCH 024/630] Move client v1 api rest servlets into a "client/v1" directory --- synapse/client/__init__.py | 14 ++++++++++++++ synapse/{rest => client/v1}/__init__.py | 0 synapse/{rest => client/v1}/admin.py | 0 synapse/{rest => client/v1}/base.py | 2 +- synapse/{rest => client/v1}/directory.py | 0 synapse/{rest => client/v1}/events.py | 2 +- synapse/{rest => client/v1}/initial_sync.py | 0 synapse/{rest => client/v1}/login.py | 0 synapse/{rest => client/v1}/presence.py | 0 synapse/{rest => client/v1}/profile.py | 0 synapse/{rest => client/v1}/register.py | 0 synapse/{rest => client/v1}/room.py | 0 synapse/{rest => client/v1}/transactions.py | 0 synapse/{rest => client/v1}/voip.py | 0 synapse/server.py | 2 +- tests/client/__init__.py | 14 ++++++++++++++ tests/{rest => client/v1}/__init__.py | 0 tests/{rest => client/v1}/test_events.py | 14 +++++++------- tests/{rest => client/v1}/test_presence.py | 2 +- tests/{rest => client/v1}/test_profile.py | 2 +- tests/{rest => client/v1}/test_rooms.py | 18 +++++++++--------- tests/{rest => client/v1}/test_typing.py | 6 +++--- tests/{rest => client/v1}/utils.py | 0 23 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 synapse/client/__init__.py rename synapse/{rest => client/v1}/__init__.py (100%) rename synapse/{rest => client/v1}/admin.py (100%) rename synapse/{rest => client/v1}/base.py (97%) rename synapse/{rest => client/v1}/directory.py (100%) rename synapse/{rest => client/v1}/events.py (97%) rename synapse/{rest => client/v1}/initial_sync.py (100%) rename synapse/{rest => client/v1}/login.py (100%) rename synapse/{rest => client/v1}/presence.py (100%) rename synapse/{rest => client/v1}/profile.py (100%) rename synapse/{rest => client/v1}/register.py (100%) rename synapse/{rest => client/v1}/room.py (100%) rename synapse/{rest => client/v1}/transactions.py (100%) rename synapse/{rest => client/v1}/voip.py (100%) create mode 100644 tests/client/__init__.py rename tests/{rest => client/v1}/__init__.py (100%) rename tests/{rest => client/v1}/test_events.py (94%) rename tests/{rest => client/v1}/test_presence.py (99%) rename tests/{rest => client/v1}/test_profile.py (99%) rename tests/{rest => client/v1}/test_rooms.py (98%) rename tests/{rest => client/v1}/test_typing.py (96%) rename tests/{rest => client/v1}/utils.py (100%) diff --git a/synapse/client/__init__.py b/synapse/client/__init__.py new file mode 100644 index 000000000..1a84d94cd --- /dev/null +++ b/synapse/client/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/rest/__init__.py b/synapse/client/v1/__init__.py similarity index 100% rename from synapse/rest/__init__.py rename to synapse/client/v1/__init__.py diff --git a/synapse/rest/admin.py b/synapse/client/v1/admin.py similarity index 100% rename from synapse/rest/admin.py rename to synapse/client/v1/admin.py diff --git a/synapse/rest/base.py b/synapse/client/v1/base.py similarity index 97% rename from synapse/rest/base.py rename to synapse/client/v1/base.py index c58394552..d005206b7 100644 --- a/synapse/rest/base.py +++ b/synapse/client/v1/base.py @@ -15,7 +15,7 @@ """ This module contains base REST classes for constructing REST servlets. """ from synapse.api.urls import CLIENT_PREFIX -from synapse.rest.transactions import HttpTransactionStore +from .transactions import HttpTransactionStore import re import logging diff --git a/synapse/rest/directory.py b/synapse/client/v1/directory.py similarity index 100% rename from synapse/rest/directory.py rename to synapse/client/v1/directory.py diff --git a/synapse/rest/events.py b/synapse/client/v1/events.py similarity index 97% rename from synapse/rest/events.py rename to synapse/client/v1/events.py index bedcb2bcc..c2515528a 100644 --- a/synapse/rest/events.py +++ b/synapse/client/v1/events.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig -from synapse.rest.base import RestServlet, client_path_pattern +from .base import RestServlet, client_path_pattern import logging diff --git a/synapse/rest/initial_sync.py b/synapse/client/v1/initial_sync.py similarity index 100% rename from synapse/rest/initial_sync.py rename to synapse/client/v1/initial_sync.py diff --git a/synapse/rest/login.py b/synapse/client/v1/login.py similarity index 100% rename from synapse/rest/login.py rename to synapse/client/v1/login.py diff --git a/synapse/rest/presence.py b/synapse/client/v1/presence.py similarity index 100% rename from synapse/rest/presence.py rename to synapse/client/v1/presence.py diff --git a/synapse/rest/profile.py b/synapse/client/v1/profile.py similarity index 100% rename from synapse/rest/profile.py rename to synapse/client/v1/profile.py diff --git a/synapse/rest/register.py b/synapse/client/v1/register.py similarity index 100% rename from synapse/rest/register.py rename to synapse/client/v1/register.py diff --git a/synapse/rest/room.py b/synapse/client/v1/room.py similarity index 100% rename from synapse/rest/room.py rename to synapse/client/v1/room.py diff --git a/synapse/rest/transactions.py b/synapse/client/v1/transactions.py similarity index 100% rename from synapse/rest/transactions.py rename to synapse/client/v1/transactions.py diff --git a/synapse/rest/voip.py b/synapse/client/v1/voip.py similarity index 100% rename from synapse/rest/voip.py rename to synapse/client/v1/voip.py diff --git a/synapse/server.py b/synapse/server.py index d861efd2f..57a95bf75 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -24,7 +24,7 @@ from synapse.events.utils import serialize_event from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers -from synapse.rest import RestServletFactory +from synapse.client.v1 import RestServletFactory from synapse.state import StateHandler from synapse.storage import DataStore from synapse.types import UserID, RoomAlias, RoomID, EventID diff --git a/tests/client/__init__.py b/tests/client/__init__.py new file mode 100644 index 000000000..1a84d94cd --- /dev/null +++ b/tests/client/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/rest/__init__.py b/tests/client/v1/__init__.py similarity index 100% rename from tests/rest/__init__.py rename to tests/client/v1/__init__.py diff --git a/tests/rest/test_events.py b/tests/client/v1/test_events.py similarity index 94% rename from tests/rest/test_events.py rename to tests/client/v1/test_events.py index d3159e2cf..9b36dd322 100644 --- a/tests/rest/test_events.py +++ b/tests/client/v1/test_events.py @@ -19,13 +19,13 @@ from tests import unittest # twisted imports from twisted.internet import defer -import synapse.rest.events -import synapse.rest.register -import synapse.rest.room +import synapse.client.v1.events +import synapse.client.v1.register +import synapse.client.v1.room from synapse.server import HomeServer -from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from ...utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -144,9 +144,9 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_clock().time_msec.return_value = 1000000 hs.get_clock().time.return_value = 1000 - synapse.rest.register.register_servlets(hs, self.mock_resource) - synapse.rest.events.register_servlets(hs, self.mock_resource) - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.register.register_servlets(hs, self.mock_resource) + synapse.client.v1.events.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) # register an account self.user_id = "sid1" diff --git a/tests/rest/test_presence.py b/tests/client/v1/test_presence.py similarity index 99% rename from tests/rest/test_presence.py rename to tests/client/v1/test_presence.py index 769c7824b..e7d636c74 100644 --- a/tests/rest/test_presence.py +++ b/tests/client/v1/test_presence.py @@ -20,7 +20,7 @@ from twisted.internet import defer from mock import Mock -from ..utils import MockHttpResource, MockKey +from ...utils import MockHttpResource, MockKey from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler diff --git a/tests/rest/test_profile.py b/tests/client/v1/test_profile.py similarity index 99% rename from tests/rest/test_profile.py rename to tests/client/v1/test_profile.py index 3a0d1e700..1182cc54e 100644 --- a/tests/rest/test_profile.py +++ b/tests/client/v1/test_profile.py @@ -20,7 +20,7 @@ from twisted.internet import defer from mock import Mock, NonCallableMock -from ..utils import MockHttpResource, MockKey +from ...utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer diff --git a/tests/rest/test_rooms.py b/tests/client/v1/test_rooms.py similarity index 98% rename from tests/rest/test_rooms.py rename to tests/client/v1/test_rooms.py index 8e65ff9a1..33a8631d7 100644 --- a/tests/rest/test_rooms.py +++ b/tests/client/v1/test_rooms.py @@ -18,7 +18,7 @@ # twisted imports from twisted.internet import defer -import synapse.rest.room +import synapse.client.v1.room from synapse.api.constants import Membership from synapse.server import HomeServer @@ -30,7 +30,7 @@ import json import urllib import types -from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from ...utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -82,7 +82,7 @@ class RoomPermissionsTestCase(RestTestCase): self.auth_user_id = self.rmcreator_id - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) self.auth = hs.get_auth() @@ -476,7 +476,7 @@ class RoomsMemberListTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) def tearDown(self): pass @@ -565,7 +565,7 @@ class RoomsCreateTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) def tearDown(self): pass @@ -668,7 +668,7 @@ class RoomTopicTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) # create the room self.room_id = yield self.create_room_as(self.user_id) @@ -783,7 +783,7 @@ class RoomMemberStateTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -919,7 +919,7 @@ class RoomMessagesTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -1023,7 +1023,7 @@ class RoomInitialSyncTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) # Since I'm getting my own presence I need to exist as far as presence # is concerned. diff --git a/tests/rest/test_typing.py b/tests/client/v1/test_typing.py similarity index 96% rename from tests/rest/test_typing.py rename to tests/client/v1/test_typing.py index 18138af1b..d6d677bde 100644 --- a/tests/rest/test_typing.py +++ b/tests/client/v1/test_typing.py @@ -18,10 +18,10 @@ # twisted imports from twisted.internet import defer -import synapse.rest.room +import synapse.client.v1.room from synapse.server import HomeServer -from ..utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey +from ...utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -104,7 +104,7 @@ class RoomTypingTestCase(RestTestCase): hs.get_handlers().room_member_handler.fetch_room_distributions_into = ( fetch_room_distributions_into) - synapse.rest.room.register_servlets(hs, self.mock_resource) + synapse.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) # Need another user to make notifications actually work diff --git a/tests/rest/utils.py b/tests/client/v1/utils.py similarity index 100% rename from tests/rest/utils.py rename to tests/client/v1/utils.py From b1b85753d759f7127fbb1c4a95005fffd3da7f4d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 15:50:17 +0000 Subject: [PATCH 025/630] Add support for storing rejected events in EventContext and data stores --- synapse/events/snapshot.py | 1 + synapse/storage/__init__.py | 11 +++++++--- synapse/storage/_base.py | 21 +++++++++++------- synapse/storage/rejections.py | 33 ++++++++++++++++++++++++++++ synapse/storage/schema/delta/v12.sql | 21 ++++++++++++++++++ synapse/storage/schema/im.sql | 7 ++++++ 6 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 synapse/storage/rejections.py create mode 100644 synapse/storage/schema/delta/v12.sql diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index 6bbba8d6b..7e98bdef2 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -20,3 +20,4 @@ class EventContext(object): self.current_state = current_state self.auth_events = auth_events self.state_group = None + self.rejected = False diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9..015fcc877 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .media_repository import MediaRepositoryStore +from .rejections import RejectionsStore from .state import StateStore from .signatures import SignatureStore @@ -66,7 +67,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): @@ -82,6 +83,7 @@ class DataStore(RoomMemberStore, RoomStore, DirectoryStore, KeyStore, StateStore, SignatureStore, EventFederationStore, MediaRepositoryStore, + RejectionsStore, ): def __init__(self, hs): @@ -224,6 +226,9 @@ class DataStore(RoomMemberStore, RoomStore, if not outlier: self._store_state_groups_txn(txn, event, context) + if context.rejected: + self._store_rejections_txn(txn, event.event_id, context.rejected) + if current_state: txn.execute( "DELETE FROM current_state_events WHERE room_id = ?", @@ -262,7 +267,7 @@ class DataStore(RoomMemberStore, RoomStore, or_replace=True, ) - if is_new_state: + if is_new_state and not context.rejected: self._simple_insert_txn( txn, "current_state_events", @@ -288,7 +293,7 @@ class DataStore(RoomMemberStore, RoomStore, or_ignore=True, ) - if not backfilled: + if not backfilled and not context.rejected: self._simple_insert_txn( txn, table="state_forward_extremities", diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f660fc6ea..2075a018b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -458,10 +458,12 @@ class SQLBaseStore(object): return [e for e in events if e] def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=False): + get_prev_content=False, allow_rejected=False): sql = ( - "SELECT internal_metadata, json, r.event_id FROM event_json as e " + "SELECT internal_metadata, json, r.event_id, reason " + "FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " + "LEFT JOIN rejections as rej on rej.event_id = e.event_id " "WHERE e.event_id = ? " "LIMIT 1 " ) @@ -473,13 +475,16 @@ class SQLBaseStore(object): if not res: return None - internal_metadata, js, redacted = res + internal_metadata, js, redacted, rejected_reason = res - return self._get_event_from_row_txn( - txn, internal_metadata, js, redacted, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - ) + if allow_rejected or not rejected_reason: + return self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + else: + return None def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py new file mode 100644 index 000000000..7d38b31f4 --- /dev/null +++ b/synapse/storage/rejections.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore + +import logging + +logger = logging.getLogger(__name__) + + +class RejectionsStore(SQLBaseStore): + def _store_rejections_txn(self, txn, event_id, reason): + self._simple_insert_txn( + txn, + table="rejections", + values={ + "event_id": event_id, + "reason": reason, + "last_failure": self._clock.time_msec(), + } + ) diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql new file mode 100644 index 000000000..bd2a8b1bb --- /dev/null +++ b/synapse/storage/schema/delta/v12.sql @@ -0,0 +1,21 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index dd00c1cd2..bc7c6b6ed 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -123,3 +123,10 @@ CREATE TABLE IF NOT EXISTS room_hosts( ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); From 73dd81ca62885110f7fe0e51e7f4e0183b495d17 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 15:57:08 +0000 Subject: [PATCH 026/630] fix pyflakes --- synapse/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index 5b622ad3b..d9fdfb34b 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -295,7 +295,7 @@ class StateHandler(object): # get around circular deps. self.hs.get_auth().check(event, auth_events) return event - except AuthError as e: + except AuthError: pass # Oh dear. From 97c68c508dac6b4b3203b3bc475ffdfd185b6e03 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Jan 2015 16:10:07 +0000 Subject: [PATCH 027/630] Move rest APIs back under the rest directory --- synapse/app/homeserver.py | 4 ++-- synapse/{client => rest}/__init__.py | 0 synapse/rest/client/__init__.py | 14 ++++++++++++++ synapse/{ => rest}/client/v1/__init__.py | 0 synapse/{ => rest}/client/v1/admin.py | 0 synapse/{ => rest}/client/v1/base.py | 0 synapse/{ => rest}/client/v1/directory.py | 0 synapse/{ => rest}/client/v1/events.py | 0 synapse/{ => rest}/client/v1/initial_sync.py | 0 synapse/{ => rest}/client/v1/login.py | 0 synapse/{ => rest}/client/v1/presence.py | 0 synapse/{ => rest}/client/v1/profile.py | 0 synapse/{ => rest}/client/v1/register.py | 0 synapse/{ => rest}/client/v1/room.py | 0 synapse/{ => rest}/client/v1/transactions.py | 0 synapse/{ => rest}/client/v1/voip.py | 0 synapse/{ => rest}/media/__init__.py | 0 synapse/{ => rest}/media/v0/__init__.py | 0 .../{ => rest}/media/v0/content_repository.py | 0 synapse/{ => rest}/media/v1/__init__.py | 0 synapse/{ => rest}/media/v1/base_resource.py | 0 synapse/{ => rest}/media/v1/download_resource.py | 0 synapse/{ => rest}/media/v1/filepath.py | 0 synapse/{ => rest}/media/v1/media_repository.py | 0 .../{ => rest}/media/v1/thumbnail_resource.py | 0 synapse/{ => rest}/media/v1/thumbnailer.py | 0 synapse/{ => rest}/media/v1/upload_resource.py | 0 synapse/server.py | 2 +- tests/client/v1/test_events.py | 12 ++++++------ tests/client/v1/test_rooms.py | 16 ++++++++-------- tests/client/v1/test_typing.py | 4 ++-- 31 files changed, 33 insertions(+), 19 deletions(-) rename synapse/{client => rest}/__init__.py (100%) create mode 100644 synapse/rest/client/__init__.py rename synapse/{ => rest}/client/v1/__init__.py (100%) rename synapse/{ => rest}/client/v1/admin.py (100%) rename synapse/{ => rest}/client/v1/base.py (100%) rename synapse/{ => rest}/client/v1/directory.py (100%) rename synapse/{ => rest}/client/v1/events.py (100%) rename synapse/{ => rest}/client/v1/initial_sync.py (100%) rename synapse/{ => rest}/client/v1/login.py (100%) rename synapse/{ => rest}/client/v1/presence.py (100%) rename synapse/{ => rest}/client/v1/profile.py (100%) rename synapse/{ => rest}/client/v1/register.py (100%) rename synapse/{ => rest}/client/v1/room.py (100%) rename synapse/{ => rest}/client/v1/transactions.py (100%) rename synapse/{ => rest}/client/v1/voip.py (100%) rename synapse/{ => rest}/media/__init__.py (100%) rename synapse/{ => rest}/media/v0/__init__.py (100%) rename synapse/{ => rest}/media/v0/content_repository.py (100%) rename synapse/{ => rest}/media/v1/__init__.py (100%) rename synapse/{ => rest}/media/v1/base_resource.py (100%) rename synapse/{ => rest}/media/v1/download_resource.py (100%) rename synapse/{ => rest}/media/v1/filepath.py (100%) rename synapse/{ => rest}/media/v1/media_repository.py (100%) rename synapse/{ => rest}/media/v1/thumbnail_resource.py (100%) rename synapse/{ => rest}/media/v1/thumbnailer.py (100%) rename synapse/{ => rest}/media/v1/upload_resource.py (100%) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index afe3d1976..cd24bbdc7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,8 +26,8 @@ from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect -from synapse.media.v0.content_repository import ContentRepoResource -from synapse.media.v1.media_repository import MediaRepositoryResource +from synapse.rest.media.v0.content_repository import ContentRepoResource +from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( diff --git a/synapse/client/__init__.py b/synapse/rest/__init__.py similarity index 100% rename from synapse/client/__init__.py rename to synapse/rest/__init__.py diff --git a/synapse/rest/client/__init__.py b/synapse/rest/client/__init__.py new file mode 100644 index 000000000..1a84d94cd --- /dev/null +++ b/synapse/rest/client/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py similarity index 100% rename from synapse/client/v1/__init__.py rename to synapse/rest/client/v1/__init__.py diff --git a/synapse/client/v1/admin.py b/synapse/rest/client/v1/admin.py similarity index 100% rename from synapse/client/v1/admin.py rename to synapse/rest/client/v1/admin.py diff --git a/synapse/client/v1/base.py b/synapse/rest/client/v1/base.py similarity index 100% rename from synapse/client/v1/base.py rename to synapse/rest/client/v1/base.py diff --git a/synapse/client/v1/directory.py b/synapse/rest/client/v1/directory.py similarity index 100% rename from synapse/client/v1/directory.py rename to synapse/rest/client/v1/directory.py diff --git a/synapse/client/v1/events.py b/synapse/rest/client/v1/events.py similarity index 100% rename from synapse/client/v1/events.py rename to synapse/rest/client/v1/events.py diff --git a/synapse/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py similarity index 100% rename from synapse/client/v1/initial_sync.py rename to synapse/rest/client/v1/initial_sync.py diff --git a/synapse/client/v1/login.py b/synapse/rest/client/v1/login.py similarity index 100% rename from synapse/client/v1/login.py rename to synapse/rest/client/v1/login.py diff --git a/synapse/client/v1/presence.py b/synapse/rest/client/v1/presence.py similarity index 100% rename from synapse/client/v1/presence.py rename to synapse/rest/client/v1/presence.py diff --git a/synapse/client/v1/profile.py b/synapse/rest/client/v1/profile.py similarity index 100% rename from synapse/client/v1/profile.py rename to synapse/rest/client/v1/profile.py diff --git a/synapse/client/v1/register.py b/synapse/rest/client/v1/register.py similarity index 100% rename from synapse/client/v1/register.py rename to synapse/rest/client/v1/register.py diff --git a/synapse/client/v1/room.py b/synapse/rest/client/v1/room.py similarity index 100% rename from synapse/client/v1/room.py rename to synapse/rest/client/v1/room.py diff --git a/synapse/client/v1/transactions.py b/synapse/rest/client/v1/transactions.py similarity index 100% rename from synapse/client/v1/transactions.py rename to synapse/rest/client/v1/transactions.py diff --git a/synapse/client/v1/voip.py b/synapse/rest/client/v1/voip.py similarity index 100% rename from synapse/client/v1/voip.py rename to synapse/rest/client/v1/voip.py diff --git a/synapse/media/__init__.py b/synapse/rest/media/__init__.py similarity index 100% rename from synapse/media/__init__.py rename to synapse/rest/media/__init__.py diff --git a/synapse/media/v0/__init__.py b/synapse/rest/media/v0/__init__.py similarity index 100% rename from synapse/media/v0/__init__.py rename to synapse/rest/media/v0/__init__.py diff --git a/synapse/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py similarity index 100% rename from synapse/media/v0/content_repository.py rename to synapse/rest/media/v0/content_repository.py diff --git a/synapse/media/v1/__init__.py b/synapse/rest/media/v1/__init__.py similarity index 100% rename from synapse/media/v1/__init__.py rename to synapse/rest/media/v1/__init__.py diff --git a/synapse/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py similarity index 100% rename from synapse/media/v1/base_resource.py rename to synapse/rest/media/v1/base_resource.py diff --git a/synapse/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py similarity index 100% rename from synapse/media/v1/download_resource.py rename to synapse/rest/media/v1/download_resource.py diff --git a/synapse/media/v1/filepath.py b/synapse/rest/media/v1/filepath.py similarity index 100% rename from synapse/media/v1/filepath.py rename to synapse/rest/media/v1/filepath.py diff --git a/synapse/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py similarity index 100% rename from synapse/media/v1/media_repository.py rename to synapse/rest/media/v1/media_repository.py diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py similarity index 100% rename from synapse/media/v1/thumbnail_resource.py rename to synapse/rest/media/v1/thumbnail_resource.py diff --git a/synapse/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py similarity index 100% rename from synapse/media/v1/thumbnailer.py rename to synapse/rest/media/v1/thumbnailer.py diff --git a/synapse/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py similarity index 100% rename from synapse/media/v1/upload_resource.py rename to synapse/rest/media/v1/upload_resource.py diff --git a/synapse/server.py b/synapse/server.py index 57a95bf75..e9add8e2b 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -24,7 +24,7 @@ from synapse.events.utils import serialize_event from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers -from synapse.client.v1 import RestServletFactory +from synapse.rest.client.v1 import RestServletFactory from synapse.state import StateHandler from synapse.storage import DataStore from synapse.types import UserID, RoomAlias, RoomID, EventID diff --git a/tests/client/v1/test_events.py b/tests/client/v1/test_events.py index 9b36dd322..e914b05a5 100644 --- a/tests/client/v1/test_events.py +++ b/tests/client/v1/test_events.py @@ -19,9 +19,9 @@ from tests import unittest # twisted imports from twisted.internet import defer -import synapse.client.v1.events -import synapse.client.v1.register -import synapse.client.v1.room +import synapse.rest.client.v1.events +import synapse.rest.client.v1.register +import synapse.rest.client.v1.room from synapse.server import HomeServer @@ -144,9 +144,9 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_clock().time_msec.return_value = 1000000 hs.get_clock().time.return_value = 1000 - synapse.client.v1.register.register_servlets(hs, self.mock_resource) - synapse.client.v1.events.register_servlets(hs, self.mock_resource) - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) # register an account self.user_id = "sid1" diff --git a/tests/client/v1/test_rooms.py b/tests/client/v1/test_rooms.py index 33a8631d7..4d529ef00 100644 --- a/tests/client/v1/test_rooms.py +++ b/tests/client/v1/test_rooms.py @@ -18,7 +18,7 @@ # twisted imports from twisted.internet import defer -import synapse.client.v1.room +import synapse.rest.client.v1.room from synapse.api.constants import Membership from synapse.server import HomeServer @@ -82,7 +82,7 @@ class RoomPermissionsTestCase(RestTestCase): self.auth_user_id = self.rmcreator_id - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) self.auth = hs.get_auth() @@ -476,7 +476,7 @@ class RoomsMemberListTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) def tearDown(self): pass @@ -565,7 +565,7 @@ class RoomsCreateTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) def tearDown(self): pass @@ -668,7 +668,7 @@ class RoomTopicTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) # create the room self.room_id = yield self.create_room_as(self.user_id) @@ -783,7 +783,7 @@ class RoomMemberStateTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -919,7 +919,7 @@ class RoomMessagesTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -1023,7 +1023,7 @@ class RoomInitialSyncTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) # Since I'm getting my own presence I need to exist as far as presence # is concerned. diff --git a/tests/client/v1/test_typing.py b/tests/client/v1/test_typing.py index d6d677bde..af3a9a6c1 100644 --- a/tests/client/v1/test_typing.py +++ b/tests/client/v1/test_typing.py @@ -18,7 +18,7 @@ # twisted imports from twisted.internet import defer -import synapse.client.v1.room +import synapse.rest.client.v1.room from synapse.server import HomeServer from ...utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey @@ -104,7 +104,7 @@ class RoomTypingTestCase(RestTestCase): hs.get_handlers().room_member_handler.fetch_room_distributions_into = ( fetch_room_distributions_into) - synapse.client.v1.room.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) # Need another user to make notifications actually work From 53584420a50d09b40f3235d8e4c033009e8eb0da Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 22 Jan 2015 16:13:27 +0000 Subject: [PATCH 028/630] Move client rest tests back under rest --- tests/{client => rest}/__init__.py | 0 tests/rest/client/__init__.py | 14 ++++++++++++++ tests/{ => rest}/client/v1/__init__.py | 0 tests/{ => rest}/client/v1/test_events.py | 2 +- tests/{ => rest}/client/v1/test_presence.py | 2 +- tests/{ => rest}/client/v1/test_profile.py | 2 +- tests/{ => rest}/client/v1/test_rooms.py | 2 +- tests/{ => rest}/client/v1/test_typing.py | 2 +- tests/{ => rest}/client/v1/utils.py | 0 9 files changed, 19 insertions(+), 5 deletions(-) rename tests/{client => rest}/__init__.py (100%) create mode 100644 tests/rest/client/__init__.py rename tests/{ => rest}/client/v1/__init__.py (100%) rename tests/{ => rest}/client/v1/test_events.py (99%) rename tests/{ => rest}/client/v1/test_presence.py (99%) rename tests/{ => rest}/client/v1/test_profile.py (99%) rename tests/{ => rest}/client/v1/test_rooms.py (99%) rename tests/{ => rest}/client/v1/test_typing.py (98%) rename tests/{ => rest}/client/v1/utils.py (100%) diff --git a/tests/client/__init__.py b/tests/rest/__init__.py similarity index 100% rename from tests/client/__init__.py rename to tests/rest/__init__.py diff --git a/tests/rest/client/__init__.py b/tests/rest/client/__init__.py new file mode 100644 index 000000000..1a84d94cd --- /dev/null +++ b/tests/rest/client/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/client/v1/__init__.py b/tests/rest/client/v1/__init__.py similarity index 100% rename from tests/client/v1/__init__.py rename to tests/rest/client/v1/__init__.py diff --git a/tests/client/v1/test_events.py b/tests/rest/client/v1/test_events.py similarity index 99% rename from tests/client/v1/test_events.py rename to tests/rest/client/v1/test_events.py index e914b05a5..0384ffbb3 100644 --- a/tests/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -25,7 +25,7 @@ import synapse.rest.client.v1.room from synapse.server import HomeServer -from ...utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock diff --git a/tests/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py similarity index 99% rename from tests/client/v1/test_presence.py rename to tests/rest/client/v1/test_presence.py index e7d636c74..0b6f7cfcc 100644 --- a/tests/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -20,7 +20,7 @@ from twisted.internet import defer from mock import Mock -from ...utils import MockHttpResource, MockKey +from ....utils import MockHttpResource, MockKey from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler diff --git a/tests/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py similarity index 99% rename from tests/client/v1/test_profile.py rename to tests/rest/client/v1/test_profile.py index 1182cc54e..47cfb10a6 100644 --- a/tests/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -20,7 +20,7 @@ from twisted.internet import defer from mock import Mock, NonCallableMock -from ...utils import MockHttpResource, MockKey +from ....utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer diff --git a/tests/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py similarity index 99% rename from tests/client/v1/test_rooms.py rename to tests/rest/client/v1/test_rooms.py index 4d529ef00..12f804054 100644 --- a/tests/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -30,7 +30,7 @@ import json import urllib import types -from ...utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock diff --git a/tests/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py similarity index 98% rename from tests/client/v1/test_typing.py rename to tests/rest/client/v1/test_typing.py index af3a9a6c1..647bcebfd 100644 --- a/tests/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -21,7 +21,7 @@ from twisted.internet import defer import synapse.rest.client.v1.room from synapse.server import HomeServer -from ...utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey +from ....utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock diff --git a/tests/client/v1/utils.py b/tests/rest/client/v1/utils.py similarity index 100% rename from tests/client/v1/utils.py rename to tests/rest/client/v1/utils.py From ca65a9d03e4e6e8bc9b42a3ff5a83c1e5294fc7b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Jan 2015 16:37:08 +0000 Subject: [PATCH 029/630] Split out TransactionQueue from replication layer --- synapse/federation/replication.py | 291 +--------------------- synapse/federation/transaction_queue.py | 314 ++++++++++++++++++++++++ 2 files changed, 316 insertions(+), 289 deletions(-) create mode 100644 synapse/federation/transaction_queue.py diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 6620532a6..accf95e40 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -20,8 +20,8 @@ a given transport. from twisted.internet import defer from .units import Transaction, Edu - from .persistence import TransactionActions +from .transaction_queue import TransactionQueue from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext @@ -62,7 +62,7 @@ class ReplicationLayer(object): # self.pdu_actions = PduActions(self.store) self.transaction_actions = TransactionActions(self.store) - self._transaction_queue = _TransactionQueue( + self._transaction_queue = TransactionQueue( hs, self.transaction_actions, transport_layer ) @@ -662,290 +662,3 @@ class ReplicationLayer(object): event.internal_metadata.outlier = outlier return event - - -class _TransactionQueue(object): - """This class makes sure we only have one transaction in flight at - a time for a given destination. - - It batches pending PDUs into single transactions. - """ - - def __init__(self, hs, transaction_actions, transport_layer): - self.server_name = hs.hostname - self.transaction_actions = transaction_actions - self.transport_layer = transport_layer - - self._clock = hs.get_clock() - self.store = hs.get_datastore() - - # Is a mapping from destinations -> deferreds. Used to keep track - # of which destinations have transactions in flight and when they are - # done - self.pending_transactions = {} - - # Is a mapping from destination -> list of - # tuple(pending pdus, deferred, order) - self.pending_pdus_by_dest = {} - # destination -> list of tuple(edu, deferred) - self.pending_edus_by_dest = {} - - # destination -> list of tuple(failure, deferred) - self.pending_failures_by_dest = {} - - # HACK to get unique tx id - self._next_txn_id = int(self._clock.time_msec()) - - @defer.inlineCallbacks - @log_function - def enqueue_pdu(self, pdu, destinations, order): - # We loop through all destinations to see whether we already have - # a transaction in progress. If we do, stick it in the pending_pdus - # table and we'll get back to it later. - - destinations = set(destinations) - destinations.discard(self.server_name) - destinations.discard("localhost") - - logger.debug("Sending to: %s", str(destinations)) - - if not destinations: - return - - deferreds = [] - - for destination in destinations: - deferred = defer.Deferred() - self.pending_pdus_by_dest.setdefault(destination, []).append( - (pdu, deferred, order) - ) - - def eb(failure): - if not deferred.called: - deferred.errback(failure) - else: - logger.warn("Failed to send pdu", failure) - - with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) - - deferreds.append(deferred) - - yield defer.DeferredList(deferreds) - - # NO inlineCallbacks - def enqueue_edu(self, edu): - destination = edu.destination - - if destination == self.server_name: - return - - deferred = defer.Deferred() - self.pending_edus_by_dest.setdefault(destination, []).append( - (edu, deferred) - ) - - def eb(failure): - if not deferred.called: - deferred.errback(failure) - else: - logger.warn("Failed to send edu", failure) - - with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) - - return deferred - - @defer.inlineCallbacks - def enqueue_failure(self, failure, destination): - deferred = defer.Deferred() - - self.pending_failures_by_dest.setdefault( - destination, [] - ).append( - (failure, deferred) - ) - - yield deferred - - @defer.inlineCallbacks - @log_function - def _attempt_new_transaction(self, destination): - - (retry_last_ts, retry_interval) = (0, 0) - retry_timings = yield self.store.get_destination_retry_timings( - destination - ) - if retry_timings: - (retry_last_ts, retry_interval) = ( - retry_timings.retry_last_ts, retry_timings.retry_interval - ) - if retry_last_ts + retry_interval > int(self._clock.time_msec()): - logger.info( - "TX [%s] not ready for retry yet - " - "dropping transaction for now", - destination, - ) - return - else: - logger.info("TX [%s] is ready for retry", destination) - - logger.info("TX [%s] _attempt_new_transaction", destination) - - if destination in self.pending_transactions: - # XXX: pending_transactions can get stuck on by a never-ending - # request at which point pending_pdus_by_dest just keeps growing. - # we need application-layer timeouts of some flavour of these - # requests - return - - # list of (pending_pdu, deferred, order) - pending_pdus = self.pending_pdus_by_dest.pop(destination, []) - pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_failures = self.pending_failures_by_dest.pop(destination, []) - - if pending_pdus: - logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", - destination, len(pending_pdus)) - - if not pending_pdus and not pending_edus and not pending_failures: - return - - logger.debug( - "TX [%s] Attempting new transaction" - " (pdus: %d, edus: %d, failures: %d)", - destination, - len(pending_pdus), - len(pending_edus), - len(pending_failures) - ) - - # Sort based on the order field - pending_pdus.sort(key=lambda t: t[2]) - - pdus = [x[0] for x in pending_pdus] - edus = [x[0] for x in pending_edus] - failures = [x[0].get_dict() for x in pending_failures] - deferreds = [ - x[1] - for x in pending_pdus + pending_edus + pending_failures - ] - - try: - self.pending_transactions[destination] = 1 - - logger.debug("TX [%s] Persisting transaction...", destination) - - transaction = Transaction.create_new( - origin_server_ts=int(self._clock.time_msec()), - transaction_id=str(self._next_txn_id), - origin=self.server_name, - destination=destination, - pdus=pdus, - edus=edus, - pdu_failures=failures, - ) - - self._next_txn_id += 1 - - yield self.transaction_actions.prepare_to_send(transaction) - - logger.debug("TX [%s] Persisted transaction", destination) - logger.info( - "TX [%s] Sending transaction [%s]", - destination, - transaction.transaction_id, - ) - - # Actually send the transaction - - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def json_data_cb(): - data = transaction.get_dict() - now = int(self._clock.time_msec()) - if "pdus" in data: - for p in data["pdus"]: - if "age_ts" in p: - unsigned = p.setdefault("unsigned", {}) - unsigned["age"] = now - int(p["age_ts"]) - del p["age_ts"] - return data - - code, response = yield self.transport_layer.send_transaction( - transaction, json_data_cb - ) - - logger.info("TX [%s] got %d response", destination, code) - - logger.debug("TX [%s] Sent transaction", destination) - logger.debug("TX [%s] Marking as delivered...", destination) - - yield self.transaction_actions.delivered( - transaction, code, response - ) - - logger.debug("TX [%s] Marked as delivered", destination) - logger.debug("TX [%s] Yielding to callbacks...", destination) - - for deferred in deferreds: - if code == 200: - if retry_last_ts: - # this host is alive! reset retry schedule - yield self.store.set_destination_retry_timings( - destination, 0, 0 - ) - deferred.callback(None) - else: - self.set_retrying(destination, retry_interval) - deferred.errback(RuntimeError("Got status %d" % code)) - - # Ensures we don't continue until all callbacks on that - # deferred have fired - try: - yield deferred - except: - pass - - logger.debug("TX [%s] Yielded to callbacks", destination) - - except Exception as e: - # We capture this here as there as nothing actually listens - # for this finishing functions deferred. - logger.warn( - "TX [%s] Problem in _attempt_transaction: %s", - destination, - e, - ) - - self.set_retrying(destination, retry_interval) - - for deferred in deferreds: - if not deferred.called: - deferred.errback(e) - - finally: - # We want to be *very* sure we delete this after we stop processing - self.pending_transactions.pop(destination, None) - - # Check to see if there is anything else to send. - self._attempt_new_transaction(destination) - - @defer.inlineCallbacks - def set_retrying(self, destination, retry_interval): - # track that this destination is having problems and we should - # give it a chance to recover before trying it again - - if retry_interval: - retry_interval *= 2 - # plateau at hourly retries for now - if retry_interval >= 60 * 60 * 1000: - retry_interval = 60 * 60 * 1000 - else: - retry_interval = 2000 # try again at first after 2 seconds - - yield self.store.set_destination_retry_timings( - destination, - int(self._clock.time_msec()), - retry_interval - ) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py new file mode 100644 index 000000000..c2cb4a1c4 --- /dev/null +++ b/synapse/federation/transaction_queue.py @@ -0,0 +1,314 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Transaction + +from synapse.util.logutils import log_function +from synapse.util.logcontext import PreserveLoggingContext + +import logging + + +logger = logging.getLogger(__name__) + + +class TransactionQueue(object): + """This class makes sure we only have one transaction in flight at + a time for a given destination. + + It batches pending PDUs into single transactions. + """ + + def __init__(self, hs, transaction_actions, transport_layer): + self.server_name = hs.hostname + self.transaction_actions = transaction_actions + self.transport_layer = transport_layer + + self._clock = hs.get_clock() + self.store = hs.get_datastore() + + # Is a mapping from destinations -> deferreds. Used to keep track + # of which destinations have transactions in flight and when they are + # done + self.pending_transactions = {} + + # Is a mapping from destination -> list of + # tuple(pending pdus, deferred, order) + self.pending_pdus_by_dest = {} + # destination -> list of tuple(edu, deferred) + self.pending_edus_by_dest = {} + + # destination -> list of tuple(failure, deferred) + self.pending_failures_by_dest = {} + + # HACK to get unique tx id + self._next_txn_id = int(self._clock.time_msec()) + + @defer.inlineCallbacks + @log_function + def enqueue_pdu(self, pdu, destinations, order): + # We loop through all destinations to see whether we already have + # a transaction in progress. If we do, stick it in the pending_pdus + # table and we'll get back to it later. + + destinations = set(destinations) + destinations.discard(self.server_name) + destinations.discard("localhost") + + logger.debug("Sending to: %s", str(destinations)) + + if not destinations: + return + + deferreds = [] + + for destination in destinations: + deferred = defer.Deferred() + self.pending_pdus_by_dest.setdefault(destination, []).append( + (pdu, deferred, order) + ) + + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send pdu", failure) + + with PreserveLoggingContext(): + self._attempt_new_transaction(destination).addErrback(eb) + + deferreds.append(deferred) + + yield defer.DeferredList(deferreds) + + # NO inlineCallbacks + def enqueue_edu(self, edu): + destination = edu.destination + + if destination == self.server_name: + return + + deferred = defer.Deferred() + self.pending_edus_by_dest.setdefault(destination, []).append( + (edu, deferred) + ) + + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send edu", failure) + + with PreserveLoggingContext(): + self._attempt_new_transaction(destination).addErrback(eb) + + return deferred + + @defer.inlineCallbacks + def enqueue_failure(self, failure, destination): + deferred = defer.Deferred() + + self.pending_failures_by_dest.setdefault( + destination, [] + ).append( + (failure, deferred) + ) + + yield deferred + + @defer.inlineCallbacks + @log_function + def _attempt_new_transaction(self, destination): + + (retry_last_ts, retry_interval) = (0, 0) + retry_timings = yield self.store.get_destination_retry_timings( + destination + ) + if retry_timings: + (retry_last_ts, retry_interval) = ( + retry_timings.retry_last_ts, retry_timings.retry_interval + ) + if retry_last_ts + retry_interval > int(self._clock.time_msec()): + logger.info( + "TX [%s] not ready for retry yet - " + "dropping transaction for now", + destination, + ) + return + else: + logger.info("TX [%s] is ready for retry", destination) + + logger.info("TX [%s] _attempt_new_transaction", destination) + + if destination in self.pending_transactions: + # XXX: pending_transactions can get stuck on by a never-ending + # request at which point pending_pdus_by_dest just keeps growing. + # we need application-layer timeouts of some flavour of these + # requests + return + + # list of (pending_pdu, deferred, order) + pending_pdus = self.pending_pdus_by_dest.pop(destination, []) + pending_edus = self.pending_edus_by_dest.pop(destination, []) + pending_failures = self.pending_failures_by_dest.pop(destination, []) + + if pending_pdus: + logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", + destination, len(pending_pdus)) + + if not pending_pdus and not pending_edus and not pending_failures: + return + + logger.debug( + "TX [%s] Attempting new transaction" + " (pdus: %d, edus: %d, failures: %d)", + destination, + len(pending_pdus), + len(pending_edus), + len(pending_failures) + ) + + # Sort based on the order field + pending_pdus.sort(key=lambda t: t[2]) + + pdus = [x[0] for x in pending_pdus] + edus = [x[0] for x in pending_edus] + failures = [x[0].get_dict() for x in pending_failures] + deferreds = [ + x[1] + for x in pending_pdus + pending_edus + pending_failures + ] + + try: + self.pending_transactions[destination] = 1 + + logger.debug("TX [%s] Persisting transaction...", destination) + + transaction = Transaction.create_new( + origin_server_ts=int(self._clock.time_msec()), + transaction_id=str(self._next_txn_id), + origin=self.server_name, + destination=destination, + pdus=pdus, + edus=edus, + pdu_failures=failures, + ) + + self._next_txn_id += 1 + + yield self.transaction_actions.prepare_to_send(transaction) + + logger.debug("TX [%s] Persisted transaction", destination) + logger.info( + "TX [%s] Sending transaction [%s]", + destination, + transaction.transaction_id, + ) + + # Actually send the transaction + + # FIXME (erikj): This is a bit of a hack to make the Pdu age + # keys work + def json_data_cb(): + data = transaction.get_dict() + now = int(self._clock.time_msec()) + if "pdus" in data: + for p in data["pdus"]: + if "age_ts" in p: + unsigned = p.setdefault("unsigned", {}) + unsigned["age"] = now - int(p["age_ts"]) + del p["age_ts"] + return data + + code, response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + + logger.info("TX [%s] got %d response", destination, code) + + logger.debug("TX [%s] Sent transaction", destination) + logger.debug("TX [%s] Marking as delivered...", destination) + + yield self.transaction_actions.delivered( + transaction, code, response + ) + + logger.debug("TX [%s] Marked as delivered", destination) + logger.debug("TX [%s] Yielding to callbacks...", destination) + + for deferred in deferreds: + if code == 200: + if retry_last_ts: + # this host is alive! reset retry schedule + yield self.store.set_destination_retry_timings( + destination, 0, 0 + ) + deferred.callback(None) + else: + self.set_retrying(destination, retry_interval) + deferred.errback(RuntimeError("Got status %d" % code)) + + # Ensures we don't continue until all callbacks on that + # deferred have fired + try: + yield deferred + except: + pass + + logger.debug("TX [%s] Yielded to callbacks", destination) + + except Exception as e: + # We capture this here as there as nothing actually listens + # for this finishing functions deferred. + logger.warn( + "TX [%s] Problem in _attempt_transaction: %s", + destination, + e, + ) + + self.set_retrying(destination, retry_interval) + + for deferred in deferreds: + if not deferred.called: + deferred.errback(e) + + finally: + # We want to be *very* sure we delete this after we stop processing + self.pending_transactions.pop(destination, None) + + # Check to see if there is anything else to send. + self._attempt_new_transaction(destination) + + @defer.inlineCallbacks + def set_retrying(self, destination, retry_interval): + # track that this destination is having problems and we should + # give it a chance to recover before trying it again + + if retry_interval: + retry_interval *= 2 + # plateau at hourly retries for now + if retry_interval >= 60 * 60 * 1000: + retry_interval = 60 * 60 * 1000 + else: + retry_interval = 2000 # try again at first after 2 seconds + + yield self.store.set_destination_retry_timings( + destination, + int(self._clock.time_msec()), + retry_interval + ) From dc93860619d56e88844e91f38f66341a32e4c704 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:37:12 +0000 Subject: [PATCH 030/630] Add rest API & store for creating push rules Also make unrecognised request error look more like synapse errors because it makes it easier to throw them from within rest classes. --- synapse/rest/push_rule.py | 195 ++++++++++++++++++++++++++++++++++ synapse/storage/push_rule.py | 196 +++++++++++++++++++++++++++++++++++ 2 files changed, 391 insertions(+) create mode 100644 synapse/rest/push_rule.py create mode 100644 synapse/storage/push_rule.py diff --git a/synapse/rest/push_rule.py b/synapse/rest/push_rule.py new file mode 100644 index 000000000..b5e74479c --- /dev/null +++ b/synapse/rest/push_rule.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError +from base import RestServlet, client_path_pattern +from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException + +import json + + +class PushRuleRestServlet(RestServlet): + PATTERN = client_path_pattern("/pushrules/.*$") + + def rule_spec_from_path(self, path): + if len(path) < 2: + raise UnrecognizedRequestError() + if path[0] != 'pushrules': + raise UnrecognizedRequestError() + + scope = path[1] + path = path[2:] + if scope not in ['global', 'device']: + raise UnrecognizedRequestError() + + device = None + if scope == 'device': + if len(path) == 0: + raise UnrecognizedRequestError() + device = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + template = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + rule_id = path[0] + + spec = { + 'scope' : scope, + 'template': template, + 'rule_id': rule_id + } + if device: + spec['device'] = device + return spec + + def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj): + if rule_template in ['override', 'underride']: + if 'conditions' not in req_obj: + raise InvalidRuleException("Missing 'conditions'") + conditions = req_obj['conditions'] + for c in conditions: + if 'kind' not in c: + raise InvalidRuleException("Condition without 'kind'") + elif rule_template == 'room': + conditions = [{ + 'kind': 'event_match', + 'key': 'room_id', + 'pattern': rule_id + }] + elif rule_template == 'sender': + conditions = [{ + 'kind': 'event_match', + 'key': 'user_id', + 'pattern': rule_id + }] + elif rule_template == 'content': + if 'pattern' not in req_obj: + raise InvalidRuleException("Content rule missing 'pattern'") + conditions = [{ + 'kind': 'event_match', + 'key': 'content.body', + 'pattern': req_obj['pattern'] + }] + else: + raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + + if 'actions' not in req_obj: + raise InvalidRuleException("No actions found") + actions = req_obj['actions'] + + for a in actions: + if a in ['notify', 'dont-notify', 'coalesce']: + pass + elif isinstance(a, dict) and 'set_sound' in a: + pass + else: + raise InvalidRuleException("Unrecognised action") + + return (conditions, actions) + + def priority_class_from_spec(self, spec): + map = { + 'underride': 0, + 'sender': 1, + 'room': 2, + 'content': 3, + 'override': 4 + } + + if spec['template'] not in map.keys(): + raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) + pc = map[spec['template']] + + if spec['scope'] == 'device': + pc += 5 + + return pc + + @defer.inlineCallbacks + def on_PUT(self, request): + spec = self.rule_spec_from_path(request.postpath) + try: + priority_class = self.priority_class_from_spec(spec) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + user = yield self.auth.get_user_by_req(request) + + content = _parse_json(request) + + try: + (conditions, actions) = self.rule_tuple_from_request_object( + spec['template'], + spec['rule_id'], + content + ) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + before = request.args.get("before", None) + if before and len(before): + before = before[0] + after = request.args.get("after", None) + if after and len(after): + after = after[0] + + try: + yield self.hs.get_datastore().add_push_rule( + user_name=user.to_string(), + rule_id=spec['rule_id'], + priority_class=priority_class, + conditions=conditions, + actions=actions, + before=before, + after=after + ) + except InconsistentRuleException as e: + raise SynapseError(400, e.message) + except RuleNotFoundException: + raise SynapseError(400, "before/after rule not found") + + defer.returnValue((200, {})) + + def on_OPTIONS(self, _): + return 200, {} + + +class InvalidRuleException(Exception): + pass + + +# XXX: C+ped from rest/room.py - surely this should be common? +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.", + errcode=Codes.NOT_JSON) + return content + except ValueError: + raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON) + + +def register_servlets(hs, http_server): + PushRuleRestServlet(hs).register(http_server) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py new file mode 100644 index 000000000..76c455760 --- /dev/null +++ b/synapse/storage/push_rule.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from ._base import SQLBaseStore, Table +from twisted.internet import defer + +import logging +import copy +import json + +logger = logging.getLogger(__name__) + + +class PushRuleStore(SQLBaseStore): + @defer.inlineCallbacks + def get_push_rules_for_user_name(self, user_name): + sql = ( + "SELECT "+",".join(PushRuleTable.fields)+ + "FROM pushers " + "WHERE user_name = ?" + ) + + rows = yield self._execute(None, sql, user_name) + + dicts = [] + for r in rows: + d = {} + for i, f in enumerate(PushRuleTable.fields): + d[f] = r[i] + dicts.append(d) + + defer.returnValue(dicts) + + @defer.inlineCallbacks + def add_push_rule(self, **kwargs): + vals = copy.copy(kwargs) + if 'conditions' in vals: + vals['conditions'] = json.dumps(vals['conditions']) + if 'actions' in vals: + vals['actions'] = json.dumps(vals['actions']) + # we could check the rest of the keys are valid column names + # but sqlite will do that anyway so I think it's just pointless. + if 'id' in vals: + del vals['id'] + + if 'after' in kwargs or 'before' in kwargs: + ret = yield self.runInteraction( + "_add_push_rule_relative_txn", + self._add_push_rule_relative_txn, + **vals + ) + defer.returnValue(ret) + else: + ret = yield self.runInteraction( + "_add_push_rule_highest_priority_txn", + self._add_push_rule_highest_priority_txn, + **vals + ) + defer.returnValue(ret) + + def _add_push_rule_relative_txn(self, txn, user_name, **kwargs): + after = None + relative_to_rule = None + if 'after' in kwargs and kwargs['after']: + after = kwargs['after'] + relative_to_rule = after + if 'before' in kwargs and kwargs['before']: + relative_to_rule = kwargs['before'] + + # get the priority of the rule we're inserting after/before + sql = ( + "SELECT priority_class, priority FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? and rule_id = ?" + ) + txn.execute(sql, (user_name, relative_to_rule)) + res = txn.fetchall() + if not res: + raise RuleNotFoundException() + (priority_class, base_rule_priority) = res[0] + + if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: + raise InconsistentRuleException( + "Given priority class does not match class of relative rule" + ) + + new_rule = copy.copy(kwargs) + if 'before' in new_rule: + del new_rule['before'] + if 'after' in new_rule: + del new_rule['after'] + new_rule['priority_class'] = priority_class + new_rule['user_name'] = user_name + + # check if the priority before/after is free + new_rule_priority = base_rule_priority + if after: + new_rule_priority -= 1 + else: + new_rule_priority += 1 + + new_rule['priority'] = new_rule_priority + + sql = ( + "SELECT COUNT(*) FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? AND priority_class = ? AND priority = ?" + ) + txn.execute(sql, (user_name, priority_class, new_rule_priority)) + res = txn.fetchall() + num_conflicting = res[0][0] + + # if there are conflicting rules, bump everything + if num_conflicting: + sql = "UPDATE "+PushRuleTable.table_name+" SET priority = priority " + if after: + sql += "-1" + else: + sql += "+1" + sql += " WHERE user_name = ? AND priority_class = ? AND priority " + if after: + sql += "<= ?" + else: + sql += ">= ?" + + txn.execute(sql, (user_name, priority_class, new_rule_priority)) + + # now insert the new rule + sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql += ",".join(new_rule.keys())+") VALUES (" + sql += ", ".join(["?" for _ in new_rule.keys()])+")" + + txn.execute(sql, new_rule.values()) + + def _add_push_rule_highest_priority_txn(self, txn, user_name, priority_class, **kwargs): + # find the highest priority rule in that class + sql = ( + "SELECT COUNT(*), MAX(priority) FROM "+PushRuleTable.table_name+ + " WHERE user_name = ? and priority_class = ?" + ) + txn.execute(sql, (user_name, priority_class)) + res = txn.fetchall() + (how_many, highest_prio) = res[0] + + new_prio = 0 + if how_many > 0: + new_prio = highest_prio + 1 + + # and insert the new rule + new_rule = copy.copy(kwargs) + if 'id' in new_rule: + del new_rule['id'] + new_rule['user_name'] = user_name + new_rule['priority_class'] = priority_class + new_rule['priority'] = new_prio + + sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql += ",".join(new_rule.keys())+") VALUES (" + sql += ", ".join(["?" for _ in new_rule.keys()])+")" + + txn.execute(sql, new_rule.values()) + +class RuleNotFoundException(Exception): + pass + + +class InconsistentRuleException(Exception): + pass + + +class PushRuleTable(Table): + table_name = "push_rules" + + fields = [ + "id", + "user_name", + "rule_id", + "priority_class", + "priority", + "conditions", + "actions", + ] + + EntryType = collections.namedtuple("PushRuleEntry", fields) \ No newline at end of file From ede491b4e0c14d44ce43dd5b152abf148b54b9ed Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:38:53 +0000 Subject: [PATCH 031/630] Oops: second part of commit dc938606 --- synapse/api/errors.py | 12 ++++++++++++ synapse/http/server.py | 8 ++------ synapse/rest/__init__.py | 3 ++- synapse/storage/__init__.py | 3 +++ synapse/storage/schema/delta/v10.sql | 13 +++++++++++++ synapse/storage/schema/pusher.sql | 13 +++++++++++++ 6 files changed, 45 insertions(+), 7 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index a4155aeba..55181fe77 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) class Codes(object): + UNRECOGNIZED = "M_UNRECOGNIZED" UNAUTHORIZED = "M_UNAUTHORIZED" FORBIDDEN = "M_FORBIDDEN" BAD_JSON = "M_BAD_JSON" @@ -82,6 +83,17 @@ class RegistrationError(SynapseError): pass +class UnrecognizedRequestError(SynapseError): + """An error indicating we don't understand the request you're trying to make""" + def __init__(self, *args, **kwargs): + if "errcode" not in kwargs: + kwargs["errcode"] = Codes.NOT_FOUND + super(UnrecognizedRequestError, self).__init__( + 400, + "Unrecognized request", + **kwargs + ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/http/server.py b/synapse/http/server.py index 8015a22ed..0f6539e1b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -16,7 +16,7 @@ from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException + cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) from synapse.util.logcontext import LoggingContext @@ -139,11 +139,7 @@ class JsonResource(HttpServer, resource.Resource): return # Huh. No one wanted to handle that? Fiiiiiine. Send 400. - self._send_response( - request, - 400, - {"error": "Unrecognized request"} - ) + raise UnrecognizedRequestError() except CodeMessageException as e: if isinstance(e, SynapseError): logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 59521d0c7..8e5877cf3 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -16,7 +16,7 @@ from . import ( room, events, register, login, profile, presence, initial_sync, directory, - voip, admin, pusher, + voip, admin, pusher, push_rule ) @@ -46,3 +46,4 @@ class RestServletFactory(object): voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) pusher.register_servlets(hs, client_resource) + push_rule.register_servlets(hs, client_resource) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 191fe462a..11706676d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .pusher import PusherStore +from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore from .state import StateStore @@ -62,6 +63,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "pusher", + "push_rules", "media_repository", ] @@ -85,6 +87,7 @@ class DataStore(RoomMemberStore, RoomStore, EventFederationStore, MediaRepositoryStore, PusherStore, + PushRuleStore ): def __init__(self, hs): diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v10.sql index b84ce20ef..8c4dfd5c1 100644 --- a/synapse/storage/schema/delta/v10.sql +++ b/synapse/storage/schema/delta/v10.sql @@ -31,3 +31,16 @@ CREATE TABLE IF NOT EXISTS pushers ( FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index b84ce20ef..8c4dfd5c1 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -31,3 +31,16 @@ CREATE TABLE IF NOT EXISTS pushers ( FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); + +CREATE TABLE IF NOT EXISTS push_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); From 7ecb49ef25937558f1a19a8fe47879d4b9116316 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 17:53:30 +0000 Subject: [PATCH 032/630] Insufficient newlines --- synapse/storage/push_rule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 76c455760..dbbb35b2a 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -172,6 +172,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) + class RuleNotFoundException(Exception): pass From 673773b21701c91997512d568bcc8d49a5586b3a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 18:27:07 +0000 Subject: [PATCH 033/630] oops, this is not its own schema file --- synapse/storage/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 11706676d..8f56d90d9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -63,7 +63,6 @@ SCHEMAS = [ "event_edges", "event_signatures", "pusher", - "push_rules", "media_repository", ] From 8a850573c9cf50dd83ba47c033b28fe2bbbaf9d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 22 Jan 2015 19:32:17 +0000 Subject: [PATCH 034/630] As yet fairly untested GET API for push rules --- synapse/api/errors.py | 14 ++- synapse/rest/client/v1/push_rule.py | 138 ++++++++++++++++++++++++++-- synapse/storage/push_rule.py | 8 +- 3 files changed, 145 insertions(+), 15 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 55181fe77..01207282d 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -87,13 +87,25 @@ class UnrecognizedRequestError(SynapseError): """An error indicating we don't understand the request you're trying to make""" def __init__(self, *args, **kwargs): if "errcode" not in kwargs: - kwargs["errcode"] = Codes.NOT_FOUND + kwargs["errcode"] = Codes.UNRECOGNIZED super(UnrecognizedRequestError, self).__init__( 400, "Unrecognized request", **kwargs ) + +class NotFoundError(SynapseError): + """An error indicating we can't find the thing you asked for""" + def __init__(self, *args, **kwargs): + if "errcode" not in kwargs: + kwargs["errcode"] = Codes.NOT_FOUND + super(UnrecognizedRequestError, self).__init__( + 404, + "Not found", + **kwargs + ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index b5e74479c..2803c1f07 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError from base import RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException @@ -24,6 +24,14 @@ import json class PushRuleRestServlet(RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") + PRIORITY_CLASS_MAP = { + 'underride': 0, + 'sender': 1, + 'room': 2, + 'content': 3, + 'override': 4 + } + PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} def rule_spec_from_path(self, path): if len(path) < 2: @@ -109,15 +117,7 @@ class PushRuleRestServlet(RestServlet): return (conditions, actions) def priority_class_from_spec(self, spec): - map = { - 'underride': 0, - 'sender': 1, - 'room': 2, - 'content': 3, - 'override': 4 - } - - if spec['template'] not in map.keys(): + if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) pc = map[spec['template']] @@ -171,10 +171,128 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_GET(self, request): + user = yield self.auth.get_user_by_req(request) + + # we build up the full structure and then decide which bits of it + # to send which means doing unnecessary work sometimes but is + # is probably not going to make a whole lot of difference + rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + + rules = {'global': {}, 'device': {}} + + rules['global'] = _add_empty_priority_class_arrays(rules['global']) + + for r in rawrules: + rulearray = None + + r["conditions"] = json.loads(r["conditions"]) + r["actions"] = json.loads(r["actions"]) + + template_name = _priority_class_to_template_name(r['priority_class']) + + if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + # per-device rule + instance_handle = _instance_handle_from_conditions(r["conditions"]) + if not instance_handle: + continue + if instance_handle not in rules['device']: + rules['device'][instance_handle] = [] + rules['device'][instance_handle] = \ + _add_empty_priority_class_arrays(rules['device'][instance_handle]) + + rulearray = rules['device'][instance_handle] + else: + rulearray = rules['global'][template_name] + + template_rule = _rule_to_template(r) + if template_rule: + rulearray.append(template_rule) + + path = request.postpath[1:] + if path == []: + defer.returnValue((200, rules)) + + if path[0] == 'global': + path = path[1:] + result = _filter_ruleset_with_path(rules['global'], path) + defer.returnValue((200, result)) + elif path[0] == 'device': + path = path[1:] + if path == []: + raise UnrecognizedRequestError + instance_handle = path[0] + if instance_handle not in rules['device']: + ret = {} + ret = _add_empty_priority_class_arrays(ret) + defer.returnValue((200, ret)) + ruleset = rules['device'][instance_handle] + result = _filter_ruleset_with_path(ruleset, path) + defer.returnValue((200, result)) + else: + raise UnrecognizedRequestError() + + def on_OPTIONS(self, _): return 200, {} +def _add_empty_priority_class_arrays(d): + for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + d[pc] = [] + return d + +def _instance_handle_from_conditions(conditions): + """ + Given a list of conditions, return the instance handle of the + device rule if there is one + """ + for c in conditions: + if c['kind'] == 'device': + return c['instance_handle'] + return None + +def _filter_ruleset_with_path(ruleset, path): + if path == []: + return ruleset + template_kind = path[0] + if template_kind not in ruleset: + raise UnrecognizedRequestError() + path = path[1:] + if path == []: + return ruleset[template_kind] + rule_id = path[0] + for r in ruleset[template_kind]: + if r['rule_id'] == rule_id: + return r + raise NotFoundError + +def _priority_class_to_template_name(pc): + if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + # per-device + prio_class_index = pc - PushRuleRestServlet.PRIORITY_CLASS_MAP['override'] + return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[prio_class_index] + else: + return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + +def _rule_to_template(rule): + template_name = _priority_class_to_template_name(rule['priority_class']) + if template_name in ['override', 'underride']: + return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} + elif template_name in ["sender", "room"]: + return {k:rule[k] for k in ["rule_id", "actions"]} + elif template_name == 'content': + if len(rule["conditions"]) != 1: + return None + thecond = rule["conditions"][0] + if "pattern" not in thecond: + return None + ret = {k:rule[k] for k in ["rule_id", "actions"]} + ret["pattern"] = thecond["pattern"] + return ret + + class InvalidRuleException(Exception): pass diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index dbbb35b2a..d087257ff 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -29,11 +29,11 @@ class PushRuleStore(SQLBaseStore): @defer.inlineCallbacks def get_push_rules_for_user_name(self, user_name): sql = ( - "SELECT "+",".join(PushRuleTable.fields)+ - "FROM pushers " - "WHERE user_name = ?" + "SELECT "+",".join(PushRuleTable.fields)+" " + "FROM "+PushRuleTable.table_name+" " + "WHERE user_name = ? " + "ORDER BY priority_class DESC, priority DESC" ) - rows = yield self._execute(None, sql, user_name) dicts = [] From cbb10879cb4bd16faf8fc010271bee966e1e6cad Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 22 Jan 2015 17:54:52 +0000 Subject: [PATCH 035/630] Much merging of test case setUp() methods to make them much more shareable --- tests/handlers/test_presence.py | 468 +++++++++++++------------------- 1 file changed, 194 insertions(+), 274 deletions(-) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index c309fbb05..805f3868b 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -63,9 +63,6 @@ class JustPresenceHandlers(object): class PresenceTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - self.clock = MockClock() self.mock_config = NonCallableMock() @@ -76,6 +73,15 @@ class PresenceTestCase(unittest.TestCase): self.mock_http_client = Mock(spec=[]) self.mock_http_client.put_json = DeferredMockCallable() + db_pool = None + hs_kwargs = {} + + if hasattr(self, "make_datastore_mock"): + hs_kwargs["datastore"] = self.make_datastore_mock() + else: + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer("test", clock=self.clock, db_pool=db_pool, @@ -84,38 +90,33 @@ class PresenceTestCase(unittest.TestCase): http_client=self.mock_http_client, config=self.mock_config, keyring=Mock(), + **hs_kwargs ) hs.handlers = JustPresenceHandlers(hs) - self.store = hs.get_datastore() + self.datastore = hs.get_datastore() - # Mock the RoomMemberHandler - room_member_handler = Mock(spec=[]) - hs.handlers.room_member_handler = room_member_handler - - # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") - - for u in self.u_apple, self.u_banana, self.u_clementine: - yield self.store.create_presence(u.localpart) - - yield self.store.set_presence_state( - self.u_apple.localpart, {"state": ONLINE, "status_msg": "Online"} - ) - - # ID of a local user that does not exist - self.u_durian = hs.parse_userid("@durian:test") - - # A remote user - self.u_cabbage = hs.parse_userid("@cabbage:elsewhere") + self.setUp_roommemberhandler_mocks(hs.handlers) self.handler = hs.get_handlers().presence_handler + self.event_source = hs.get_event_sources().sources["presence"] + self.distributor = hs.get_distributor() + self.distributor.declare("user_joined_room") + + yield self.setUp_users(hs) + + def setUp_roommemberhandler_mocks(self, handlers): self.room_id = "a-room" self.room_members = [] + room_member_handler = handlers.room_member_handler = Mock(spec=[ + "get_rooms_for_user", + "get_room_members", + "fetch_room_distributions_into", + ]) + self.room_member_handler = room_member_handler + def get_rooms_for_user(user): if user in self.room_members: return defer.succeed([self.room_id]) @@ -130,12 +131,144 @@ class PresenceTestCase(unittest.TestCase): return defer.succeed([]) room_member_handler.get_room_members = get_room_members + @defer.inlineCallbacks + def fetch_room_distributions_into(room_id, localusers=None, + remotedomains=None, ignore_user=None): + + members = yield get_room_members(room_id) + for member in members: + if ignore_user is not None and member == ignore_user: + continue + + if member.is_mine: + if localusers is not None: + localusers.add(member) + else: + if remotedomains is not None: + remotedomains.add(member.domain) + room_member_handler.fetch_room_distributions_into = ( + fetch_room_distributions_into) + + self.setUp_datastore_room_mocks(self.datastore) + + def setUp_datastore_room_mocks(self, datastore): + def get_room_hosts(room_id): + if room_id == self.room_id: + hosts = set([u.domain for u in self.room_members]) + return defer.succeed(hosts) + else: + return defer.succeed([]) + datastore.get_joined_hosts_for_room = get_room_hosts + def user_rooms_intersect(userlist): room_member_ids = map(lambda u: u.to_string(), self.room_members) shared = all(map(lambda i: i in room_member_ids, userlist)) return defer.succeed(shared) - self.store.user_rooms_intersect = user_rooms_intersect + datastore.user_rooms_intersect = user_rooms_intersect + + @defer.inlineCallbacks + def setUp_users(self, hs): + # Some local users to test with + self.u_apple = hs.parse_userid("@apple:test") + self.u_banana = hs.parse_userid("@banana:test") + self.u_clementine = hs.parse_userid("@clementine:test") + + for u in self.u_apple, self.u_banana, self.u_clementine: + yield self.datastore.create_presence(u.localpart) + + yield self.datastore.set_presence_state( + self.u_apple.localpart, {"state": ONLINE, "status_msg": "Online"} + ) + + # ID of a local user that does not exist + self.u_durian = hs.parse_userid("@durian:test") + + # A remote user + self.u_cabbage = hs.parse_userid("@cabbage:elsewhere") + + +class MockedDatastoreTestCase(PresenceTestCase): + def make_datastore_mock(self): + datastore = Mock(spec=[ + # Bits that Federation needs + "prep_send_transaction", + "delivered_txn", + "get_received_txn_response", + "set_received_txn_response", + "get_destination_retry_timings", + ]) + + self.setUp_datastore_federation_mocks(datastore) + self.setUp_datastore_presence_mocks(datastore) + + return datastore + + def setUp_datastore_federation_mocks(self, datastore): + datastore.get_destination_retry_timings.return_value = ( + defer.succeed(DestinationsTable.EntryType("", 0, 0)) + ) + + def get_received_txn_response(*args): + return defer.succeed(None) + datastore.get_received_txn_response = get_received_txn_response + + def setUp_datastore_presence_mocks(self, datastore): + self.current_user_state = { + "apple": OFFLINE, + "banana": OFFLINE, + "clementine": OFFLINE, + "fig": OFFLINE, + } + + def get_presence_state(user_localpart): + return defer.succeed( + {"state": self.current_user_state[user_localpart], + "status_msg": None, + "mtime": 123456000} + ) + datastore.get_presence_state = get_presence_state + + def set_presence_state(user_localpart, new_state): + was = self.current_user_state[user_localpart] + self.current_user_state[user_localpart] = new_state["state"] + return defer.succeed({"state": was}) + datastore.set_presence_state = set_presence_state + + def get_presence_list(user_localpart, accepted): + if not user_localpart in self.PRESENCE_LIST: + return defer.succeed([]) + return defer.succeed([ + {"observed_user_id": u} for u in + self.PRESENCE_LIST[user_localpart]]) + datastore.get_presence_list = get_presence_list + + def is_presence_visible(observed_localpart, observer_userid): + return True + datastore.is_presence_visible = is_presence_visible + + @defer.inlineCallbacks + def setUp_users(self, hs): + # Some local users to test with + self.u_apple = hs.parse_userid("@apple:test") + self.u_banana = hs.parse_userid("@banana:test") + self.u_clementine = hs.parse_userid("@clementine:test") + self.u_durian = hs.parse_userid("@durian:test") + self.u_elderberry = hs.parse_userid("@elderberry:test") + self.u_fig = hs.parse_userid("@fig:test") + + # Remote user + self.u_onion = hs.parse_userid("@onion:farm") + self.u_potato = hs.parse_userid("@potato:remote") + + yield + + +class PresenceStateTestCase(PresenceTestCase): + """ Tests presence management. """ + @defer.inlineCallbacks + def setUp(self): + yield super(PresenceStateTestCase, self).setUp() self.mock_start = Mock() self.mock_stop = Mock() @@ -143,10 +276,6 @@ class PresenceTestCase(unittest.TestCase): self.handler.start_polling_presence = self.mock_start self.handler.stop_polling_presence = self.mock_stop - -class PresenceStateTestCase(PresenceTestCase): - """ Tests presence management. """ - @defer.inlineCallbacks def test_get_my_state(self): state = yield self.handler.get_state( @@ -160,7 +289,7 @@ class PresenceStateTestCase(PresenceTestCase): @defer.inlineCallbacks def test_get_allowed_state(self): - yield self.store.allow_presence_visible( + yield self.datastore.allow_presence_visible( observed_localpart=self.u_apple.localpart, observer_userid=self.u_banana.to_string(), ) @@ -208,7 +337,7 @@ class PresenceStateTestCase(PresenceTestCase): {"state": UNAVAILABLE, "status_msg": "Away", "mtime": 1000000}, - (yield self.store.get_presence_state(self.u_apple.localpart)) + (yield self.datastore.get_presence_state(self.u_apple.localpart)) ) self.mock_start.assert_called_with(self.u_apple, @@ -227,6 +356,15 @@ class PresenceStateTestCase(PresenceTestCase): class PresenceInvitesTestCase(PresenceTestCase): """ Tests presence management. """ + @defer.inlineCallbacks + def setUp(self): + yield super(PresenceInvitesTestCase, self).setUp() + + self.mock_start = Mock() + self.mock_stop = Mock() + + self.handler.start_polling_presence = self.mock_start + self.handler.stop_polling_presence = self.mock_stop @defer.inlineCallbacks def test_invite_local(self): @@ -238,10 +376,10 @@ class PresenceInvitesTestCase(PresenceTestCase): self.assertEquals( [{"observed_user_id": "@banana:test", "accepted": 1}], - (yield self.store.get_presence_list(self.u_apple.localpart)) + (yield self.datastore.get_presence_list(self.u_apple.localpart)) ) self.assertTrue( - (yield self.store.is_presence_visible( + (yield self.datastore.is_presence_visible( observed_localpart=self.u_banana.localpart, observer_userid=self.u_apple.to_string(), )) @@ -257,7 +395,7 @@ class PresenceInvitesTestCase(PresenceTestCase): self.assertEquals( [], - (yield self.store.get_presence_list(self.u_apple.localpart)) + (yield self.datastore.get_presence_list(self.u_apple.localpart)) ) @defer.inlineCallbacks @@ -282,7 +420,7 @@ class PresenceInvitesTestCase(PresenceTestCase): self.assertEquals( [{"observed_user_id": "@cabbage:elsewhere", "accepted": 0}], - (yield self.store.get_presence_list(self.u_apple.localpart)) + (yield self.datastore.get_presence_list(self.u_apple.localpart)) ) yield put_json.await_calls() @@ -317,7 +455,7 @@ class PresenceInvitesTestCase(PresenceTestCase): ) self.assertTrue( - (yield self.store.is_presence_visible( + (yield self.datastore.is_presence_visible( observed_localpart=self.u_apple.localpart, observer_userid=self.u_cabbage.to_string(), )) @@ -356,7 +494,7 @@ class PresenceInvitesTestCase(PresenceTestCase): @defer.inlineCallbacks def test_accepted_remote(self): - yield self.store.add_presence_list_pending( + yield self.datastore.add_presence_list_pending( observer_localpart=self.u_apple.localpart, observed_userid=self.u_cabbage.to_string(), ) @@ -373,7 +511,7 @@ class PresenceInvitesTestCase(PresenceTestCase): self.assertEquals( [{"observed_user_id": "@cabbage:elsewhere", "accepted": 1}], - (yield self.store.get_presence_list(self.u_apple.localpart)) + (yield self.datastore.get_presence_list(self.u_apple.localpart)) ) self.mock_start.assert_called_with( @@ -381,7 +519,7 @@ class PresenceInvitesTestCase(PresenceTestCase): @defer.inlineCallbacks def test_denied_remote(self): - yield self.store.add_presence_list_pending( + yield self.datastore.add_presence_list_pending( observer_localpart=self.u_apple.localpart, observed_userid="@eggplant:elsewhere", ) @@ -398,16 +536,16 @@ class PresenceInvitesTestCase(PresenceTestCase): self.assertEquals( [], - (yield self.store.get_presence_list(self.u_apple.localpart)) + (yield self.datastore.get_presence_list(self.u_apple.localpart)) ) @defer.inlineCallbacks def test_drop_local(self): - yield self.store.add_presence_list_pending( + yield self.datastore.add_presence_list_pending( observer_localpart=self.u_apple.localpart, observed_userid=self.u_banana.to_string(), ) - yield self.store.set_presence_list_accepted( + yield self.datastore.set_presence_list_accepted( observer_localpart=self.u_apple.localpart, observed_userid=self.u_banana.to_string(), ) @@ -419,7 +557,7 @@ class PresenceInvitesTestCase(PresenceTestCase): self.assertEquals( [], - (yield self.store.get_presence_list(self.u_apple.localpart)) + (yield self.datastore.get_presence_list(self.u_apple.localpart)) ) self.mock_stop.assert_called_with( @@ -427,11 +565,11 @@ class PresenceInvitesTestCase(PresenceTestCase): @defer.inlineCallbacks def test_drop_remote(self): - yield self.store.add_presence_list_pending( + yield self.datastore.add_presence_list_pending( observer_localpart=self.u_apple.localpart, observed_userid=self.u_cabbage.to_string(), ) - yield self.store.set_presence_list_accepted( + yield self.datastore.set_presence_list_accepted( observer_localpart=self.u_apple.localpart, observed_userid=self.u_cabbage.to_string(), ) @@ -443,16 +581,16 @@ class PresenceInvitesTestCase(PresenceTestCase): self.assertEquals( [], - (yield self.store.get_presence_list(self.u_apple.localpart)) + (yield self.datastore.get_presence_list(self.u_apple.localpart)) ) @defer.inlineCallbacks def test_get_presence_list(self): - yield self.store.add_presence_list_pending( + yield self.datastore.add_presence_list_pending( observer_localpart=self.u_apple.localpart, observed_userid=self.u_banana.to_string(), ) - yield self.store.set_presence_list_accepted( + yield self.datastore.set_presence_list_accepted( observer_localpart=self.u_apple.localpart, observed_userid=self.u_banana.to_string(), ) @@ -467,7 +605,7 @@ class PresenceInvitesTestCase(PresenceTestCase): ], presence) -class PresencePushTestCase(unittest.TestCase): +class PresencePushTestCase(MockedDatastoreTestCase): """ Tests steady-state presence status updates. They assert that presence state update messages are pushed around the place @@ -477,139 +615,9 @@ class PresencePushTestCase(unittest.TestCase): presence handler; namely the _local_pushmap and _remote_recvmap. BE WARNED... """ - def setUp(self): - self.clock = MockClock() - - self.mock_http_client = Mock(spec=[]) - self.mock_http_client.put_json = DeferredMockCallable() - - self.mock_federation_resource = MockHttpResource() - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - clock=self.clock, - db_pool=None, - datastore=Mock(spec=[ - "set_presence_state", - "get_joined_hosts_for_room", - - # Bits that Federation needs - "prep_send_transaction", - "delivered_txn", - "get_received_txn_response", - "set_received_txn_response", - "get_destination_retry_timings", - ]), - handlers=None, - resource_for_client=Mock(), - resource_for_federation=self.mock_federation_resource, - http_client=self.mock_http_client, - config=self.mock_config, - keyring=Mock(), - ) - hs.handlers = JustPresenceHandlers(hs) - - self.datastore = hs.get_datastore() - self.datastore.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) - ) - - def get_received_txn_response(*args): - return defer.succeed(None) - self.datastore.get_received_txn_response = get_received_txn_response - - self.handler = hs.get_handlers().presence_handler - self.event_source = hs.get_event_sources().sources["presence"] - - # Mock the RoomMemberHandler - hs.handlers.room_member_handler = Mock(spec=[ - "get_rooms_for_user", - "get_room_members", - ]) - self.room_member_handler = hs.handlers.room_member_handler - - self.room_id = "a-room" - self.room_members = [] - - def get_rooms_for_user(user): - if user in self.room_members: - return defer.succeed([self.room_id]) - else: - return defer.succeed([]) - self.room_member_handler.get_rooms_for_user = get_rooms_for_user - - def get_room_members(room_id): - if room_id == self.room_id: - return defer.succeed(self.room_members) - else: - return defer.succeed([]) - self.room_member_handler.get_room_members = get_room_members - - def get_room_hosts(room_id): - if room_id == self.room_id: - hosts = set([u.domain for u in self.room_members]) - return defer.succeed(hosts) - else: - return defer.succeed([]) - self.datastore.get_joined_hosts_for_room = get_room_hosts - - def user_rooms_intersect(userlist): - room_member_ids = map(lambda u: u.to_string(), self.room_members) - - shared = all(map(lambda i: i in room_member_ids, userlist)) - return defer.succeed(shared) - self.datastore.user_rooms_intersect = user_rooms_intersect - - @defer.inlineCallbacks - def fetch_room_distributions_into(room_id, localusers=None, - remotedomains=None, ignore_user=None): - - members = yield get_room_members(room_id) - for member in members: - if ignore_user is not None and member == ignore_user: - continue - - if member.is_mine: - if localusers is not None: - localusers.add(member) - else: - if remotedomains is not None: - remotedomains.add(member.domain) - self.room_member_handler.fetch_room_distributions_into = ( - fetch_room_distributions_into) - - def get_presence_list(user_localpart, accepted=None): - if user_localpart == "apple": - return defer.succeed([ - {"observed_user_id": "@banana:test"}, - {"observed_user_id": "@clementine:test"}, - ]) - else: - return defer.succeed([]) - self.datastore.get_presence_list = get_presence_list - - def is_presence_visible(observer_userid, observed_localpart): - if (observed_localpart == "clementine" and - observer_userid == "@banana:test"): - return False - return False - self.datastore.is_presence_visible = is_presence_visible - - self.distributor = hs.get_distributor() - self.distributor.declare("user_joined_room") - - # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") - self.u_durian = hs.parse_userid("@durian:test") - self.u_elderberry = hs.parse_userid("@elderberry:test") - - # Remote user - self.u_onion = hs.parse_userid("@onion:farm") - self.u_potato = hs.parse_userid("@potato:remote") + PRESENCE_LIST = { + 'apple': [ "@banana:test", "@clementine:test" ], + } @defer.inlineCallbacks def test_push_local(self): @@ -982,7 +990,7 @@ class PresencePushTestCase(unittest.TestCase): put_json.await_calls() -class PresencePollingTestCase(unittest.TestCase): +class PresencePollingTestCase(MockedDatastoreTestCase): """ Tests presence status polling. """ # For this test, we have three local users; apple is watching and is @@ -995,106 +1003,18 @@ class PresencePollingTestCase(unittest.TestCase): 'fig': [ "@potato:remote" ], } - + @defer.inlineCallbacks def setUp(self): - self.mock_http_client = Mock(spec=[]) - self.mock_http_client.put_json = DeferredMockCallable() - - self.mock_federation_resource = MockHttpResource() - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - clock=MockClock(), - db_pool=None, - datastore=Mock(spec=[ - # Bits that Federation needs - "prep_send_transaction", - "delivered_txn", - "get_received_txn_response", - "set_received_txn_response", - "get_destination_retry_timings", - ]), - handlers=None, - resource_for_client=Mock(), - resource_for_federation=self.mock_federation_resource, - http_client=self.mock_http_client, - config=self.mock_config, - keyring=Mock(), - ) - hs.handlers = JustPresenceHandlers(hs) - - self.datastore = hs.get_datastore() - self.datastore.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) - ) - - def get_received_txn_response(*args): - return defer.succeed(None) - self.datastore.get_received_txn_response = get_received_txn_response + yield super(PresencePollingTestCase, self).setUp() self.mock_update_client = Mock() def update(*args,**kwargs): - # print "mock_update_client: Args=%s, kwargs=%s" %(args, kwargs,) return defer.succeed(None) - self.mock_update_client.side_effect = update - self.handler = hs.get_handlers().presence_handler self.handler.push_update_to_clients = self.mock_update_client - hs.handlers.room_member_handler = Mock(spec=[ - "get_rooms_for_user", - ]) - # For this test no users are ever in rooms - def get_rooms_for_user(user): - return defer.succeed([]) - hs.handlers.room_member_handler.get_rooms_for_user = get_rooms_for_user - - # Mocked database state - # Local users always start offline - self.current_user_state = { - "apple": OFFLINE, - "banana": OFFLINE, - "clementine": OFFLINE, - "fig": OFFLINE, - } - - def get_presence_state(user_localpart): - return defer.succeed( - {"state": self.current_user_state[user_localpart], - "status_msg": None, - "mtime": 123456000} - ) - self.datastore.get_presence_state = get_presence_state - - def set_presence_state(user_localpart, new_state): - was = self.current_user_state[user_localpart] - self.current_user_state[user_localpart] = new_state["state"] - return defer.succeed({"state": was}) - self.datastore.set_presence_state = set_presence_state - - def get_presence_list(user_localpart, accepted): - return defer.succeed([ - {"observed_user_id": u} for u in - self.PRESENCE_LIST[user_localpart]]) - self.datastore.get_presence_list = get_presence_list - - def is_presence_visible(observed_localpart, observer_userid): - return True - self.datastore.is_presence_visible = is_presence_visible - - # Local users - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") - self.u_fig = hs.parse_userid("@fig:test") - - # Remote users - self.u_potato = hs.parse_userid("@potato:remote") - @defer.inlineCallbacks def test_push_local(self): # apple goes online From 3a243c53f41a719c4de62630db6018f97d5b94ae Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 22 Jan 2015 20:06:08 +0000 Subject: [PATCH 036/630] Rename MockedDatastoreTestCase to MockedDatastorePresenceTestCase since it is still presence-specific --- tests/handlers/test_presence.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 805f3868b..56e90177f 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -188,7 +188,7 @@ class PresenceTestCase(unittest.TestCase): self.u_cabbage = hs.parse_userid("@cabbage:elsewhere") -class MockedDatastoreTestCase(PresenceTestCase): +class MockedDatastorePresenceTestCase(PresenceTestCase): def make_datastore_mock(self): datastore = Mock(spec=[ # Bits that Federation needs @@ -605,7 +605,7 @@ class PresenceInvitesTestCase(PresenceTestCase): ], presence) -class PresencePushTestCase(MockedDatastoreTestCase): +class PresencePushTestCase(MockedDatastorePresenceTestCase): """ Tests steady-state presence status updates. They assert that presence state update messages are pushed around the place @@ -990,7 +990,7 @@ class PresencePushTestCase(MockedDatastoreTestCase): put_json.await_calls() -class PresencePollingTestCase(MockedDatastoreTestCase): +class PresencePollingTestCase(MockedDatastorePresenceTestCase): """ Tests presence status polling. """ # For this test, we have three local users; apple is watching and is From 6927b6b19783d7134eba3461c4fafe4efdec40f1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:21:47 +0000 Subject: [PATCH 037/630] This really serves me right for ever making a map called 'map'. --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2803c1f07..7df3fc7f0 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -119,7 +119,7 @@ class PushRuleRestServlet(RestServlet): def priority_class_from_spec(self, spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) - pc = map[spec['template']] + pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] if spec['scope'] == 'device': pc += 5 From bcd48b9636071543fa64e7fb066275d1c9c1e363 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:28:25 +0000 Subject: [PATCH 038/630] Fix adding rules without before/after & add the rule that we couldn't find to the error --- synapse/rest/client/v1/push_rule.py | 4 ++-- synapse/storage/push_rule.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 7df3fc7f0..77a077247 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -166,8 +166,8 @@ class PushRuleRestServlet(RestServlet): ) except InconsistentRuleException as e: raise SynapseError(400, e.message) - except RuleNotFoundException: - raise SynapseError(400, "before/after rule not found") + except RuleNotFoundException as e: + raise SynapseError(400, e.message) defer.returnValue((200, {})) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index d087257ff..2366090e0 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -46,7 +46,7 @@ class PushRuleStore(SQLBaseStore): defer.returnValue(dicts) @defer.inlineCallbacks - def add_push_rule(self, **kwargs): + def add_push_rule(self, before, after, **kwargs): vals = copy.copy(kwargs) if 'conditions' in vals: vals['conditions'] = json.dumps(vals['conditions']) @@ -57,10 +57,12 @@ class PushRuleStore(SQLBaseStore): if 'id' in vals: del vals['id'] - if 'after' in kwargs or 'before' in kwargs: + if before or after: ret = yield self.runInteraction( "_add_push_rule_relative_txn", self._add_push_rule_relative_txn, + before=before, + after=after, **vals ) defer.returnValue(ret) @@ -89,7 +91,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() if not res: - raise RuleNotFoundException() + raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) (priority_class, base_rule_priority) = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: From f87586e661101849a90f9d106b207a529e4cf689 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 10:32:40 +0000 Subject: [PATCH 039/630] right super() param --- synapse/api/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 01207282d..4f59e1742 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -100,7 +100,7 @@ class NotFoundError(SynapseError): def __init__(self, *args, **kwargs): if "errcode" not in kwargs: kwargs["errcode"] = Codes.NOT_FOUND - super(UnrecognizedRequestError, self).__init__( + super(NotFoundError, self).__init__( 404, "Not found", **kwargs From 7256def8e43bf5ab982cb7e785fb1334a1ef4ab8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 10:37:38 +0000 Subject: [PATCH 040/630] Merge rest servlets into the client json resource object --- synapse/app/homeserver.py | 5 ++--- synapse/rest/client/v1/__init__.py | 18 +++++++----------- synapse/server.py | 10 ---------- tests/rest/client/v1/test_presence.py | 9 ++++++--- tests/rest/client/v1/test_profile.py | 4 +++- 5 files changed, 18 insertions(+), 28 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index cd24bbdc7..fabe8ddac 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -37,6 +37,7 @@ from synapse.api.urls import ( from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext +from synapse.rest.client.v1 import ClientV1RestResource from daemonize import Daemonize import twisted.manhole.telnet @@ -59,7 +60,7 @@ class SynapseHomeServer(HomeServer): return MatrixFederationHttpClient(self) def build_resource_for_client(self): - return JsonResource() + return ClientV1RestResource(self) def build_resource_for_federation(self): return JsonResource() @@ -224,8 +225,6 @@ def setup(): content_addr=config.content_addr, ) - hs.register_servlets() - hs.create_resource_tree( web_client=config.webclient, redirect_root_to_web_client=True, diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 88ec9cd27..8bb89b2f6 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -19,22 +19,18 @@ from . import ( voip, admin, ) +from synapse.http.server import JsonResource -class RestServletFactory(object): - """ A factory for creating REST servlets. - - These REST servlets represent the entire client-server REST API. Generally - speaking, they serve as wrappers around events and the handlers that - process them. - - See synapse.events for information on synapse events. - """ +class ClientV1RestResource(JsonResource): + """A resource for version 1 of the matrix client API.""" def __init__(self, hs): - client_resource = hs.get_resource_for_client() + JsonResource.__init__(self) + self.register_servlets(self, hs) - # TODO(erikj): There *must* be a better way of doing this. + @staticmethod + def register_servlets(client_resource, hs): room.register_servlets(hs, client_resource) events.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) diff --git a/synapse/server.py b/synapse/server.py index e9add8e2b..476d80937 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -24,7 +24,6 @@ from synapse.events.utils import serialize_event from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers -from synapse.rest.client.v1 import RestServletFactory from synapse.state import StateHandler from synapse.storage import DataStore from synapse.types import UserID, RoomAlias, RoomID, EventID @@ -203,9 +202,6 @@ class HomeServer(BaseHomeServer): def build_auth(self): return Auth(self) - def build_rest_servlet_factory(self): - return RestServletFactory(self) - def build_state_handler(self): return StateHandler(self) @@ -229,9 +225,3 @@ class HomeServer(BaseHomeServer): clock=self.get_clock(), hostname=self.hostname, ) - - def register_servlets(self): - """ Register all servlets associated with this HomeServer. - """ - # Simply building the ServletFactory is sufficient to have it register - self.get_rest_servlet_factory() diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 0b6f7cfcc..783720ac2 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -25,6 +25,8 @@ from ....utils import MockHttpResource, MockKey from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler from synapse.server import HomeServer +from synapse.rest.client.v1 import presence +from synapse.rest.client.v1 import events OFFLINE = PresenceState.OFFLINE @@ -86,7 +88,7 @@ class PresenceStateTestCase(unittest.TestCase): return defer.succeed([]) room_member_handler.get_rooms_for_user = get_rooms_for_user - hs.register_servlets() + presence.register_servlets(hs, self.mock_resource) self.u_apple = hs.parse_userid(myid) @@ -172,7 +174,7 @@ class PresenceListTestCase(unittest.TestCase): hs.get_auth().get_user_by_token = _get_user_by_token - hs.register_servlets() + presence.register_servlets(hs, self.mock_resource) self.u_apple = hs.parse_userid("@apple:test") self.u_banana = hs.parse_userid("@banana:test") @@ -283,7 +285,8 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_auth().get_user_by_req = _get_user_by_req - hs.register_servlets() + presence.register_servlets(hs, self.mock_resource) + events.register_servlets(hs, self.mock_resource) hs.handlers.room_member_handler = Mock(spec=[]) diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 47cfb10a6..5b5c3edc2 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -25,6 +25,8 @@ from ....utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer +from synapse.rest.client.v1 import profile + myid = "@1234ABCD:test" PATH_PREFIX = "/_matrix/client/api/v1" @@ -61,7 +63,7 @@ class ProfileTestCase(unittest.TestCase): hs.get_handlers().profile_handler = self.mock_handler - hs.register_servlets() + profile.register_servlets(hs, self.mock_resource) @defer.inlineCallbacks def test_get_my_name(self): From 49fe31792bc0cf709248e592baefb8f34606236a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 11:19:02 +0000 Subject: [PATCH 041/630] Add slightly pedantic trailing slash error. --- synapse/api/errors.py | 7 ++++++- synapse/rest/client/v1/push_rule.py | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 4f59e1742..5872e82d0 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -88,9 +88,14 @@ class UnrecognizedRequestError(SynapseError): def __init__(self, *args, **kwargs): if "errcode" not in kwargs: kwargs["errcode"] = Codes.UNRECOGNIZED + message = None + if len(args) == 0: + message = "Unrecognized request" + else: + message = args[0] super(UnrecognizedRequestError, self).__init__( 400, - "Unrecognized request", + message, **kwargs ) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 77a077247..6f108431b 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -32,6 +32,8 @@ class PushRuleRestServlet(RestServlet): 'override': 4 } PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} + SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR =\ + "Unrecognised request: You probably wanted a trailing slash" def rule_spec_from_path(self, path): if len(path) < 2: @@ -211,10 +213,14 @@ class PushRuleRestServlet(RestServlet): rulearray.append(template_rule) path = request.postpath[1:] - if path == []: - defer.returnValue((200, rules)) - if path[0] == 'global': + if path == []: + # we're a reference impl: pedantry is our job. + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + + if path[0] == '': + defer.returnValue((200, rules)) + elif path[0] == 'global': path = path[1:] result = _filter_ruleset_with_path(rules['global'], path) defer.returnValue((200, result)) @@ -255,12 +261,17 @@ def _instance_handle_from_conditions(conditions): def _filter_ruleset_with_path(ruleset, path): if path == []: + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + + if path[0] == '': return ruleset template_kind = path[0] if template_kind not in ruleset: raise UnrecognizedRequestError() path = path[1:] if path == []: + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + if path[0] == '': return ruleset[template_kind] rule_id = path[0] for r in ruleset[template_kind]: From 5759bec43cb52862a8d455afb8cd9d1c5660bc3d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 11:47:15 +0000 Subject: [PATCH 042/630] Replace hs.parse_userid with UserID.from_string --- synapse/api/auth.py | 9 +++++---- synapse/handlers/_base.py | 5 +++-- synapse/handlers/events.py | 3 ++- synapse/handlers/federation.py | 13 ++++++------ synapse/handlers/message.py | 13 ++++++------ synapse/handlers/presence.py | 23 +++++++++++---------- synapse/handlers/profile.py | 3 ++- synapse/handlers/room.py | 14 ++++++------- synapse/handlers/typing.py | 3 ++- synapse/rest/client/v1/admin.py | 4 +++- synapse/rest/client/v1/presence.py | 15 +++++++------- synapse/rest/client/v1/profile.py | 13 ++++++------ synapse/rest/client/v1/room.py | 5 +++-- synapse/server.py | 6 ------ synapse/storage/roommember.py | 5 +++-- tests/handlers/test_presence.py | 29 ++++++++++++++------------- tests/handlers/test_presencelike.py | 9 +++++---- tests/handlers/test_profile.py | 8 ++++---- tests/handlers/test_room.py | 9 +++++---- tests/handlers/test_typing.py | 7 ++++--- tests/rest/client/v1/test_presence.py | 19 +++++++++--------- tests/rest/client/v1/test_profile.py | 3 ++- tests/rest/client/v1/test_rooms.py | 21 +++++++++---------- tests/rest/client/v1/test_typing.py | 5 +++-- tests/storage/test_presence.py | 5 +++-- tests/storage/test_profile.py | 3 ++- tests/storage/test_redaction.py | 5 +++-- tests/storage/test_room.py | 3 ++- tests/storage/test_roommember.py | 7 ++++--- tests/storage/test_stream.py | 5 +++-- tests/test_types.py | 6 ------ 31 files changed, 145 insertions(+), 133 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e31482cfa..a342a0e0d 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,6 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.types import UserID import logging @@ -104,7 +105,7 @@ class Auth(object): for event in curr_state: if event.type == EventTypes.Member: try: - if self.hs.parse_userid(event.state_key).domain != host: + if UserID.from_string(event.state_key).domain != host: continue except: logger.warn("state_key not user_id: %s", event.state_key) @@ -337,7 +338,7 @@ class Auth(object): user_info = { "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), - "user": self.hs.parse_userid(ret.get("name")), + "user": UserID.from_string(ret.get("name")), } defer.returnValue(user_info) @@ -461,7 +462,7 @@ class Auth(object): "You are not allowed to set others state" ) else: - sender_domain = self.hs.parse_userid( + sender_domain = UserID.from_string( event.user_id ).domain @@ -496,7 +497,7 @@ class Auth(object): # Validate users for k, v in user_list.items(): try: - self.hs.parse_userid(k) + UserID.from_string(k) except: raise SynapseError(400, "Not a valid user_id: %s" % (k,)) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index f33d17a31..1773fa20a 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -19,6 +19,7 @@ from synapse.api.errors import LimitExceededError, SynapseError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes +from synapse.types import UserID import logging @@ -113,7 +114,7 @@ class BaseHandler(object): if event.type == EventTypes.Member: if event.content["membership"] == Membership.INVITE: - invitee = self.hs.parse_userid(event.state_key) + invitee = UserID.from_string(event.state_key) if not self.hs.is_mine(invitee): # TODO: Can we add signature from remote server in a nicer # way? If we have been invited by a remote server, we need @@ -134,7 +135,7 @@ class BaseHandler(object): if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: destinations.add( - self.hs.parse_userid(s.state_key).domain + UserID.from_string(s.state_key).domain ) except SynapseError: logger.warn( diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 103bc67c4..01e67b081 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function +from synapse.types import UserID from ._base import BaseHandler @@ -48,7 +49,7 @@ class EventStreamHandler(BaseHandler): @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, as_client_event=True): - auth_user = self.hs.parse_userid(auth_user_id) + auth_user = UserID.from_string(auth_user_id) try: if auth_user not in self._streams_per_user: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 81203bf1a..bcdcc90a1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -28,6 +28,7 @@ from synapse.crypto.event_signing import ( compute_event_signature, check_event_content_hash, add_hashes_and_signatures, ) +from synapse.types import UserID from syutil.jsonutil import encode_canonical_json from twisted.internet import defer @@ -227,7 +228,7 @@ class FederationHandler(BaseHandler): extra_users = [] if event.type == EventTypes.Member: target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) + target_user = UserID.from_string(target_user_id) extra_users.append(target_user) yield self.notifier.on_new_room_event( @@ -236,7 +237,7 @@ class FederationHandler(BaseHandler): if event.type == EventTypes.Member: if event.membership == Membership.JOIN: - user = self.hs.parse_userid(event.state_key) + user = UserID.from_string(event.state_key) yield self.distributor.fire( "user_joined_room", user=user, room_id=event.room_id ) @@ -491,7 +492,7 @@ class FederationHandler(BaseHandler): extra_users = [] if event.type == EventTypes.Member: target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) + target_user = UserID.from_string(target_user_id) extra_users.append(target_user) yield self.notifier.on_new_room_event( @@ -500,7 +501,7 @@ class FederationHandler(BaseHandler): if event.type == EventTypes.Member: if event.content["membership"] == Membership.JOIN: - user = self.hs.parse_userid(event.state_key) + user = UserID.from_string(event.state_key) yield self.distributor.fire( "user_joined_room", user=user, room_id=event.room_id ) @@ -514,7 +515,7 @@ class FederationHandler(BaseHandler): if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: destinations.add( - self.hs.parse_userid(s.state_key).domain + UserID.from_string(s.state_key).domain ) except: logger.warn( @@ -565,7 +566,7 @@ class FederationHandler(BaseHandler): backfilled=False, ) - target_user = self.hs.parse_userid(event.state_key) + target_user = UserID.from_string(event.state_key) yield self.notifier.on_new_room_event( event, extra_users=[target_user], ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index f2a2f1693..6a1104a89 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -20,6 +20,7 @@ from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig from synapse.events.validator import EventValidator from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -89,7 +90,7 @@ class MessageHandler(BaseHandler): yield self.hs.get_event_sources().get_current_token() ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( user, pagin_config.get_source_config("room"), room_id @@ -130,13 +131,13 @@ class MessageHandler(BaseHandler): if ratelimit: self.ratelimit(builder.user_id) # TODO(paul): Why does 'event' not have a 'user' object? - user = self.hs.parse_userid(builder.user_id) + user = UserID.from_string(builder.user_id) assert self.hs.is_mine(user), "User must be our own: %s" % (user,) if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: - joinee = self.hs.parse_userid(builder.state_key) + joinee = UserID.from_string(builder.state_key) # If event doesn't include a display name, add one. yield self.distributor.fire( "collect_presencelike_data", @@ -237,7 +238,7 @@ class MessageHandler(BaseHandler): membership_list=[Membership.INVITE, Membership.JOIN] ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) rooms_ret = [] @@ -316,7 +317,7 @@ class MessageHandler(BaseHandler): # TODO(paul): I wish I was called with user objects not user_id # strings... - auth_user = self.hs.parse_userid(user_id) + auth_user = UserID.from_string(user_id) # TODO: These concurrently state_tuples = yield self.state_handler.get_current_state(room_id) @@ -349,7 +350,7 @@ class MessageHandler(BaseHandler): for m in room_members: try: member_presence = yield presence_handler.get_state( - target_user=self.hs.parse_userid(m.user_id), + target_user=UserID.from_string(m.user_id), auth_user=auth_user, as_event=True, ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8aeed9927..d66bfea7b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -20,6 +20,7 @@ from synapse.api.constants import PresenceState from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -96,22 +97,22 @@ class PresenceHandler(BaseHandler): self.federation.register_edu_handler( "m.presence_invite", lambda origin, content: self.invite_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) self.federation.register_edu_handler( "m.presence_accept", lambda origin, content: self.accept_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) self.federation.register_edu_handler( "m.presence_deny", lambda origin, content: self.deny_presence( - observed_user=hs.parse_userid(content["observed_user"]), - observer_user=hs.parse_userid(content["observer_user"]), + observed_user=UserID.from_string(content["observed_user"]), + observer_user=UserID.from_string(content["observer_user"]), ) ) @@ -418,7 +419,7 @@ class PresenceHandler(BaseHandler): ) for p in presence: - observed_user = self.hs.parse_userid(p.pop("observed_user_id")) + observed_user = UserID.from_string(p.pop("observed_user_id")) p["observed_user"] = observed_user p.update(self._get_or_offline_usercache(observed_user).get_state()) if "last_active" in p: @@ -441,7 +442,7 @@ class PresenceHandler(BaseHandler): user.localpart, accepted=True ) target_users = set([ - self.hs.parse_userid(x["observed_user_id"]) for x in presence + UserID.from_string(x["observed_user_id"]) for x in presence ]) # Also include people in all my rooms @@ -646,7 +647,7 @@ class PresenceHandler(BaseHandler): deferreds = [] for push in content.get("push", []): - user = self.hs.parse_userid(push["user_id"]) + user = UserID.from_string(push["user_id"]) logger.debug("Incoming presence update from %s", user) @@ -694,7 +695,7 @@ class PresenceHandler(BaseHandler): del self._user_cachemap[user] for poll in content.get("poll", []): - user = self.hs.parse_userid(poll) + user = UserID.from_string(poll) if not self.hs.is_mine(user): continue @@ -709,7 +710,7 @@ class PresenceHandler(BaseHandler): deferreds.append(self._push_presence_remote(user, origin)) for unpoll in content.get("unpoll", []): - user = self.hs.parse_userid(unpoll) + user = UserID.from_string(unpoll) if not self.hs.is_mine(user): continue diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 7777d3cc9..03b2159c5 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException from synapse.api.constants import EventTypes, Membership from synapse.util.logcontext import PreserveLoggingContext +from synapse.types import UserID from ._base import BaseHandler @@ -169,7 +170,7 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def on_profile_query(self, args): - user = self.hs.parse_userid(args["user_id"]) + user = UserID.from_string(args["user_id"]) if not self.hs.is_mine(user): raise SynapseError(400, "User is not hosted on this Home Server") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6d0db18e5..0242288c4 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -64,7 +64,7 @@ class RoomCreationHandler(BaseHandler): invite_list = config.get("invite", []) for i in invite_list: try: - self.hs.parse_userid(i) + UserID.from_string(i) except: raise SynapseError(400, "Invalid user_id: %s" % (i,)) @@ -114,7 +114,7 @@ class RoomCreationHandler(BaseHandler): servers=[self.hs.hostname], ) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) creation_events = self._create_events_for_new_room( user, room_id, is_public=is_public ) @@ -250,7 +250,7 @@ class RoomMemberHandler(BaseHandler): users = yield self.store.get_users_in_room(room_id) - defer.returnValue([hs.parse_userid(u) for u in users]) + defer.returnValue([UserID.from_string(u) for u in users]) @defer.inlineCallbacks def fetch_room_distributions_into(self, room_id, localusers=None, @@ -368,7 +368,7 @@ class RoomMemberHandler(BaseHandler): ) if prev_state and prev_state.membership == Membership.JOIN: - user = self.hs.parse_userid(event.user_id) + user = UserID.from_string(event.user_id) self.distributor.fire( "user_left_room", user=user, room_id=event.room_id ) @@ -412,7 +412,7 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _do_join(self, event, context, room_host=None, do_auth=True): - joinee = self.hs.parse_userid(event.state_key) + joinee = UserID.from_string(event.state_key) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -476,7 +476,7 @@ class RoomMemberHandler(BaseHandler): do_auth=do_auth, ) - user = self.hs.parse_userid(event.user_id) + user = UserID.from_string(event.user_id) yield self.distributor.fire( "user_joined_room", user=user, room_id=room_id ) @@ -526,7 +526,7 @@ class RoomMemberHandler(BaseHandler): do_auth): yield run_on_reactor() - target_user = self.hs.parse_userid(event.state_key) + target_user = UserID.from_string(event.state_key) yield self.handle_new_client_event( event, diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index cd9638dd0..c69787005 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError, AuthError +from synapse.types import UserID import logging @@ -185,7 +186,7 @@ class TypingNotificationHandler(BaseHandler): @defer.inlineCallbacks def _recv_edu(self, origin, content): room_id = content["room_id"] - user = self.homeserver.parse_userid(content["user_id"]) + user = UserID.from_string(content["user_id"]) localusers = set() diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 0aa83514c..4aefb9405 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -16,6 +16,8 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError +from synapse.types import UserID + from base import RestServlet, client_path_pattern import logging @@ -28,7 +30,7 @@ class WhoisRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - target_user = self.hs.parse_userid(user_id) + target_user = UserID.from_string(user_id) auth_user = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index ca4d2d21f..22fcb7d7d 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -18,7 +18,8 @@ from twisted.internet import defer from synapse.api.errors import SynapseError -from base import RestServlet, client_path_pattern +from synapse.types import UserID +from .base import RestServlet, client_path_pattern import json import logging @@ -32,7 +33,7 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( target_user=user, auth_user=auth_user) @@ -42,7 +43,7 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) state = {} try: @@ -77,7 +78,7 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") @@ -97,7 +98,7 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") @@ -118,7 +119,7 @@ class PresenceListRestServlet(RestServlet): raise SynapseError(400, "Bad invite value.") if len(u) == 0: continue - invited_user = self.hs.parse_userid(u) + invited_user = UserID.from_string(u) yield self.handlers.presence_handler.send_invite( observer_user=user, observed_user=invited_user ) @@ -129,7 +130,7 @@ class PresenceListRestServlet(RestServlet): raise SynapseError(400, "Bad drop value.") if len(u) == 0: continue - dropped_user = self.hs.parse_userid(u) + dropped_user = UserID.from_string(u) yield self.handlers.presence_handler.drop( observer_user=user, observed_user=dropped_user ) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index dc6eb424b..39297930c 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -16,7 +16,8 @@ """ This module contains REST servlets to do with profile: /profile/ """ from twisted.internet import defer -from base import RestServlet, client_path_pattern +from .base import RestServlet, client_path_pattern +from synapse.types import UserID import json @@ -26,7 +27,7 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) displayname = yield self.handlers.profile_handler.get_displayname( user, @@ -37,7 +38,7 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) try: content = json.loads(request.content.read()) @@ -59,7 +60,7 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) avatar_url = yield self.handlers.profile_handler.get_avatar_url( user, @@ -70,7 +71,7 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) try: content = json.loads(request.content.read()) @@ -92,7 +93,7 @@ class ProfileRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) displayname = yield self.handlers.profile_handler.get_displayname( user, diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 48bba2a5f..c5837b340 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,6 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID import json import logging @@ -289,7 +290,7 @@ class RoomMemberListRestServlet(RestServlet): for event in members["chunk"]: # FIXME: should probably be state_key here, not user_id - target_user = self.hs.parse_userid(event["user_id"]) + target_user = UserID.from_string(event["user_id"]) # Presence is an optional cache; don't fail if we can't fetch it try: presence_handler = self.handlers.presence_handler @@ -478,7 +479,7 @@ class RoomTypingRestServlet(RestServlet): auth_user = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) - target_user = self.hs.parse_userid(urllib.unquote(user_id)) + target_user = UserID.from_string(urllib.unquote(user_id)) content = _parse_json(request) diff --git a/synapse/server.py b/synapse/server.py index 476d80937..52a21aaf7 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -127,12 +127,6 @@ class BaseHomeServer(object): # TODO: Why are these parse_ methods so high up along with other globals? # Surely these should be in a util package or in the api package? - # Other utility methods - def parse_userid(self, s): - """Parse the string given by 's' as a User ID and return a UserID - object.""" - return UserID.from_string(s) - def parse_roomalias(self, s): """Parse the string given by 's' as a Room Alias and return a RoomAlias object.""" diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index e59e65529..c69dd995c 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -20,6 +20,7 @@ from collections import namedtuple from ._base import SQLBaseStore from synapse.api.constants import Membership +from synapse.types import UserID import logging @@ -39,7 +40,7 @@ class RoomMemberStore(SQLBaseStore): """ try: target_user_id = event.state_key - domain = self.hs.parse_userid(target_user_id).domain + domain = UserID.from_string(target_user_id).domain except: logger.exception( "Failed to parse target_user_id=%s", target_user_id @@ -84,7 +85,7 @@ class RoomMemberStore(SQLBaseStore): for e in member_events: try: joined_domains.add( - self.hs.parse_userid(e.state_key).domain + UserID.from_string(e.state_key).domain ) except: # FIXME: How do we deal with invalid user ids in the db? diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 56e90177f..5621a8afa 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -17,7 +17,7 @@ from tests import unittest from twisted.internet import defer, reactor -from mock import Mock, call, ANY, NonCallableMock, patch +from mock import Mock, call, ANY, NonCallableMock import json from tests.utils import ( @@ -31,6 +31,7 @@ from synapse.api.errors import SynapseError from synapse.handlers.presence import PresenceHandler, UserPresenceCache from synapse.streams.config import SourcePaginationConfig from synapse.storage.transactions import DestinationsTable +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE UNAVAILABLE = PresenceState.UNAVAILABLE @@ -170,9 +171,9 @@ class PresenceTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp_users(self, hs): # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") for u in self.u_apple, self.u_banana, self.u_clementine: yield self.datastore.create_presence(u.localpart) @@ -182,10 +183,10 @@ class PresenceTestCase(unittest.TestCase): ) # ID of a local user that does not exist - self.u_durian = hs.parse_userid("@durian:test") + self.u_durian = UserID.from_string("@durian:test") # A remote user - self.u_cabbage = hs.parse_userid("@cabbage:elsewhere") + self.u_cabbage = UserID.from_string("@cabbage:elsewhere") class MockedDatastorePresenceTestCase(PresenceTestCase): @@ -250,16 +251,16 @@ class MockedDatastorePresenceTestCase(PresenceTestCase): @defer.inlineCallbacks def setUp_users(self, hs): # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") - self.u_durian = hs.parse_userid("@durian:test") - self.u_elderberry = hs.parse_userid("@elderberry:test") - self.u_fig = hs.parse_userid("@fig:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") + self.u_durian = UserID.from_string("@durian:test") + self.u_elderberry = UserID.from_string("@elderberry:test") + self.u_fig = UserID.from_string("@fig:test") # Remote user - self.u_onion = hs.parse_userid("@onion:farm") - self.u_potato = hs.parse_userid("@potato:remote") + self.u_onion = UserID.from_string("@onion:farm") + self.u_potato = UserID.from_string("@potato:remote") yield diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 0584e4c8b..3cdbb186a 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -27,6 +27,7 @@ from synapse.server import HomeServer from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler from synapse.handlers.profile import ProfileHandler +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE @@ -136,12 +137,12 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): lambda u: defer.succeed([])) # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") - self.u_clementine = hs.parse_userid("@clementine:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") + self.u_clementine = UserID.from_string("@clementine:test") # Remote user - self.u_potato = hs.parse_userid("@potato:remote") + self.u_potato = UserID.from_string("@potato:remote") self.mock_get_joined = ( self.datastore.get_rooms_for_user_where_membership_is diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 25b172aa5..7b9590c11 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -22,7 +22,7 @@ from mock import Mock, NonCallableMock from synapse.api.errors import AuthError from synapse.server import HomeServer from synapse.handlers.profile import ProfileHandler -from synapse.api.constants import Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -71,9 +71,9 @@ class ProfileTestCase(unittest.TestCase): self.store = hs.get_datastore() - self.frank = hs.parse_userid("@1234ABCD:test") - self.bob = hs.parse_userid("@4567:test") - self.alice = hs.parse_userid("@alice:remote") + self.frank = UserID.from_string("@1234ABCD:test") + self.bob = UserID.from_string("@4567:test") + self.alice = UserID.from_string("@alice:remote") yield self.store.create_profile(self.frank.localpart) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index d3253b48b..9a23b3812 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -15,12 +15,13 @@ from twisted.internet import defer -from tests import unittest +from .. import unittest from synapse.api.constants import EventTypes, Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler from synapse.handlers.profile import ProfileHandler from synapse.server import HomeServer +from synapse.types import UserID from ..utils import MockKey from mock import Mock, NonCallableMock @@ -164,7 +165,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event, context=context, ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[self.hs.parse_userid(target_user_id)] + event, extra_users=[UserID.from_string(target_user_id)] ) self.assertFalse(self.datastore.get_room.called) self.assertFalse(self.datastore.store_room.called) @@ -174,7 +175,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def test_simple_join(self): room_id = "!foo:red" user_id = "@bob:red" - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) join_signal_observer = Mock() self.distributor.observe("user_joined_room", join_signal_observer) @@ -252,7 +253,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def test_simple_leave(self): room_id = "!foo:red" user_id = "@bob:red" - user = self.hs.parse_userid(user_id) + user = UserID.from_string(user_id) builder = self.hs.get_event_builder_factory().new({ "type": EventTypes.Member, diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 6a498b23a..8a7fc028d 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -27,6 +27,7 @@ from synapse.server import HomeServer from synapse.handlers.typing import TypingNotificationHandler from synapse.storage.transactions import DestinationsTable +from synapse.types import UserID def _expect_edu(destination, edu_type, content, origin="test"): @@ -153,11 +154,11 @@ class TypingNotificationsTestCase(unittest.TestCase): self.auth.check_joined_room = check_joined_room # Some local users to test with - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") # Remote user - self.u_onion = hs.parse_userid("@onion:farm") + self.u_onion = UserID.from_string("@onion:farm") @defer.inlineCallbacks def test_started_typing_local(self): diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 783720ac2..65d5cc491 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -27,6 +27,7 @@ from synapse.handlers.presence import PresenceHandler from synapse.server import HomeServer from synapse.rest.client.v1 import presence from synapse.rest.client.v1 import events +from synapse.types import UserID OFFLINE = PresenceState.OFFLINE @@ -71,7 +72,7 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(myid), + "user": UserID.from_string(myid), "admin": False, "device_id": None, } @@ -90,7 +91,7 @@ class PresenceStateTestCase(unittest.TestCase): presence.register_servlets(hs, self.mock_resource) - self.u_apple = hs.parse_userid(myid) + self.u_apple = UserID.from_string(myid) @defer.inlineCallbacks def test_get_my_status(self): @@ -161,12 +162,12 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(myid), + "user": UserID.from_string(myid), "admin": False, "device_id": None, } - room_member_handler = hs.handlers.room_member_handler = Mock( + hs.handlers.room_member_handler = Mock( spec=[ "get_rooms_for_user", ] @@ -176,8 +177,8 @@ class PresenceListTestCase(unittest.TestCase): presence.register_servlets(hs, self.mock_resource) - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_get_my_list(self): @@ -281,7 +282,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_clock().time_msec.return_value = 1000000 def _get_user_by_req(req=None): - return hs.parse_userid(myid) + return UserID.from_string(myid) hs.get_auth().get_user_by_req = _get_user_by_req @@ -322,8 +323,8 @@ class PresenceEventStreamTestCase(unittest.TestCase): self.presence = hs.get_handlers().presence_handler - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_shortpoll(self): diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 5b5c3edc2..39cd68d82 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -24,6 +24,7 @@ from ....utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer +from synapse.types import UserID from synapse.rest.client.v1 import profile @@ -57,7 +58,7 @@ class ProfileTestCase(unittest.TestCase): ) def _get_user_by_req(request=None): - return hs.parse_userid(myid) + return UserID.from_string(myid) hs.get_auth().get_user_by_req = _get_user_by_req diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 12f804054..76ed550b7 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -22,13 +22,10 @@ import synapse.rest.client.v1.room from synapse.api.constants import Membership from synapse.server import HomeServer +from synapse.types import UserID -from tests import unittest - -# python imports import json import urllib -import types from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase @@ -70,7 +67,7 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -466,7 +463,7 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -555,7 +552,7 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -657,7 +654,7 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -773,7 +770,7 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -909,7 +906,7 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -1013,7 +1010,7 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -1028,7 +1025,7 @@ class RoomInitialSyncTestCase(RestTestCase): # Since I'm getting my own presence I need to exist as far as presence # is concerned. hs.get_handlers().presence_handler.registered_user( - hs.parse_userid(self.user_id) + UserID.from_string(self.user_id) ) # create the room diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 647bcebfd..c89b37d00 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -20,6 +20,7 @@ from twisted.internet import defer import synapse.rest.client.v1.room from synapse.server import HomeServer +from synapse.types import UserID from ....utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase @@ -69,7 +70,7 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_token(token=None): return { - "user": hs.parse_userid(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, } @@ -82,7 +83,7 @@ class RoomTypingTestCase(RestTestCase): def get_room_members(room_id): if room_id == self.room_id: - return defer.succeed([hs.parse_userid(self.user_id)]) + return defer.succeed([UserID.from_string(self.user_id)]) else: return defer.succeed([]) diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py index 9655d3cf4..1ab193736 100644 --- a/tests/storage/test_presence.py +++ b/tests/storage/test_presence.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.presence import PresenceStore +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockClock @@ -37,8 +38,8 @@ class PresenceStoreTestCase(unittest.TestCase): self.store = PresenceStore(hs) - self.u_apple = hs.parse_userid("@apple:test") - self.u_banana = hs.parse_userid("@banana:test") + self.u_apple = UserID.from_string("@apple:test") + self.u_banana = UserID.from_string("@banana:test") @defer.inlineCallbacks def test_state(self): diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index 5d36723c2..84381241b 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.profile import ProfileStore +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool @@ -36,7 +37,7 @@ class ProfileStoreTestCase(unittest.TestCase): self.store = ProfileStore(hs) - self.u_frank = hs.parse_userid("@frank:test") + self.u_frank = UserID.from_string("@frank:test") @defer.inlineCallbacks def test_displayname(self): diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 9806fbc69..a16ccad88 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -48,8 +49,8 @@ class RedactionTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") self.room1 = hs.parse_roomid("!abc123:test") diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index e7739776e..c6bfde069 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool @@ -40,7 +41,7 @@ class RoomStoreTestCase(unittest.TestCase): self.room = hs.parse_roomid("!abcde:test") self.alias = hs.parse_roomalias("#a-room-name:test") - self.u_creator = hs.parse_userid("@creator:test") + self.u_creator = UserID.from_string("@creator:test") yield self.store.store_room(self.room.to_string(), room_creator_user_id=self.u_creator.to_string(), diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index a23a8189d..6b7930b1d 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -49,11 +50,11 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") # User elsewhere on another host - self.u_charlie = hs.parse_userid("@charlie:elsewhere") + self.u_charlie = UserID.from_string("@charlie:elsewhere") self.room = hs.parse_roomid("!abc123:test") diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 9247fc579..d7c7f64d5 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -48,8 +49,8 @@ class StreamStoreTestCase(unittest.TestCase): self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler - self.u_alice = hs.parse_userid("@alice:test") - self.u_bob = hs.parse_userid("@bob:test") + self.u_alice = UserID.from_string("@alice:test") + self.u_bob = UserID.from_string("@bob:test") self.room1 = hs.parse_roomid("!abc123:test") self.room2 = hs.parse_roomid("!xyx987:test") diff --git a/tests/test_types.py b/tests/test_types.py index bfb9e6f54..2de7f22ab 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -42,12 +42,6 @@ class UserIDTestCase(unittest.TestCase): self.assertTrue(userA == userAagain) self.assertTrue(userA != userB) - def test_via_homeserver(self): - user = mock_homeserver.parse_userid("@3456ijkl:my.domain") - - self.assertEquals("3456ijkl", user.localpart) - self.assertEquals("my.domain", user.domain) - class RoomAliasTestCase(unittest.TestCase): From 1c06c48ce2db3c6355e29de1533aebf36bc3775b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 11:55:12 +0000 Subject: [PATCH 043/630] Replace hs.parse_roomid with RoomID.from_string --- synapse/handlers/room.py | 2 -- synapse/rest/client/v1/room.py | 4 ++-- synapse/server.py | 7 +------ tests/storage/test_directory.py | 3 ++- tests/storage/test_redaction.py | 4 ++-- tests/storage/test_room.py | 6 +++--- tests/storage/test_roommember.py | 4 ++-- tests/storage/test_stream.py | 6 +++--- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0242288c4..edb96cec8 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -246,8 +246,6 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def get_room_members(self, room_id): - hs = self.hs - users = yield self.store.get_users_in_room(room_id) defer.returnValue([UserID.from_string(u) for u in users]) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c5837b340..f0a9c932c 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,7 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID import json import logging @@ -227,7 +227,7 @@ class JoinRoomAliasServlet(RestServlet): identifier = self.hs.parse_roomalias(room_identifier) is_room_alias = True except SynapseError: - identifier = self.hs.parse_roomid(room_identifier) + identifier = RoomID.from_string(room_identifier) # TODO: Support for specifying the home server to join with? diff --git a/synapse/server.py b/synapse/server.py index 52a21aaf7..4dfff0427 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,7 +26,7 @@ from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import UserID, RoomAlias, RoomID, EventID +from synapse.types import RoomAlias, EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -132,11 +132,6 @@ class BaseHomeServer(object): object.""" return RoomAlias.from_string(s) - def parse_roomid(self, s): - """Parse the string given by 's' as a Room ID and return a RoomID - object.""" - return RoomID.from_string(s) - def parse_eventid(self, s): """Parse the string given by 's' as a Event ID and return a EventID object.""" diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py index e9c242cc0..1bc639176 100644 --- a/tests/storage/test_directory.py +++ b/tests/storage/test_directory.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.directory import DirectoryStore +from synapse.types import RoomID from tests.utils import SQLiteMemoryDbPool @@ -37,7 +38,7 @@ class DirectoryStoreTestCase(unittest.TestCase): self.store = DirectoryStore(hs) - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") self.alias = hs.parse_roomalias("#my-room:test") @defer.inlineCallbacks diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index a16ccad88..0713dfab6 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -52,7 +52,7 @@ class RedactionTestCase(unittest.TestCase): self.u_alice = UserID.from_string("@alice:test") self.u_bob = UserID.from_string("@bob:test") - self.room1 = hs.parse_roomid("!abc123:test") + self.room1 = RoomID.from_string("!abc123:test") self.depth = 1 diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index c6bfde069..baec3a3bb 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool @@ -39,7 +39,7 @@ class RoomStoreTestCase(unittest.TestCase): # management of the 'room_aliases' table self.store = hs.get_datastore() - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") self.alias = hs.parse_roomalias("#a-room-name:test") self.u_creator = UserID.from_string("@creator:test") @@ -98,7 +98,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.event_factory = hs.get_event_factory(); - self.room = hs.parse_roomid("!abcde:test") + self.room = RoomID.from_string("!abcde:test") yield self.store.store_room(self.room.to_string(), room_creator_user_id="@creator:text", diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 6b7930b1d..2b9048e2a 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -56,7 +56,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): # User elsewhere on another host self.u_charlie = UserID.from_string("@charlie:elsewhere") - self.room = hs.parse_roomid("!abc123:test") + self.room = RoomID.from_string("!abc123:test") @defer.inlineCallbacks def inject_room_member(self, room, user, membership, replaces_state=None): diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index d7c7f64d5..b7f6e2aa8 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID +from synapse.types import UserID, RoomID from tests.utils import SQLiteMemoryDbPool, MockKey @@ -52,8 +52,8 @@ class StreamStoreTestCase(unittest.TestCase): self.u_alice = UserID.from_string("@alice:test") self.u_bob = UserID.from_string("@bob:test") - self.room1 = hs.parse_roomid("!abc123:test") - self.room2 = hs.parse_roomid("!xyx987:test") + self.room1 = RoomID.from_string("!abc123:test") + self.room2 = RoomID.from_string("!xyx987:test") self.depth = 1 From ada711504efb4dd25fe1123d38a0b2d196b9890a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 13:21:58 +0000 Subject: [PATCH 044/630] Replace hs.parse_roomalias with RoomAlias.from_string --- synapse/handlers/directory.py | 3 ++- synapse/rest/client/v1/directory.py | 9 +++++---- synapse/rest/client/v1/room.py | 4 ++-- synapse/server.py | 7 +------ tests/handlers/test_directory.py | 7 ++++--- tests/storage/test_directory.py | 4 ++-- tests/storage/test_room.py | 4 ++-- tests/test_types.py | 6 ------ 8 files changed, 18 insertions(+), 26 deletions(-) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 91fceda2a..58e9a9156 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import SynapseError, Codes, CodeMessageException from synapse.api.constants import EventTypes +from synapse.types import RoomAlias import logging @@ -122,7 +123,7 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def on_directory_query(self, args): - room_alias = self.hs.parse_roomalias(args["room_alias"]) + room_alias = RoomAlias.from_string(args["room_alias"]) if not self.hs.is_mine(room_alias): raise SynapseError( 400, "Room Alias is not hosted on this Home Server" diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 7ff44fdd9..1f33ec9e8 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -17,7 +17,8 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError, Codes -from base import RestServlet, client_path_pattern +from synapse.types import RoomAlias +from .base import RestServlet, client_path_pattern import json import logging @@ -35,7 +36,7 @@ class ClientDirectoryServer(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_alias): - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) dir_handler = self.handlers.directory_handler res = yield dir_handler.get_association(room_alias) @@ -53,7 +54,7 @@ class ClientDirectoryServer(RestServlet): logger.debug("Got content: %s", content) - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) logger.debug("Got room name: %s", room_alias.to_string()) @@ -92,7 +93,7 @@ class ClientDirectoryServer(RestServlet): dir_handler = self.handlers.directory_handler - room_alias = self.hs.parse_roomalias(room_alias) + room_alias = RoomAlias.from_string(room_alias) yield dir_handler.delete_association( user.to_string(), room_alias diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f0a9c932c..42712d4a7 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -20,7 +20,7 @@ from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership -from synapse.types import UserID, RoomID +from synapse.types import UserID, RoomID, RoomAlias import json import logging @@ -224,7 +224,7 @@ class JoinRoomAliasServlet(RestServlet): identifier = None is_room_alias = False try: - identifier = self.hs.parse_roomalias(room_identifier) + identifier = RoomAlias.from_string(room_identifier) is_room_alias = True except SynapseError: identifier = RoomID.from_string(room_identifier) diff --git a/synapse/server.py b/synapse/server.py index 4dfff0427..41a26ad91 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,7 +26,7 @@ from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import RoomAlias, EventID +from synapse.types import EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -127,11 +127,6 @@ class BaseHomeServer(object): # TODO: Why are these parse_ methods so high up along with other globals? # Surely these should be in a util package or in the api package? - def parse_roomalias(self, s): - """Parse the string given by 's' as a Room Alias and return a RoomAlias - object.""" - return RoomAlias.from_string(s) - def parse_eventid(self, s): """Parse the string given by 's' as a Event ID and return a EventID object.""" diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 8e164e4be..22119de46 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -21,6 +21,7 @@ from mock import Mock from synapse.server import HomeServer from synapse.handlers.directory import DirectoryHandler +from synapse.types import RoomAlias from tests.utils import SQLiteMemoryDbPool, MockKey @@ -65,9 +66,9 @@ class DirectoryTestCase(unittest.TestCase): self.store = hs.get_datastore() - self.my_room = hs.parse_roomalias("#my-room:test") - self.your_room = hs.parse_roomalias("#your-room:test") - self.remote_room = hs.parse_roomalias("#another:remote") + self.my_room = RoomAlias.from_string("#my-room:test") + self.your_room = RoomAlias.from_string("#your-room:test") + self.remote_room = RoomAlias.from_string("#another:remote") @defer.inlineCallbacks def test_get_local_association(self): diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py index 1bc639176..bc9ebf35e 100644 --- a/tests/storage/test_directory.py +++ b/tests/storage/test_directory.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.storage.directory import DirectoryStore -from synapse.types import RoomID +from synapse.types import RoomID, RoomAlias from tests.utils import SQLiteMemoryDbPool @@ -39,7 +39,7 @@ class DirectoryStoreTestCase(unittest.TestCase): self.store = DirectoryStore(hs) self.room = RoomID.from_string("!abcde:test") - self.alias = hs.parse_roomalias("#my-room:test") + self.alias = RoomAlias.from_string("#my-room:test") @defer.inlineCallbacks def test_room_to_alias(self): diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index baec3a3bb..71e5d3414 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.server import HomeServer from synapse.api.constants import EventTypes -from synapse.types import UserID, RoomID +from synapse.types import UserID, RoomID, RoomAlias from tests.utils import SQLiteMemoryDbPool @@ -40,7 +40,7 @@ class RoomStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.room = RoomID.from_string("!abcde:test") - self.alias = hs.parse_roomalias("#a-room-name:test") + self.alias = RoomAlias.from_string("#a-room-name:test") self.u_creator = UserID.from_string("@creator:test") yield self.store.store_room(self.room.to_string(), diff --git a/tests/test_types.py b/tests/test_types.py index 2de7f22ab..b29a8415b 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -56,9 +56,3 @@ class RoomAliasTestCase(unittest.TestCase): room = RoomAlias("channel", "my.domain") self.assertEquals(room.to_string(), "#channel:my.domain") - - def test_via_homeserver(self): - room = mock_homeserver.parse_roomalias("#elsewhere:my.domain") - - self.assertEquals("elsewhere", room.localpart) - self.assertEquals("my.domain", room.domain) From 6188c4f69c2f902410b43bc50c0ae8d488b4d93c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:23:10 +0000 Subject: [PATCH 045/630] make per-device rules work --- synapse/rest/client/v1/push_rule.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 6f108431b..417fd368d 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -73,7 +73,7 @@ class PushRuleRestServlet(RestServlet): spec['device'] = device return spec - def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj): + def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj, device=None): if rule_template in ['override', 'underride']: if 'conditions' not in req_obj: raise InvalidRuleException("Missing 'conditions'") @@ -104,6 +104,12 @@ class PushRuleRestServlet(RestServlet): else: raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + if device: + conditions.append({ + 'kind': 'device', + 'instance_handle': device + }) + if 'actions' not in req_obj: raise InvalidRuleException("No actions found") actions = req_obj['actions'] @@ -144,7 +150,8 @@ class PushRuleRestServlet(RestServlet): (conditions, actions) = self.rule_tuple_from_request_object( spec['template'], spec['rule_id'], - content + content, + device=spec['device'] if 'device' in spec else None ) except InvalidRuleException as e: raise SynapseError(400, e.message) @@ -200,11 +207,11 @@ class PushRuleRestServlet(RestServlet): if not instance_handle: continue if instance_handle not in rules['device']: - rules['device'][instance_handle] = [] + rules['device'][instance_handle] = {} rules['device'][instance_handle] = \ _add_empty_priority_class_arrays(rules['device'][instance_handle]) - rulearray = rules['device'][instance_handle] + rulearray = rules['device'][instance_handle][template_name] else: rulearray = rules['global'][template_name] @@ -227,7 +234,10 @@ class PushRuleRestServlet(RestServlet): elif path[0] == 'device': path = path[1:] if path == []: - raise UnrecognizedRequestError + raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + if path[0] == '': + defer.returnValue((200, rules['device'])) + instance_handle = path[0] if instance_handle not in rules['device']: ret = {} From c4652d7772a0d9b374fc178502a71efd03d35d48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 13:25:07 +0000 Subject: [PATCH 046/630] Remove hs.parse_eventid --- synapse/server.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index 41a26ad91..32013b1a9 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,7 +26,6 @@ from synapse.api.auth import Auth from synapse.handlers import Handlers from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import EventID from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -124,14 +123,6 @@ class BaseHomeServer(object): setattr(BaseHomeServer, "get_%s" % (depname), _get) - # TODO: Why are these parse_ methods so high up along with other globals? - # Surely these should be in a util package or in the api package? - - def parse_eventid(self, s): - """Parse the string given by 's' as a Event ID and return a EventID - object.""" - return EventID.from_string(s) - def serialize_event(self, e, as_client_event=True): return serialize_event(self, e, as_client_event) From 54c689c8199336b819c632a3e996120cb13007db Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:14 +0000 Subject: [PATCH 047/630] stray space --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 417fd368d..9cb249403 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -65,7 +65,7 @@ class PushRuleRestServlet(RestServlet): rule_id = path[0] spec = { - 'scope' : scope, + 'scope': scope, 'template': template, 'rule_id': rule_id } From 98e1080555965c650e09ed09bdac3b52daeda123 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:36 +0000 Subject: [PATCH 048/630] redundant parens --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 9cb249403..46b8c3f62 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -122,7 +122,7 @@ class PushRuleRestServlet(RestServlet): else: raise InvalidRuleException("Unrecognised action") - return (conditions, actions) + return conditions, actions def priority_class_from_spec(self, spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): From d3e72b4d8788e64cd4b1d9668382639476730b4d Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:25:58 +0000 Subject: [PATCH 049/630] Make string format tuple an actual tuple --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 46b8c3f62..35ffcba3d 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -102,7 +102,7 @@ class PushRuleRestServlet(RestServlet): 'pattern': req_obj['pattern'] }] else: - raise InvalidRuleException("Unknown rule template: %s" % (rule_template)) + raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) if device: conditions.append({ From b3f66ea6fb5871d7eeb71b466740adb61a89c0d2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:28:00 +0000 Subject: [PATCH 050/630] more pep8 --- synapse/rest/client/v1/push_rule.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 35ffcba3d..ce2f0febf 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -249,7 +249,6 @@ class PushRuleRestServlet(RestServlet): else: raise UnrecognizedRequestError() - def on_OPTIONS(self, _): return 200, {} @@ -259,6 +258,7 @@ def _add_empty_priority_class_arrays(d): d[pc] = [] return d + def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -289,6 +289,7 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -297,12 +298,13 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: - return {k:rule[k] for k in ["rule_id", "actions"]} + return {k: rule[k] for k in ["rule_id", "actions"]} elif template_name == 'content': if len(rule["conditions"]) != 1: return None From fc7a05c443c169b07b917bb2692bdd7c738824fb Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 13:36:01 +0000 Subject: [PATCH 051/630] more pep8 suggestions --- synapse/push/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 3ee652f3b..47da31e50 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -74,7 +74,6 @@ class Pusher(object): defer.returnValue(ctx) - @defer.inlineCallbacks def start(self): if not self.last_token: @@ -116,7 +115,7 @@ class Pusher(object): processed = False if self._should_notify_for_event(single_event): rejected = yield self.dispatch_push(single_event) - if not rejected == False: + if not rejected is False: processed = True for pk in rejected: if pk != self.pushkey: From 4be637cb120f00e5de99c12a79e908e2f26da8af Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 14:09:51 +0000 Subject: [PATCH 052/630] Extract the client v1 base RestServlet to a separate class --- synapse/rest/client/v1/admin.py | 4 +-- synapse/rest/client/v1/base.py | 40 ++++---------------------- synapse/rest/client/v1/directory.py | 4 +-- synapse/rest/client/v1/events.py | 6 ++-- synapse/rest/client/v1/initial_sync.py | 4 +-- synapse/rest/client/v1/login.py | 8 +++--- synapse/rest/client/v1/presence.py | 6 ++-- synapse/rest/client/v1/profile.py | 8 +++--- synapse/rest/client/v1/register.py | 4 +-- synapse/rest/client/v1/room.py | 28 +++++++++--------- synapse/rest/client/v1/voip.py | 4 +-- 11 files changed, 44 insertions(+), 72 deletions(-) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 4aefb9405..1051d96f9 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -18,14 +18,14 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError from synapse.types import UserID -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern import logging logger = logging.getLogger(__name__) -class WhoisRestServlet(RestServlet): +class WhoisRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/admin/whois/(?P[^/]*)") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py index d005206b7..72332bdb1 100644 --- a/synapse/rest/client/v1/base.py +++ b/synapse/rest/client/v1/base.py @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" This module contains base REST classes for constructing REST servlets. """ +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.http.servlet import RestServlet from synapse.api.urls import CLIENT_PREFIX from .transactions import HttpTransactionStore import re @@ -37,44 +40,13 @@ def client_path_pattern(path_regex): return re.compile("^" + CLIENT_PREFIX + path_regex) -class RestServlet(object): - - """ A Synapse REST Servlet. - - An implementing class can either provide its own custom 'register' method, - or use the automatic pattern handling provided by the base class. - - To use this latter, the implementing class instead provides a `PATTERN` - class attribute containing a pre-compiled regular expression. The automatic - register method will then use this method to register any of the following - instance methods associated with the corresponding HTTP method: - - on_GET - on_PUT - on_POST - on_DELETE - on_OPTIONS - - Automatically handles turning CodeMessageExceptions thrown by these methods - into the appropriate HTTP response. +class ClientV1RestServlet(RestServlet): + """A base Synapse REST Servlet for the client version 1 API. """ def __init__(self, hs): self.hs = hs - self.handlers = hs.get_handlers() self.builder_factory = hs.get_event_builder_factory() self.auth = hs.get_auth() self.txns = HttpTransactionStore() - - def register(self, http_server): - """ Register this servlet with the given HTTP server. """ - if hasattr(self, "PATTERN"): - pattern = self.PATTERN - - for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): - if hasattr(self, "on_%s" % (method)): - method_handler = getattr(self, "on_%s" % (method)) - http_server.register_path(method, pattern, method_handler) - else: - raise NotImplementedError("RestServlet must register something.") diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 1f33ec9e8..15ae8749b 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError, Codes from synapse.types import RoomAlias -from .base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import json import logging @@ -31,7 +31,7 @@ def register_servlets(hs, http_server): ClientDirectoryServer(hs).register(http_server) -class ClientDirectoryServer(RestServlet): +class ClientDirectoryServer(ClientV1RestServlet): PATTERN = client_path_pattern("/directory/room/(?P[^/]*)$") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index c2515528a..c69de5686 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig -from .base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import logging @@ -26,7 +26,7 @@ import logging logger = logging.getLogger(__name__) -class EventStreamRestServlet(RestServlet): +class EventStreamRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/events$") DEFAULT_LONGPOLL_TIME_MS = 30000 @@ -61,7 +61,7 @@ class EventStreamRestServlet(RestServlet): # TODO: Unit test gets, with and without auth, with different kinds of events. -class EventRestServlet(RestServlet): +class EventRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/events/(?P[^/]*)$") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index b13d56b28..357fa845b 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -16,11 +16,11 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern # TODO: Needs unit testing -class InitialSyncRestServlet(RestServlet): +class InitialSyncRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/initialSync$") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 6b8deff67..7116ac98e 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -17,12 +17,12 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.types import UserID -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern import json -class LoginRestServlet(RestServlet): +class LoginRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login$") PASS_TYPE = "m.login.password" @@ -64,7 +64,7 @@ class LoginRestServlet(RestServlet): defer.returnValue((200, result)) -class LoginFallbackRestServlet(RestServlet): +class LoginFallbackRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login/fallback$") def on_GET(self, request): @@ -73,7 +73,7 @@ class LoginFallbackRestServlet(RestServlet): return (200, {}) -class PasswordResetRestServlet(RestServlet): +class PasswordResetRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login/reset") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 22fcb7d7d..b6c207e66 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.types import UserID -from .base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import json import logging @@ -27,7 +27,7 @@ import logging logger = logging.getLogger(__name__) -class PresenceStatusRestServlet(RestServlet): +class PresenceStatusRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/presence/(?P[^/]*)/status") @defer.inlineCallbacks @@ -72,7 +72,7 @@ class PresenceStatusRestServlet(RestServlet): return (200, {}) -class PresenceListRestServlet(RestServlet): +class PresenceListRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/presence/list/(?P[^/]*)") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 39297930c..24f8d5695 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -16,13 +16,13 @@ """ This module contains REST servlets to do with profile: /profile/ """ from twisted.internet import defer -from .base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID import json -class ProfileDisplaynameRestServlet(RestServlet): +class ProfileDisplaynameRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") @defer.inlineCallbacks @@ -55,7 +55,7 @@ class ProfileDisplaynameRestServlet(RestServlet): return (200, {}) -class ProfileAvatarURLRestServlet(RestServlet): +class ProfileAvatarURLRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/avatar_url") @defer.inlineCallbacks @@ -88,7 +88,7 @@ class ProfileAvatarURLRestServlet(RestServlet): return (200, {}) -class ProfileRestServlet(RestServlet): +class ProfileRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)") @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index e3b26902d..c0423c2d4 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.api.constants import LoginType -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor @@ -42,7 +42,7 @@ else: compare_digest = lambda a, b: a == b -class RegisterRestServlet(RestServlet): +class RegisterRestServlet(ClientV1RestServlet): """Handles registration with the home server. This servlet is in control of the registration flow; the registration diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 42712d4a7..f06e3ddb9 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -16,7 +16,7 @@ """ This module contains REST servlets to do with rooms: /rooms/ """ from twisted.internet import defer -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership @@ -30,7 +30,7 @@ import urllib logger = logging.getLogger(__name__) -class RoomCreateRestServlet(RestServlet): +class RoomCreateRestServlet(ClientV1RestServlet): # No PATTERN; we have custom dispatch rules here def register(self, http_server): @@ -94,7 +94,7 @@ class RoomCreateRestServlet(RestServlet): # TODO: Needs unit testing for generic events -class RoomStateEventRestServlet(RestServlet): +class RoomStateEventRestServlet(ClientV1RestServlet): def register(self, http_server): # /room/$roomid/state/$eventtype no_state_key = "/rooms/(?P[^/]*)/state/(?P[^/]*)$" @@ -163,7 +163,7 @@ class RoomStateEventRestServlet(RestServlet): # TODO: Needs unit testing for generic events + feedback -class RoomSendEventRestServlet(RestServlet): +class RoomSendEventRestServlet(ClientV1RestServlet): def register(self, http_server): # /rooms/$roomid/send/$event_type[/$txn_id] @@ -206,7 +206,7 @@ class RoomSendEventRestServlet(RestServlet): # TODO: Needs unit testing for room ID + alias joins -class JoinRoomAliasServlet(RestServlet): +class JoinRoomAliasServlet(ClientV1RestServlet): def register(self, http_server): # /join/$room_identifier[/$txn_id] @@ -265,7 +265,7 @@ class JoinRoomAliasServlet(RestServlet): # TODO: Needs unit testing -class PublicRoomListRestServlet(RestServlet): +class PublicRoomListRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/publicRooms$") @defer.inlineCallbacks @@ -276,7 +276,7 @@ class PublicRoomListRestServlet(RestServlet): # TODO: Needs unit testing -class RoomMemberListRestServlet(RestServlet): +class RoomMemberListRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/members$") @defer.inlineCallbacks @@ -305,7 +305,7 @@ class RoomMemberListRestServlet(RestServlet): # TODO: Needs unit testing -class RoomMessageListRestServlet(RestServlet): +class RoomMessageListRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/messages$") @defer.inlineCallbacks @@ -329,7 +329,7 @@ class RoomMessageListRestServlet(RestServlet): # TODO: Needs unit testing -class RoomStateRestServlet(RestServlet): +class RoomStateRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/state$") @defer.inlineCallbacks @@ -345,7 +345,7 @@ class RoomStateRestServlet(RestServlet): # TODO: Needs unit testing -class RoomInitialSyncRestServlet(RestServlet): +class RoomInitialSyncRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/initialSync$") @defer.inlineCallbacks @@ -360,7 +360,7 @@ class RoomInitialSyncRestServlet(RestServlet): defer.returnValue((200, content)) -class RoomTriggerBackfill(RestServlet): +class RoomTriggerBackfill(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") @defer.inlineCallbacks @@ -379,7 +379,7 @@ class RoomTriggerBackfill(RestServlet): # TODO: Needs unit testing -class RoomMembershipRestServlet(RestServlet): +class RoomMembershipRestServlet(ClientV1RestServlet): def register(self, http_server): # /rooms/$roomid/[invite|join|leave] @@ -431,7 +431,7 @@ class RoomMembershipRestServlet(RestServlet): defer.returnValue(response) -class RoomRedactEventRestServlet(RestServlet): +class RoomRedactEventRestServlet(ClientV1RestServlet): def register(self, http_server): PATTERN = ("/rooms/(?P[^/]*)/redact/(?P[^/]*)") register_txn_path(self, PATTERN, http_server) @@ -469,7 +469,7 @@ class RoomRedactEventRestServlet(RestServlet): defer.returnValue(response) -class RoomTypingRestServlet(RestServlet): +class RoomTypingRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern( "/rooms/(?P[^/]*)/typing/(?P[^/]*)$" ) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 011c35e69..822d863ce 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from base import RestServlet, client_path_pattern +from base import ClientV1RestServlet, client_path_pattern import hmac @@ -23,7 +23,7 @@ import hashlib import base64 -class VoipRestServlet(RestServlet): +class VoipRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/voip/turnServer$") @defer.inlineCallbacks From e0bf18addf643bd72a0cd5cb8dbc127dc7a7e8d2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 14:16:28 +0000 Subject: [PATCH 053/630] Add RestServlet base class in synapse/http/servlet.py --- synapse/http/servlet.py | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 synapse/http/servlet.py diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py new file mode 100644 index 000000000..d5ccf2742 --- /dev/null +++ b/synapse/http/servlet.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" This module contains base REST classes for constructing REST servlets. """ + +import logging + + +logger = logging.getLogger(__name__) + + +class RestServlet(object): + + """ A Synapse REST Servlet. + + An implementing class can either provide its own custom 'register' method, + or use the automatic pattern handling provided by the base class. + + To use this latter, the implementing class instead provides a `PATTERN` + class attribute containing a pre-compiled regular expression. The automatic + register method will then use this method to register any of the following + instance methods associated with the corresponding HTTP method: + + on_GET + on_PUT + on_POST + on_DELETE + on_OPTIONS + + Automatically handles turning CodeMessageExceptions thrown by these methods + into the appropriate HTTP response. + """ + + def register(self, http_server): + """ Register this servlet with the given HTTP server. """ + if hasattr(self, "PATTERN"): + pattern = self.PATTERN + + for method in ("GET", "PUT", "POST", "OPTIONS", "DELETE"): + if hasattr(self, "on_%s" % (method)): + method_handler = getattr(self, "on_%s" % (method)) + http_server.register_path(method, pattern, method_handler) + else: + raise NotImplementedError("RestServlet must register something.") From 3b9cc882a50c886afd7d2cf1eaa7e02e8b0d0d51 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Jan 2015 15:42:52 +0000 Subject: [PATCH 054/630] Add storage method have_events --- synapse/storage/__init__.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 015fcc877..4f0990960 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -422,6 +422,35 @@ class DataStore(RoomMemberStore, RoomStore, ], ) + def have_events(self, event_ids): + """Given a list of event ids, check if we have already processed them. + + Returns: + dict: Has an entry for each event id we already have seen. Maps to + the rejected reason string if we rejected the event, else maps to + None. + """ + def f(txn): + sql = ( + "SELECT e.event_id, reason FROM events as e " + "LEFT JOIN rejections as r ON e.event_id = r.event_id " + "WHERE event_id = ?" + ) + + res = {} + for event_id in event_ids: + txn.execute(sql, (event_id,)) + row = txn.fetchone() + if row: + _, rejected = row + res[event_id] = rejected + + return res + + return self.runInteraction( + "have_events", f, + ) + def schema_path(schema): """ Get a filesystem path for the named database schema From 30a89d2fdb8db9a06974fd9732c4eab5263dea79 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Jan 2015 15:51:21 +0000 Subject: [PATCH 055/630] Update .gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 2ed22b1cd..3899c5a25 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,9 @@ uploads .idea/ media_store/ + +*.tac + +build/ + +localhost-800*/ From f21f9fa3c51db49212c42adfe6972025e1d27a15 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 17:07:06 +0000 Subject: [PATCH 056/630] Use push settings! --- synapse/push/__init__.py | 91 ++++++++++++++++++++++++++--- synapse/push/httppusher.py | 9 +-- synapse/rest/client/v1/push_rule.py | 43 +++++++++----- 3 files changed, 117 insertions(+), 26 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 47da31e50..53d331969 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -21,6 +21,8 @@ from synapse.types import StreamToken import synapse.util.async import logging +import fnmatch +import json logger = logging.getLogger(__name__) @@ -29,6 +31,7 @@ class Pusher(object): INITIAL_BACKOFF = 1000 MAX_BACKOFF = 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000 + DEFAULT_ACTIONS = ['notify'] def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, @@ -37,7 +40,7 @@ class Pusher(object): self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() - self.instance_handle = instance_handle, + self.instance_handle = instance_handle self.user_name = user_name self.app_id = app_id self.app_display_name = app_display_name @@ -51,7 +54,8 @@ class Pusher(object): self.failing_since = failing_since self.alive = True - def _should_notify_for_event(self, ev): + @defer.inlineCallbacks + def _actions_for_event(self, ev): """ This should take into account notification settings that the user has configured both globally and per-room when we have the ability @@ -59,8 +63,47 @@ class Pusher(object): """ if ev['user_id'] == self.user_name: # let's assume you probably know about messages you sent yourself + defer.returnValue(['dont_notify']) + + rules = yield self.store.get_push_rules_for_user_name(self.user_name) + + for r in rules: + matches = True + + conditions = json.loads(r['conditions']) + actions = json.loads(r['actions']) + + for c in conditions: + matches &= self._event_fulfills_condition(ev, c) + # ignore rules with no actions (we have an explict 'dont_notify' + if len(actions) == 0: + logger.warn( + "Ignoring rule id %s with no actions for user %s" % + (r['rule_id'], r['user_name']) + ) + continue + if matches: + defer.returnValue(actions) + + defer.returnValue(Pusher.DEFAULT_ACTIONS) + + def _event_fulfills_condition(self, ev, condition): + if condition['kind'] == 'event_match': + if 'pattern' not in condition: + logger.warn("event_match condition with no pattern") + return False + pat = condition['pattern'] + + val = _value_for_dotted_key(condition['key'], ev) + if fnmatch.fnmatch(val, pat): + return True return False - return True + elif condition['kind'] == 'device': + if 'instance_handle' not in condition: + return True + return condition['instance_handle'] == self.instance_handle + else: + return True @defer.inlineCallbacks def get_context_for_event(self, ev): @@ -113,8 +156,23 @@ class Pusher(object): continue processed = False - if self._should_notify_for_event(single_event): - rejected = yield self.dispatch_push(single_event) + actions = yield self._actions_for_event(single_event) + tweaks = _tweaks_for_actions(actions) + + if len(actions) == 0: + logger.warn("Empty actions! Using default action.") + actions = Pusher.DEFAULT_ACTIONS + if 'notify' not in actions and 'dont_notify' not in actions: + logger.warn("Neither notify nor dont_notify in actions: adding default") + actions.extend(Pusher.DEFAULT_ACTIONS) + if 'dont_notify' in actions: + logger.debug( + "%s for %s: dont_notify", + single_event['event_id'], self.user_name + ) + processed = True + else: + rejected = yield self.dispatch_push(single_event, tweaks) if not rejected is False: processed = True for pk in rejected: @@ -133,8 +191,6 @@ class Pusher(object): yield self.hs.get_pusherpool().remove_pusher( self.app_id, pk ) - else: - processed = True if not self.alive: continue @@ -202,7 +258,7 @@ class Pusher(object): def stop(self): self.alive = False - def dispatch_push(self, p): + def dispatch_push(self, p, tweaks): """ Overridden by implementing classes to actually deliver the notification :param p: The event to notify for as a single event from the event stream @@ -214,6 +270,25 @@ class Pusher(object): pass +def _value_for_dotted_key(dotted_key, event): + parts = dotted_key.split(".") + val = event + while len(parts) > 0: + if parts[0] not in val: + return None + val = val[parts[0]] + parts = parts[1:] + return val + +def _tweaks_for_actions(actions): + tweaks = {} + for a in actions: + if not isinstance(a, dict): + continue + if 'set_sound' in a: + tweaks['sound'] = a['set_sound'] + return tweaks + class PusherConfigException(Exception): def __init__(self, msg): super(PusherConfigException, self).__init__(msg) \ No newline at end of file diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 46433ad4a..25db1dded 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -52,7 +52,7 @@ class HttpPusher(Pusher): del self.data_minus_url['url'] @defer.inlineCallbacks - def _build_notification_dict(self, event): + def _build_notification_dict(self, event, tweaks): # we probably do not want to push for every presence update # (we may want to be able to set up notifications when specific # people sign in, but we'd want to only deliver the pertinent ones) @@ -83,7 +83,8 @@ class HttpPusher(Pusher): 'app_id': self.app_id, 'pushkey': self.pushkey, 'pushkey_ts': long(self.pushkey_ts / 1000), - 'data': self.data_minus_url + 'data': self.data_minus_url, + 'tweaks': tweaks } ] } @@ -97,8 +98,8 @@ class HttpPusher(Pusher): defer.returnValue(d) @defer.inlineCallbacks - def dispatch_push(self, event): - notification_dict = yield self._build_notification_dict(event) + def dispatch_push(self, event, tweaks): + notification_dict = yield self._build_notification_dict(event, tweaks) if not notification_dict: defer.returnValue([]) try: diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index ce2f0febf..9dc2c0e11 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -96,10 +96,15 @@ class PushRuleRestServlet(RestServlet): elif rule_template == 'content': if 'pattern' not in req_obj: raise InvalidRuleException("Content rule missing 'pattern'") + pat = req_obj['pattern'] + if pat.strip("*?[]") == pat: + # no special glob characters so we assume the user means + # 'contains this string' rather than 'is this string' + pat = "*%s*" % (pat) conditions = [{ 'kind': 'event_match', 'key': 'content.body', - 'pattern': req_obj['pattern'] + 'pattern': pat }] else: raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) @@ -115,7 +120,7 @@ class PushRuleRestServlet(RestServlet): actions = req_obj['actions'] for a in actions: - if a in ['notify', 'dont-notify', 'coalesce']: + if a in ['notify', 'dont_notify', 'coalesce']: pass elif isinstance(a, dict) and 'set_sound' in a: pass @@ -124,21 +129,11 @@ class PushRuleRestServlet(RestServlet): return conditions, actions - def priority_class_from_spec(self, spec): - if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): - raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) - pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] - - if spec['scope'] == 'device': - pc += 5 - - return pc - @defer.inlineCallbacks def on_PUT(self, request): spec = self.rule_spec_from_path(request.postpath) try: - priority_class = self.priority_class_from_spec(spec) + priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: raise SynapseError(400, e.message) @@ -204,6 +199,7 @@ class PushRuleRestServlet(RestServlet): if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device rule instance_handle = _instance_handle_from_conditions(r["conditions"]) + r = _strip_device_condition(r) if not instance_handle: continue if instance_handle not in rules['device']: @@ -239,6 +235,7 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, rules['device'])) instance_handle = path[0] + path = path[1:] if instance_handle not in rules['device']: ret = {} ret = _add_empty_priority_class_arrays(ret) @@ -290,10 +287,21 @@ def _filter_ruleset_with_path(ruleset, path): raise NotFoundError +def _priority_class_from_spec(spec): + if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) + pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] + + if spec['scope'] == 'device': + pc += len(PushRuleRestServlet.PRIORITY_CLASS_MAP) + + return pc + + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device - prio_class_index = pc - PushRuleRestServlet.PRIORITY_CLASS_MAP['override'] + prio_class_index = pc - len(PushRuleRestServlet.PRIORITY_CLASS_MAP) return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[prio_class_index] else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] @@ -316,6 +324,13 @@ def _rule_to_template(rule): return ret +def _strip_device_condition(rule): + for i,c in enumerate(rule['conditions']): + if c['kind'] == 'device': + del rule['conditions'][i] + return rule + + class InvalidRuleException(Exception): pass From 5f84ba8ea1991dff279f0135f474d9debfd1419a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 23 Jan 2015 17:49:37 +0000 Subject: [PATCH 057/630] Add API to delete push rules. --- synapse/rest/client/v1/push_rule.py | 41 ++++++++++++++++++++++++++++- synapse/storage/push_rule.py | 9 +++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 9dc2c0e11..50bf5b900 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,7 +15,8 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError +from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ + StoreError from base import RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException @@ -175,6 +176,44 @@ class PushRuleRestServlet(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_DELETE(self, request): + spec = self.rule_spec_from_path(request.postpath) + try: + priority_class = _priority_class_from_spec(spec) + except InvalidRuleException as e: + raise SynapseError(400, e.message) + + user = yield self.auth.get_user_by_req(request) + + if 'device' in spec: + rules = yield self.hs.get_datastore().get_push_rules_for_user_name( + user.to_string() + ) + + for r in rules: + conditions = json.loads(r['conditions']) + ih = _instance_handle_from_conditions(conditions) + if ih == spec['device'] and r['priority_class'] == priority_class: + yield self.hs.get_datastore().delete_push_rule( + user.to_string(), spec['rule_id'] + ) + defer.returnValue((200, {})) + raise NotFoundError() + else: + try: + yield self.hs.get_datastore().delete_push_rule( + user.to_string(), spec['rule_id'], + priority_class=priority_class + ) + defer.returnValue((200, {})) + except StoreError as e: + if e.code == 404: + raise NotFoundError() + else: + raise + + @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 2366090e0..ca04f2cce 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -174,6 +174,15 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) + @defer.inlineCallbacks + def delete_push_rule(self, user_name, rule_id): + yield self._simple_delete_one( + PushRuleTable.table_name, + { + 'user_name': user_name, + 'rule_id': rule_id + } + ) class RuleNotFoundException(Exception): pass From 85419e12571ff07f2978d6013e592ac9bc207ee0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 23 Jan 2015 18:37:13 +0000 Subject: [PATCH 058/630] Stop complaining about Synapse Angular SDK 0.6.1 --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index b1fae991e..4182ad990 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], - "matrix_angular_sdk==0.6.0": ["syweb==0.6.0"], + "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], "Twisted>=14.0.0": ["twisted>=14.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], From e26340cee7049b6c36f4c3451ec7524fa6b80d1c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:31:29 +0000 Subject: [PATCH 059/630] Start implementing the v2_alpha sync API --- synapse/api/urls.py | 1 + synapse/http/servlet.py | 57 +++++++++ synapse/rest/client/v2_alpha/__init__.py | 33 ++++++ synapse/rest/client/v2_alpha/_base.py | 38 ++++++ synapse/rest/client/v2_alpha/sync.py | 143 +++++++++++++++++++++++ 5 files changed, 272 insertions(+) create mode 100644 synapse/rest/client/v2_alpha/__init__.py create mode 100644 synapse/rest/client/v2_alpha/_base.py create mode 100644 synapse/rest/client/v2_alpha/sync.py diff --git a/synapse/api/urls.py b/synapse/api/urls.py index a29939204..693c0efda 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -16,6 +16,7 @@ """Contains the URL paths to prefix various aspects of the server with. """ CLIENT_PREFIX = "/_matrix/client/api/v1" +CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index d5ccf2742..a4eb6c817 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -15,6 +15,8 @@ """ This module contains base REST classes for constructing REST servlets. """ +from synapse.api.errors import SynapseError + import logging @@ -54,3 +56,58 @@ class RestServlet(object): http_server.register_path(method, pattern, method_handler) else: raise NotImplementedError("RestServlet must register something.") + + @staticmethod + def parse_integer(request, name, default=None, required=False): + if name in request.args: + try: + return int(request.args[name][0]) + except: + message = "Query parameter %r must be an integer" % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing integer query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_boolean(request, name, default=None, required=False): + if name in request.args: + try: + return { + "true": True, + "false": False, + }[request.args[name][0]] + except: + message = ( + "Boolean query parameter %r must be one of" + " ['true', 'false']" + ) % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing boolean query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_string(request, name, default=None, required=False, + allowed_values=None, param_type="string"): + if name in request.args: + value = request.args[name][0] + if allowed_values is not None and value not in allowed_values: + message = "Query parameter %r must be one of [%s]" % ( + name, ", ".join(repr(v) for v in allowed_values) + ) + raise SynapseError(message) + else: + return value + else: + if required: + message = "Missing %s query parameter %r" % (param_type, name) + raise SynapseError(400, message) + else: + return default diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py new file mode 100644 index 000000000..e75f9d250 --- /dev/null +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from . import ( + sync, +) + +from synapse.http.server import JsonResource + + +class ClientV2AlphaRestResource(JsonResource): + """A resource for version 2 alpha of the matrix client API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(client_resource, hs): + sync.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py new file mode 100644 index 000000000..22dc5cb86 --- /dev/null +++ b/synapse/rest/client/v2_alpha/_base.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def client_v2_pattern(path_regex): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py new file mode 100644 index 000000000..39bb5ec8e --- /dev/null +++ b/synapse/rest/client/v2_alpha/sync.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.http.servlet import RestServlet +from ._base import client_v2_pattern + +import logging + +logger = logging.getLogger(__name__) + + +class SyncRestServlet(RestServlet): + """ + + GET parameters:: + timeout(int): How long to wait for new events in milliseconds. + limit(int): Maxiumum number of events per room to return. + gap(bool): Create gaps the message history if limit is exceeded to + ensure that the client has the most recent messages. Defaults to + "true". + sort(str,str): tuple of sort key (e.g. "timeline") and direction + (e.g. "asc", "desc"). Defaults to "timeline,asc". + since(batch_token): Batch token when asking for incremental deltas. + set_presence(str): What state the device presence should be set to. + default is "online". + backfill(bool): Should the HS request message history from other + servers. This may take a long time making it unsuitable for clients + expecting a prompt response. Defaults to "true". + filter(filter_id): A filter to apply to the events returned. + filter_*: Filter override parameters. + + Response JSON:: + { + "next_batch": // batch token for the next /sync + "private_user_data": // private events for this user. + "public_user_data": // public events for all users including the + // public events for this user. + "rooms": [{ // List of rooms with updates. + "room_id": // Id of the room being updated + "limited": // Was the per-room event limit exceeded? + "published": // Is the room published by our HS? + "event_map": // Map of EventID -> event JSON. + "events": { // The recent events in the room if gap is "true" + // otherwise the next events in the room. + "batch": [] // list of EventIDs in the "event_map". + "prev_batch": // back token for getting previous events. + } + "state": [] // list of EventIDs updating the current state to + // be what it should be at the end of the batch. + }] + } + """ + + + PATTERN = client_v2_pattern("/sync$") + ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) + ALLOWED_PRESENCE = set(["online", "offline", "idle"]) + + def __init__(self, hs): + super(SyncRestServlet, self).__init__() + self.auth = hs.get_auth() + #self.sync_handler = hs.get_handlers().sync_hanlder + + @defer.inlineCallbacks + def on_GET(self, request): + user = yield self.auth.get_user_by_req(request) + + timeout = self.parse_integer(request, "timeout", default=0) + limit = self.parse_integer(request, "limit", default=None) + gap = self.parse_boolean(request, "gap", default=True) + sort = self.parse_string( + request, "sort", default="timeline,asc", + allowed_values=self.ALLOWED_SORT + ) + since = self.parse_string(request, "since") + set_presence = self.parse_string( + request, "set_presence", default="online", + allowed_values=self.ALLOWED_PRESENCE + ) + backfill = self.parse_boolean(request, "backfill", default=True) + filter_id = self.parse_string(request, "filter", default=None) + + logger.info( + "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r," + " set_presence=%r, backfill=%r, filter_id=%r" % ( + user, timeout, limit, gap, sort, since, set_presence, + backfill, filter_id + ) + ) + + # TODO(mjark): Load filter and apply overrides. + # filter = self.filters.load_fitler(filter_id_str) + # filter = filter.apply_overrides(http_request) + # if filter.matches(event): + # # stuff + + # if timeout != 0: + # register for updates from the event stream + + #rooms = [] + + if gap: + pass + # now_stream_token = get_current_stream_token + # for room_id in get_rooms_for_user(user, filter=filter): + # state, events, start, end, limited, published = updates_for_room( + # from=since, to=now_stream_token, limit=limit, + # anchor_to_start=False + # ) + # rooms[room_id] = (state, events, start, limited, published) + # next_stream_token = now. + else: + pass + # now_stream_token = get_current_stream_token + # for room_id in get_rooms_for_user(user, filter=filter) + # state, events, start, end, limited, published = updates_for_room( + # from=since, to=now_stream_token, limit=limit, + # anchor_to_start=False + # ) + # next_stream_token = min(next_stream_token, end) + + + response_content = {} + + defer.returnValue((200, response_content)) + + +def register_servlets(hs, http_server): + SyncRestServlet(hs).register(http_server) From 2b1799883db3facf63b7982a0302542082414974 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:49:05 +0000 Subject: [PATCH 060/630] Add client v2_alpha resource to synapse server resource tree --- synapse/app/homeserver.py | 7 ++++++- synapse/server.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fabe8ddac..40d28dcbd 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -32,12 +32,13 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext from synapse.rest.client.v1 import ClientV1RestResource +from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource from daemonize import Daemonize import twisted.manhole.telnet @@ -62,6 +63,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_client(self): return ClientV1RestResource(self) + def build_resource_for_client_v2_alpha(self): + return ClientV2AlphaRestResource(self) + def build_resource_for_federation(self): return JsonResource() @@ -105,6 +109,7 @@ class SynapseHomeServer(HomeServer): # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] desired_tree = [ (CLIENT_PREFIX, self.get_resource_for_client()), + (CLIENT_V2_ALPHA_PREFIX, self.get_resource_for_client_v2_alpha()), (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), diff --git a/synapse/server.py b/synapse/server.py index 32013b1a9..92ed2c5e3 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -70,6 +70,7 @@ class BaseHomeServer(object): 'notifier', 'distributor', 'resource_for_client', + 'resource_for_client_v2_alpha', 'resource_for_federation', 'resource_for_web_client', 'resource_for_content_repo', From 7b814d3f7fc8137426bc97fb80751753eb8eb94b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 23 Jan 2015 18:54:51 +0000 Subject: [PATCH 061/630] Add client v2_alpha resource to synapse server resource tree --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 7 ++- synapse/http/servlet.py | 57 ++++++++++++++++++++++++ synapse/rest/client/v2_alpha/__init__.py | 29 ++++++++++++ synapse/rest/client/v2_alpha/_base.py | 38 ++++++++++++++++ synapse/server.py | 1 + 6 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/__init__.py create mode 100644 synapse/rest/client/v2_alpha/_base.py diff --git a/synapse/api/urls.py b/synapse/api/urls.py index a29939204..693c0efda 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -16,6 +16,7 @@ """Contains the URL paths to prefix various aspects of the server with. """ CLIENT_PREFIX = "/_matrix/client/api/v1" +CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fabe8ddac..40d28dcbd 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -32,12 +32,13 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext from synapse.rest.client.v1 import ClientV1RestResource +from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource from daemonize import Daemonize import twisted.manhole.telnet @@ -62,6 +63,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_client(self): return ClientV1RestResource(self) + def build_resource_for_client_v2_alpha(self): + return ClientV2AlphaRestResource(self) + def build_resource_for_federation(self): return JsonResource() @@ -105,6 +109,7 @@ class SynapseHomeServer(HomeServer): # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] desired_tree = [ (CLIENT_PREFIX, self.get_resource_for_client()), + (CLIENT_V2_ALPHA_PREFIX, self.get_resource_for_client_v2_alpha()), (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index d5ccf2742..a4eb6c817 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -15,6 +15,8 @@ """ This module contains base REST classes for constructing REST servlets. """ +from synapse.api.errors import SynapseError + import logging @@ -54,3 +56,58 @@ class RestServlet(object): http_server.register_path(method, pattern, method_handler) else: raise NotImplementedError("RestServlet must register something.") + + @staticmethod + def parse_integer(request, name, default=None, required=False): + if name in request.args: + try: + return int(request.args[name][0]) + except: + message = "Query parameter %r must be an integer" % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing integer query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_boolean(request, name, default=None, required=False): + if name in request.args: + try: + return { + "true": True, + "false": False, + }[request.args[name][0]] + except: + message = ( + "Boolean query parameter %r must be one of" + " ['true', 'false']" + ) % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing boolean query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + @staticmethod + def parse_string(request, name, default=None, required=False, + allowed_values=None, param_type="string"): + if name in request.args: + value = request.args[name][0] + if allowed_values is not None and value not in allowed_values: + message = "Query parameter %r must be one of [%s]" % ( + name, ", ".join(repr(v) for v in allowed_values) + ) + raise SynapseError(message) + else: + return value + else: + if required: + message = "Missing %s query parameter %r" % (param_type, name) + raise SynapseError(400, message) + else: + return default diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py new file mode 100644 index 000000000..bb740e280 --- /dev/null +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from synapse.http.server import JsonResource + + +class ClientV2AlphaRestResource(JsonResource): + """A resource for version 2 alpha of the matrix client API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(client_resource, hs): + pass diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py new file mode 100644 index 000000000..22dc5cb86 --- /dev/null +++ b/synapse/rest/client/v2_alpha/_base.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def client_v2_pattern(path_regex): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) diff --git a/synapse/server.py b/synapse/server.py index 32013b1a9..92ed2c5e3 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -70,6 +70,7 @@ class BaseHomeServer(object): 'notifier', 'distributor', 'resource_for_client', + 'resource_for_client_v2_alpha', 'resource_for_federation', 'resource_for_web_client', 'resource_for_content_repo', From 7b8861924130821c1bbd05ce65260209a993f759 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Jan 2015 10:45:24 +0000 Subject: [PATCH 062/630] Split up replication_layer module into client, server and transaction queue --- synapse/federation/federation_client.py | 293 ++++++++++++ synapse/federation/federation_server.py | 345 ++++++++++++++ synapse/federation/replication.py | 608 +----------------------- synapse/federation/transaction_queue.py | 9 +- synapse/storage/__init__.py | 2 +- 5 files changed, 654 insertions(+), 603 deletions(-) create mode 100644 synapse/federation/federation_client.py create mode 100644 synapse/federation/federation_server.py diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py new file mode 100644 index 000000000..c80f4c61b --- /dev/null +++ b/synapse/federation/federation_client.py @@ -0,0 +1,293 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Edu + +from synapse.util.logutils import log_function +from synapse.events import FrozenEvent + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationClient(object): + @log_function + def send_pdu(self, pdu, destinations): + """Informs the replication layer about a new PDU generated within the + home server that should be transmitted to others. + + TODO: Figure out when we should actually resolve the deferred. + + Args: + pdu (Pdu): The new Pdu. + + Returns: + Deferred: Completes when we have successfully processed the PDU + and replicated it to any interested remote home servers. + """ + order = self._order + self._order += 1 + + logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) + + # TODO, add errback, etc. + self._transaction_queue.enqueue_pdu(pdu, destinations, order) + + logger.debug( + "[%s] transaction_layer.enqueue_pdu... done", + pdu.event_id + ) + + @log_function + def send_edu(self, destination, edu_type, content): + edu = Edu( + origin=self.server_name, + destination=destination, + edu_type=edu_type, + content=content, + ) + + # TODO, add errback, etc. + self._transaction_queue.enqueue_edu(edu) + return defer.succeed(None) + + @log_function + def send_failure(self, failure, destination): + self._transaction_queue.enqueue_failure(failure, destination) + return defer.succeed(None) + + @log_function + def make_query(self, destination, query_type, args, + retry_on_dns_fail=True): + """Sends a federation Query to a remote homeserver of the given type + and arguments. + + Args: + destination (str): Domain name of the remote homeserver + query_type (str): Category of the query type; should match the + handler name used in register_query_handler(). + args (dict): Mapping of strings to strings containing the details + of the query request. + + Returns: + a Deferred which will eventually yield a JSON object from the + response + """ + return self.transport_layer.make_query( + destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail + ) + + @defer.inlineCallbacks + @log_function + def backfill(self, dest, context, limit, extremities): + """Requests some more historic PDUs for the given context from the + given destination server. + + Args: + dest (str): The remote home server to ask. + context (str): The context to backfill. + limit (int): The maximum number of PDUs to return. + extremities (list): List of PDU id and origins of the first pdus + we have seen from the context + + Returns: + Deferred: Results in the received PDUs. + """ + logger.debug("backfill extrem=%s", extremities) + + # If there are no extremeties then we've (probably) reached the start. + if not extremities: + return + + transaction_data = yield self.transport_layer.backfill( + dest, context, extremities, limit) + + logger.debug("backfill transaction_data=%s", repr(transaction_data)) + + pdus = [ + self.event_from_pdu_json(p, outlier=False) + for p in transaction_data["pdus"] + ] + + defer.returnValue(pdus) + + @defer.inlineCallbacks + @log_function + def get_pdu(self, destinations, event_id, outlier=False): + """Requests the PDU with given origin and ID from the remote home + servers. + + Will attempt to get the PDU from each destination in the list until + one succeeds. + + This will persist the PDU locally upon receipt. + + Args: + destinations (list): Which home servers to query + pdu_origin (str): The home server that originally sent the pdu. + event_id (str) + outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if + it's from an arbitary point in the context as opposed to part + of the current block of PDUs. Defaults to `False` + + Returns: + Deferred: Results in the requested PDU. + """ + + # TODO: Rate limit the number of times we try and get the same event. + + pdu = None + for destination in destinations: + try: + transaction_data = yield self.transport_layer.get_event( + destination, event_id + ) + except Exception as e: + logger.info( + "Failed to get PDU %s from %s because %s", + event_id, destination, e, + ) + continue + + logger.debug("transaction_data %r", transaction_data) + + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction_data["pdus"] + ] + + if pdu_list: + pdu = pdu_list[0] + # TODO: We need to check signatures here + break + + defer.returnValue(pdu) + + @defer.inlineCallbacks + @log_function + def get_state_for_room(self, destination, room_id, event_id): + """Requests all of the `current` state PDUs for a given room from + a remote home server. + + Args: + destination (str): The remote homeserver to query for the state. + room_id (str): The id of the room we're interested in. + event_id (str): The id of the event we want the state at. + + Returns: + Deferred: Results in a list of PDUs. + """ + + result = yield self.transport_layer.get_room_state( + destination, room_id, event_id=event_id, + ) + + pdus = [ + self.event_from_pdu_json(p, outlier=True) for p in result["pdus"] + ] + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in result.get("auth_chain", []) + ] + + defer.returnValue((pdus, auth_chain)) + + @defer.inlineCallbacks + @log_function + def get_event_auth(self, destination, room_id, event_id): + res = yield self.transport_layer.get_event_auth( + destination, room_id, event_id, + ) + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in res["auth_chain"] + ] + + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue(auth_chain) + + @defer.inlineCallbacks + def make_join(self, destination, room_id, user_id): + ret = yield self.transport_layer.make_join( + destination, room_id, user_id + ) + + pdu_dict = ret["event"] + + logger.debug("Got response to make_join: %s", pdu_dict) + + defer.returnValue(self.event_from_pdu_json(pdu_dict)) + + @defer.inlineCallbacks + def send_join(self, destination, pdu): + time_now = self._clock.time_msec() + _, content = yield self.transport_layer.send_join( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) + + logger.debug("Got content: %s", content) + + state = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("state", []) + ] + + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("auth_chain", []) + ] + + auth_chain.sort(key=lambda e: e.depth) + + defer.returnValue({ + "state": state, + "auth_chain": auth_chain, + }) + + @defer.inlineCallbacks + def send_invite(self, destination, room_id, event_id, pdu): + time_now = self._clock.time_msec() + code, content = yield self.transport_layer.send_invite( + destination=destination, + room_id=room_id, + event_id=event_id, + content=pdu.get_pdu_json(time_now), + ) + + pdu_dict = content["event"] + + logger.debug("Got response to send_invite: %s", pdu_dict) + + defer.returnValue(self.event_from_pdu_json(pdu_dict)) + + def event_from_pdu_json(self, pdu_json, outlier=False): + event = FrozenEvent( + pdu_json + ) + + event.internal_metadata.outlier = outlier + + return event diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py new file mode 100644 index 000000000..0597725ce --- /dev/null +++ b/synapse/federation/federation_server.py @@ -0,0 +1,345 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from .units import Transaction, Edu + +from synapse.util.logutils import log_function +from synapse.util.logcontext import PreserveLoggingContext +from synapse.events import FrozenEvent + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationServer(object): + def set_handler(self, handler): + """Sets the handler that the replication layer will use to communicate + receipt of new PDUs from other home servers. The required methods are + documented on :py:class:`.ReplicationHandler`. + """ + self.handler = handler + + def register_edu_handler(self, edu_type, handler): + if edu_type in self.edu_handlers: + raise KeyError("Already have an EDU handler for %s" % (edu_type,)) + + self.edu_handlers[edu_type] = handler + + def register_query_handler(self, query_type, handler): + """Sets the handler callable that will be used to handle an incoming + federation Query of the given type. + + Args: + query_type (str): Category name of the query, which should match + the string used by make_query. + handler (callable): Invoked to handle incoming queries of this type + + handler is invoked as: + result = handler(args) + + where 'args' is a dict mapping strings to strings of the query + arguments. It should return a Deferred that will eventually yield an + object to encode as JSON. + """ + if query_type in self.query_handlers: + raise KeyError( + "Already have a Query handler for %s" % (query_type,) + ) + + self.query_handlers[query_type] = handler + + @defer.inlineCallbacks + @log_function + def on_backfill_request(self, origin, room_id, versions, limit): + pdus = yield self.handler.on_backfill_request( + origin, room_id, versions, limit + ) + + defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) + + @defer.inlineCallbacks + @log_function + def on_incoming_transaction(self, transaction_data): + transaction = Transaction(**transaction_data) + + for p in transaction.pdus: + if "unsigned" in p: + unsigned = p["unsigned"] + if "age" in unsigned: + p["age"] = unsigned["age"] + if "age" in p: + p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) + del p["age"] + + pdu_list = [ + self.event_from_pdu_json(p) for p in transaction.pdus + ] + + logger.debug("[%s] Got transaction", transaction.transaction_id) + + response = yield self.transaction_actions.have_responded(transaction) + + if response: + logger.debug("[%s] We've already responed to this request", + transaction.transaction_id) + defer.returnValue(response) + return + + logger.debug("[%s] Transaction is new", transaction.transaction_id) + + with PreserveLoggingContext(): + dl = [] + for pdu in pdu_list: + dl.append(self._handle_new_pdu(transaction.origin, pdu)) + + if hasattr(transaction, "edus"): + for edu in [Edu(**x) for x in transaction.edus]: + self.received_edu( + transaction.origin, + edu.edu_type, + edu.content + ) + + results = yield defer.DeferredList(dl) + + ret = [] + for r in results: + if r[0]: + ret.append({}) + else: + logger.exception(r[1]) + ret.append({"error": str(r[1])}) + + logger.debug("Returning: %s", str(ret)) + + yield self.transaction_actions.set_response( + transaction, + 200, response + ) + defer.returnValue((200, response)) + + def received_edu(self, origin, edu_type, content): + if edu_type in self.edu_handlers: + self.edu_handlers[edu_type](origin, content) + else: + logger.warn("Received EDU of type %s with no handler", edu_type) + + @defer.inlineCallbacks + @log_function + def on_context_state_request(self, origin, room_id, event_id): + if event_id: + pdus = yield self.handler.get_state_for_pdu( + origin, room_id, event_id, + ) + auth_chain = yield self.store.get_auth_chain( + [pdu.event_id for pdu in pdus] + ) + else: + raise NotImplementedError("Specify an event") + + defer.returnValue((200, { + "pdus": [pdu.get_pdu_json() for pdu in pdus], + "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], + })) + + @defer.inlineCallbacks + @log_function + def on_pdu_request(self, origin, event_id): + pdu = yield self._get_persisted_pdu(origin, event_id) + + if pdu: + defer.returnValue( + (200, self._transaction_from_pdus([pdu]).get_dict()) + ) + else: + defer.returnValue((404, "")) + + @defer.inlineCallbacks + @log_function + def on_pull_request(self, origin, versions): + raise NotImplementedError("Pull transactions not implemented") + + @defer.inlineCallbacks + def on_query_request(self, query_type, args): + if query_type in self.query_handlers: + response = yield self.query_handlers[query_type](args) + defer.returnValue((200, response)) + else: + defer.returnValue( + (404, "No handler for Query type '%s'" % (query_type,)) + ) + + @defer.inlineCallbacks + def on_make_join_request(self, room_id, user_id): + pdu = yield self.handler.on_make_join_request(room_id, user_id) + time_now = self._clock.time_msec() + defer.returnValue({"event": pdu.get_pdu_json(time_now)}) + + @defer.inlineCallbacks + def on_invite_request(self, origin, content): + pdu = self.event_from_pdu_json(content) + ret_pdu = yield self.handler.on_invite_request(origin, pdu) + time_now = self._clock.time_msec() + defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)})) + + @defer.inlineCallbacks + def on_send_join_request(self, origin, content): + logger.debug("on_send_join_request: content: %s", content) + pdu = self.event_from_pdu_json(content) + logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) + res_pdus = yield self.handler.on_send_join_request(origin, pdu) + time_now = self._clock.time_msec() + defer.returnValue((200, { + "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], + "auth_chain": [ + p.get_pdu_json(time_now) for p in res_pdus["auth_chain"] + ], + })) + + @defer.inlineCallbacks + def on_event_auth(self, origin, room_id, event_id): + time_now = self._clock.time_msec() + auth_pdus = yield self.handler.on_event_auth(event_id) + defer.returnValue((200, { + "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], + })) + + @log_function + def _get_persisted_pdu(self, origin, event_id, do_auth=True): + """ Get a PDU from the database with given origin and id. + + Returns: + Deferred: Results in a `Pdu`. + """ + return self.handler.get_persisted_pdu( + origin, event_id, do_auth=do_auth + ) + + def _transaction_from_pdus(self, pdu_list): + """Returns a new Transaction containing the given PDUs suitable for + transmission. + """ + time_now = self._clock.time_msec() + pdus = [p.get_pdu_json(time_now) for p in pdu_list] + return Transaction( + origin=self.server_name, + pdus=pdus, + origin_server_ts=int(time_now), + destination=None, + ) + + @defer.inlineCallbacks + @log_function + def _handle_new_pdu(self, origin, pdu, max_recursion=10): + # We reprocess pdus when we have seen them only as outliers + existing = yield self._get_persisted_pdu( + origin, pdu.event_id, do_auth=False + ) + + already_seen = ( + existing and ( + not existing.internal_metadata.is_outlier() + or pdu.internal_metadata.is_outlier() + ) + ) + if already_seen: + logger.debug("Already seen pdu %s", pdu.event_id) + defer.returnValue({}) + return + + state = None + + auth_chain = [] + + have_seen = yield self.store.have_events( + [e for e, _ in pdu.prev_events] + ) + + # Get missing pdus if necessary. + if not pdu.internal_metadata.is_outlier(): + # We only backfill backwards to the min depth. + min_depth = yield self.handler.get_min_depth_for_context( + pdu.room_id + ) + + logger.debug( + "_handle_new_pdu min_depth for %s: %d", + pdu.room_id, min_depth + ) + + if min_depth and pdu.depth > min_depth and max_recursion > 0: + for event_id, hashes in pdu.prev_events: + if event_id not in have_seen: + logger.debug( + "_handle_new_pdu requesting pdu %s", + event_id + ) + + try: + new_pdu = yield self.federation_client.get_pdu( + [origin, pdu.origin], + event_id=event_id, + ) + + if new_pdu: + yield self._handle_new_pdu( + origin, + new_pdu, + max_recursion=max_recursion-1 + ) + + logger.debug("Processed pdu %s", event_id) + else: + logger.warn("Failed to get PDU %s", event_id) + except: + # TODO(erikj): Do some more intelligent retries. + logger.exception("Failed to get PDU") + else: + # We need to get the state at this event, since we have reached + # a backward extremity edge. + logger.debug( + "_handle_new_pdu getting state for %s", + pdu.room_id + ) + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) + + ret = yield self.handler.on_receive_pdu( + origin, + pdu, + backfilled=False, + state=state, + auth_chain=auth_chain, + ) + + defer.returnValue(ret) + + def __str__(self): + return "" % self.server_name + + def event_from_pdu_json(self, pdu_json, outlier=False): + event = FrozenEvent( + pdu_json + ) + + event.internal_metadata.outlier = outlier + + return event diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index accf95e40..9ef483492 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -17,15 +17,12 @@ a given transport. """ -from twisted.internet import defer +from .federation_client import FederationClient +from .federation_server import FederationServer -from .units import Transaction, Edu -from .persistence import TransactionActions from .transaction_queue import TransactionQueue -from synapse.util.logutils import log_function -from synapse.util.logcontext import PreserveLoggingContext -from synapse.events import FrozenEvent +from .persistence import TransactionActions import logging @@ -33,7 +30,7 @@ import logging logger = logging.getLogger(__name__) -class ReplicationLayer(object): +class ReplicationLayer(FederationClient, FederationServer): """This layer is responsible for replicating with remote home servers over the given transport. I.e., does the sending and receiving of PDUs to remote home servers. @@ -58,607 +55,20 @@ class ReplicationLayer(object): self.transport_layer.register_received_handler(self) self.transport_layer.register_request_handler(self) - self.store = hs.get_datastore() - # self.pdu_actions = PduActions(self.store) - self.transaction_actions = TransactionActions(self.store) + self.federation_client = self - self._transaction_queue = TransactionQueue( - hs, self.transaction_actions, transport_layer - ) + self.store = hs.get_datastore() self.handler = None self.edu_handlers = {} self.query_handlers = {} - self._order = 0 - self._clock = hs.get_clock() - self.event_builder_factory = hs.get_event_builder_factory() + self.transaction_actions = TransactionActions(self.store) + self._transaction_queue = TransactionQueue(hs, transport_layer) - def set_handler(self, handler): - """Sets the handler that the replication layer will use to communicate - receipt of new PDUs from other home servers. The required methods are - documented on :py:class:`.ReplicationHandler`. - """ - self.handler = handler - - def register_edu_handler(self, edu_type, handler): - if edu_type in self.edu_handlers: - raise KeyError("Already have an EDU handler for %s" % (edu_type,)) - - self.edu_handlers[edu_type] = handler - - def register_query_handler(self, query_type, handler): - """Sets the handler callable that will be used to handle an incoming - federation Query of the given type. - - Args: - query_type (str): Category name of the query, which should match - the string used by make_query. - handler (callable): Invoked to handle incoming queries of this type - - handler is invoked as: - result = handler(args) - - where 'args' is a dict mapping strings to strings of the query - arguments. It should return a Deferred that will eventually yield an - object to encode as JSON. - """ - if query_type in self.query_handlers: - raise KeyError( - "Already have a Query handler for %s" % (query_type,) - ) - - self.query_handlers[query_type] = handler - - @log_function - def send_pdu(self, pdu, destinations): - """Informs the replication layer about a new PDU generated within the - home server that should be transmitted to others. - - TODO: Figure out when we should actually resolve the deferred. - - Args: - pdu (Pdu): The new Pdu. - - Returns: - Deferred: Completes when we have successfully processed the PDU - and replicated it to any interested remote home servers. - """ - order = self._order - self._order += 1 - - logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) - - # TODO, add errback, etc. - self._transaction_queue.enqueue_pdu(pdu, destinations, order) - - logger.debug( - "[%s] transaction_layer.enqueue_pdu... done", - pdu.event_id - ) - - @log_function - def send_edu(self, destination, edu_type, content): - edu = Edu( - origin=self.server_name, - destination=destination, - edu_type=edu_type, - content=content, - ) - - # TODO, add errback, etc. - self._transaction_queue.enqueue_edu(edu) - return defer.succeed(None) - - @log_function - def send_failure(self, failure, destination): - self._transaction_queue.enqueue_failure(failure, destination) - return defer.succeed(None) - - @log_function - def make_query(self, destination, query_type, args, - retry_on_dns_fail=True): - """Sends a federation Query to a remote homeserver of the given type - and arguments. - - Args: - destination (str): Domain name of the remote homeserver - query_type (str): Category of the query type; should match the - handler name used in register_query_handler(). - args (dict): Mapping of strings to strings containing the details - of the query request. - - Returns: - a Deferred which will eventually yield a JSON object from the - response - """ - return self.transport_layer.make_query( - destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail - ) - - @defer.inlineCallbacks - @log_function - def backfill(self, dest, context, limit, extremities): - """Requests some more historic PDUs for the given context from the - given destination server. - - Args: - dest (str): The remote home server to ask. - context (str): The context to backfill. - limit (int): The maximum number of PDUs to return. - extremities (list): List of PDU id and origins of the first pdus - we have seen from the context - - Returns: - Deferred: Results in the received PDUs. - """ - logger.debug("backfill extrem=%s", extremities) - - # If there are no extremeties then we've (probably) reached the start. - if not extremities: - return - - transaction_data = yield self.transport_layer.backfill( - dest, context, extremities, limit) - - logger.debug("backfill transaction_data=%s", repr(transaction_data)) - - transaction = Transaction(**transaction_data) - - pdus = [ - self.event_from_pdu_json(p, outlier=False) - for p in transaction.pdus - ] - for pdu in pdus: - yield self._handle_new_pdu(dest, pdu, backfilled=True) - - defer.returnValue(pdus) - - @defer.inlineCallbacks - @log_function - def get_pdu(self, destination, event_id, outlier=False): - """Requests the PDU with given origin and ID from the remote home - server. - - This will persist the PDU locally upon receipt. - - Args: - destination (str): Which home server to query - pdu_origin (str): The home server that originally sent the pdu. - event_id (str) - outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if - it's from an arbitary point in the context as opposed to part - of the current block of PDUs. Defaults to `False` - - Returns: - Deferred: Results in the requested PDU. - """ - - transaction_data = yield self.transport_layer.get_event( - destination, event_id - ) - - transaction = Transaction(**transaction_data) - - pdu_list = [ - self.event_from_pdu_json(p, outlier=outlier) - for p in transaction.pdus - ] - - pdu = None - if pdu_list: - pdu = pdu_list[0] - yield self._handle_new_pdu(destination, pdu) - - defer.returnValue(pdu) - - @defer.inlineCallbacks - @log_function - def get_state_for_room(self, destination, room_id, event_id): - """Requests all of the `current` state PDUs for a given room from - a remote home server. - - Args: - destination (str): The remote homeserver to query for the state. - room_id (str): The id of the room we're interested in. - event_id (str): The id of the event we want the state at. - - Returns: - Deferred: Results in a list of PDUs. - """ - - result = yield self.transport_layer.get_room_state( - destination, room_id, event_id=event_id, - ) - - pdus = [ - self.event_from_pdu_json(p, outlier=True) for p in result["pdus"] - ] - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in result.get("auth_chain", []) - ] - - defer.returnValue((pdus, auth_chain)) - - @defer.inlineCallbacks - @log_function - def get_event_auth(self, destination, room_id, event_id): - res = yield self.transport_layer.get_event_auth( - destination, room_id, event_id, - ) - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in res["auth_chain"] - ] - - auth_chain.sort(key=lambda e: e.depth) - - defer.returnValue(auth_chain) - - @defer.inlineCallbacks - @log_function - def on_backfill_request(self, origin, room_id, versions, limit): - pdus = yield self.handler.on_backfill_request( - origin, room_id, versions, limit - ) - - defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) - - @defer.inlineCallbacks - @log_function - def on_incoming_transaction(self, transaction_data): - transaction = Transaction(**transaction_data) - - for p in transaction.pdus: - if "unsigned" in p: - unsigned = p["unsigned"] - if "age" in unsigned: - p["age"] = unsigned["age"] - if "age" in p: - p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) - del p["age"] - - pdu_list = [ - self.event_from_pdu_json(p) for p in transaction.pdus - ] - - logger.debug("[%s] Got transaction", transaction.transaction_id) - - response = yield self.transaction_actions.have_responded(transaction) - - if response: - logger.debug("[%s] We've already responed to this request", - transaction.transaction_id) - defer.returnValue(response) - return - - logger.debug("[%s] Transaction is new", transaction.transaction_id) - - with PreserveLoggingContext(): - dl = [] - for pdu in pdu_list: - dl.append(self._handle_new_pdu(transaction.origin, pdu)) - - if hasattr(transaction, "edus"): - for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu( - transaction.origin, - edu.edu_type, - edu.content - ) - - results = yield defer.DeferredList(dl) - - ret = [] - for r in results: - if r[0]: - ret.append({}) - else: - logger.exception(r[1]) - ret.append({"error": str(r[1])}) - - logger.debug("Returning: %s", str(ret)) - - yield self.transaction_actions.set_response( - transaction, - 200, response - ) - defer.returnValue((200, response)) - - def received_edu(self, origin, edu_type, content): - if edu_type in self.edu_handlers: - self.edu_handlers[edu_type](origin, content) - else: - logger.warn("Received EDU of type %s with no handler", edu_type) - - @defer.inlineCallbacks - @log_function - def on_context_state_request(self, origin, room_id, event_id): - if event_id: - pdus = yield self.handler.get_state_for_pdu( - origin, room_id, event_id, - ) - auth_chain = yield self.store.get_auth_chain( - [pdu.event_id for pdu in pdus] - ) - else: - raise NotImplementedError("Specify an event") - - defer.returnValue((200, { - "pdus": [pdu.get_pdu_json() for pdu in pdus], - "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], - })) - - @defer.inlineCallbacks - @log_function - def on_pdu_request(self, origin, event_id): - pdu = yield self._get_persisted_pdu(origin, event_id) - - if pdu: - defer.returnValue( - (200, self._transaction_from_pdus([pdu]).get_dict()) - ) - else: - defer.returnValue((404, "")) - - @defer.inlineCallbacks - @log_function - def on_pull_request(self, origin, versions): - raise NotImplementedError("Pull transactions not implemented") - - @defer.inlineCallbacks - def on_query_request(self, query_type, args): - if query_type in self.query_handlers: - response = yield self.query_handlers[query_type](args) - defer.returnValue((200, response)) - else: - defer.returnValue( - (404, "No handler for Query type '%s'" % (query_type,)) - ) - - @defer.inlineCallbacks - def on_make_join_request(self, room_id, user_id): - pdu = yield self.handler.on_make_join_request(room_id, user_id) - time_now = self._clock.time_msec() - defer.returnValue({"event": pdu.get_pdu_json(time_now)}) - - @defer.inlineCallbacks - def on_invite_request(self, origin, content): - pdu = self.event_from_pdu_json(content) - ret_pdu = yield self.handler.on_invite_request(origin, pdu) - time_now = self._clock.time_msec() - defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)})) - - @defer.inlineCallbacks - def on_send_join_request(self, origin, content): - logger.debug("on_send_join_request: content: %s", content) - pdu = self.event_from_pdu_json(content) - logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) - res_pdus = yield self.handler.on_send_join_request(origin, pdu) - time_now = self._clock.time_msec() - defer.returnValue((200, { - "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]], - "auth_chain": [ - p.get_pdu_json(time_now) for p in res_pdus["auth_chain"] - ], - })) - - @defer.inlineCallbacks - def on_event_auth(self, origin, room_id, event_id): - time_now = self._clock.time_msec() - auth_pdus = yield self.handler.on_event_auth(event_id) - defer.returnValue((200, { - "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], - })) - - @defer.inlineCallbacks - def make_join(self, destination, room_id, user_id): - ret = yield self.transport_layer.make_join( - destination, room_id, user_id - ) - - pdu_dict = ret["event"] - - logger.debug("Got response to make_join: %s", pdu_dict) - - defer.returnValue(self.event_from_pdu_json(pdu_dict)) - - @defer.inlineCallbacks - def send_join(self, destination, pdu): - time_now = self._clock.time_msec() - _, content = yield self.transport_layer.send_join( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) - - logger.debug("Got content: %s", content) - - state = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("state", []) - ] - - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("auth_chain", []) - ] - - auth_chain.sort(key=lambda e: e.depth) - - defer.returnValue({ - "state": state, - "auth_chain": auth_chain, - }) - - @defer.inlineCallbacks - def send_invite(self, destination, room_id, event_id, pdu): - time_now = self._clock.time_msec() - code, content = yield self.transport_layer.send_invite( - destination=destination, - room_id=room_id, - event_id=event_id, - content=pdu.get_pdu_json(time_now), - ) - - pdu_dict = content["event"] - - logger.debug("Got response to send_invite: %s", pdu_dict) - - defer.returnValue(self.event_from_pdu_json(pdu_dict)) - - @log_function - def _get_persisted_pdu(self, origin, event_id, do_auth=True): - """ Get a PDU from the database with given origin and id. - - Returns: - Deferred: Results in a `Pdu`. - """ - return self.handler.get_persisted_pdu( - origin, event_id, do_auth=do_auth - ) - - def _transaction_from_pdus(self, pdu_list): - """Returns a new Transaction containing the given PDUs suitable for - transmission. - """ - time_now = self._clock.time_msec() - pdus = [p.get_pdu_json(time_now) for p in pdu_list] - return Transaction( - origin=self.server_name, - pdus=pdus, - origin_server_ts=int(time_now), - destination=None, - ) - - @defer.inlineCallbacks - @log_function - def _handle_new_pdu(self, origin, pdu, backfilled=False): - # We reprocess pdus when we have seen them only as outliers - existing = yield self._get_persisted_pdu( - origin, pdu.event_id, do_auth=False - ) - - already_seen = ( - existing and ( - not existing.internal_metadata.is_outlier() - or pdu.internal_metadata.is_outlier() - ) - ) - if already_seen: - logger.debug("Already seen pdu %s", pdu.event_id) - defer.returnValue({}) - return - - state = None - - auth_chain = [] - - # We need to make sure we have all the auth events. - # for e_id, _ in pdu.auth_events: - # exists = yield self._get_persisted_pdu( - # origin, - # e_id, - # do_auth=False - # ) - # - # if not exists: - # try: - # logger.debug( - # "_handle_new_pdu fetch missing auth event %s from %s", - # e_id, - # origin, - # ) - # - # yield self.get_pdu( - # origin, - # event_id=e_id, - # outlier=True, - # ) - # - # logger.debug("Processed pdu %s", e_id) - # except: - # logger.warn( - # "Failed to get auth event %s from %s", - # e_id, - # origin - # ) - - # Get missing pdus if necessary. - if not pdu.internal_metadata.is_outlier(): - # We only backfill backwards to the min depth. - min_depth = yield self.handler.get_min_depth_for_context( - pdu.room_id - ) - - logger.debug( - "_handle_new_pdu min_depth for %s: %d", - pdu.room_id, min_depth - ) - - if min_depth and pdu.depth > min_depth: - for event_id, hashes in pdu.prev_events: - exists = yield self._get_persisted_pdu( - origin, - event_id, - do_auth=False - ) - - if not exists: - logger.debug( - "_handle_new_pdu requesting pdu %s", - event_id - ) - - try: - yield self.get_pdu( - origin, - event_id=event_id, - ) - logger.debug("Processed pdu %s", event_id) - except: - # TODO(erikj): Do some more intelligent retries. - logger.exception("Failed to get PDU") - else: - # We need to get the state at this event, since we have reached - # a backward extremity edge. - logger.debug( - "_handle_new_pdu getting state for %s", - pdu.room_id - ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) - - if not backfilled: - ret = yield self.handler.on_receive_pdu( - origin, - pdu, - backfilled=backfilled, - state=state, - auth_chain=auth_chain, - ) - else: - ret = None - - # yield self.pdu_actions.mark_as_processed(pdu) - - defer.returnValue(ret) + self._order = 0 def __str__(self): return "" % self.server_name - - def event_from_pdu_json(self, pdu_json, outlier=False): - event = FrozenEvent( - pdu_json - ) - - event.internal_metadata.outlier = outlier - - return event diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index c2cb4a1c4..9d4f2c09a 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -16,6 +16,7 @@ from twisted.internet import defer +from .persistence import TransactionActions from .units import Transaction from synapse.util.logutils import log_function @@ -34,13 +35,15 @@ class TransactionQueue(object): It batches pending PDUs into single transactions. """ - def __init__(self, hs, transaction_actions, transport_layer): + def __init__(self, hs, transport_layer): self.server_name = hs.hostname - self.transaction_actions = transaction_actions + + self.store = hs.get_datastore() + self.transaction_actions = TransactionActions(self.store) + self.transport_layer = transport_layer self._clock = hs.get_clock() - self.store = hs.get_datastore() # Is a mapping from destinations -> deferreds. Used to keep track # of which destinations have transactions in flight and when they are diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4f0990960..27d835db7 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -434,7 +434,7 @@ class DataStore(RoomMemberStore, RoomStore, sql = ( "SELECT e.event_id, reason FROM events as e " "LEFT JOIN rejections as r ON e.event_id = r.event_id " - "WHERE event_id = ?" + "WHERE e.event_id = ?" ) res = {} From d07dfe5392a0ea9bde4bff174c4dc4e4fdba33be Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 26 Jan 2015 14:32:17 +0000 Subject: [PATCH 063/630] Create (empty) v2_alpha REST tests directory --- tests/rest/client/v2_alpha/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/rest/client/v2_alpha/__init__.py diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py new file mode 100644 index 000000000..803f97ea4 --- /dev/null +++ b/tests/rest/client/v2_alpha/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + From c92d64a6c35713aabaed11e8ef1e62d2fb84a875 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Jan 2015 14:33:11 +0000 Subject: [PATCH 064/630] Make it the responsibility of the replication layer to check signature and hashes. --- synapse/federation/federation_client.py | 108 +++++++++++++++++++++--- synapse/federation/federation_server.py | 89 ++++++++++++++++--- synapse/federation/replication.py | 2 + 3 files changed, 173 insertions(+), 26 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index c80f4c61b..91b44cd8b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -20,6 +20,13 @@ from .units import Edu from synapse.util.logutils import log_function from synapse.events import FrozenEvent +from synapse.events.utils import prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import SynapseError import logging @@ -126,6 +133,11 @@ class FederationClient(object): for p in transaction_data["pdus"] ] + for i, pdu in enumerate(pdus): + pdus[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + defer.returnValue(pdus) @defer.inlineCallbacks @@ -159,6 +171,22 @@ class FederationClient(object): transaction_data = yield self.transport_layer.get_event( destination, event_id ) + + logger.debug("transaction_data %r", transaction_data) + + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction_data["pdus"] + ] + + if pdu_list: + pdu = pdu_list[0] + + # Check signatures are correct. + pdu = yield self._check_sigs_and_hash(pdu) + + break + except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -166,18 +194,6 @@ class FederationClient(object): ) continue - logger.debug("transaction_data %r", transaction_data) - - pdu_list = [ - self.event_from_pdu_json(p, outlier=outlier) - for p in transaction_data["pdus"] - ] - - if pdu_list: - pdu = pdu_list[0] - # TODO: We need to check signatures here - break - defer.returnValue(pdu) @defer.inlineCallbacks @@ -208,6 +224,16 @@ class FederationClient(object): for p in result.get("auth_chain", []) ] + for i, pdu in enumerate(pdus): + pdus[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + defer.returnValue((pdus, auth_chain)) @defer.inlineCallbacks @@ -222,6 +248,11 @@ class FederationClient(object): for p in res["auth_chain"] ] + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + auth_chain.sort(key=lambda e: e.depth) defer.returnValue(auth_chain) @@ -260,6 +291,16 @@ class FederationClient(object): for p in content.get("auth_chain", []) ] + for i, pdu in enumerate(state): + state[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + for i, pdu in enumerate(auth_chain): + auth_chain[i] = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + auth_chain.sort(key=lambda e: e.depth) defer.returnValue({ @@ -281,7 +322,14 @@ class FederationClient(object): logger.debug("Got response to send_invite: %s", pdu_dict) - defer.returnValue(self.event_from_pdu_json(pdu_dict)) + pdu = self.event_from_pdu_json(pdu_dict) + + # Check signatures are correct. + pdu = yield self._check_sigs_and_hash(pdu) + + # FIXME: We should handle signature failures more gracefully. + + defer.returnValue(pdu) def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( @@ -291,3 +339,37 @@ class FederationClient(object): event.internal_metadata.outlier = outlier return event + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + try: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 0597725ce..fc5342afa 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -21,6 +21,13 @@ from .units import Transaction, Edu from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.events import FrozenEvent +from synapse.events.utils import prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import FederationError, SynapseError import logging @@ -97,8 +104,10 @@ class FederationServer(object): response = yield self.transaction_actions.have_responded(transaction) if response: - logger.debug("[%s] We've already responed to this request", - transaction.transaction_id) + logger.debug( + "[%s] We've already responed to this request", + transaction.transaction_id + ) defer.returnValue(response) return @@ -253,6 +262,9 @@ class FederationServer(object): origin, pdu.event_id, do_auth=False ) + # FIXME: Currently we fetch an event again when we already have it + # if it has been marked as an outlier. + already_seen = ( existing and ( not existing.internal_metadata.is_outlier() @@ -264,14 +276,27 @@ class FederationServer(object): defer.returnValue({}) return + # Check signature. + try: + pdu = yield self._check_sigs_and_hash(pdu) + except SynapseError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=pdu.event_id, + ) + state = None auth_chain = [] have_seen = yield self.store.have_events( - [e for e, _ in pdu.prev_events] + [ev for ev, _ in pdu.prev_events] ) + fetch_state = False + # Get missing pdus if necessary. if not pdu.internal_metadata.is_outlier(): # We only backfill backwards to the min depth. @@ -311,16 +336,20 @@ class FederationServer(object): except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") - else: - # We need to get the state at this event, since we have reached - # a backward extremity edge. - logger.debug( - "_handle_new_pdu getting state for %s", - pdu.room_id - ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) + fetch_state = True + else: + fetch_state = True + + if fetch_state: + # We need to get the state at this event, since we haven't + # processed all the prev events. + logger.debug( + "_handle_new_pdu getting state for %s", + pdu.room_id + ) + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) ret = yield self.handler.on_receive_pdu( origin, @@ -343,3 +372,37 @@ class FederationServer(object): event.internal_metadata.outlier = outlier return event + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + try: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9ef483492..e442c6c5d 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -51,6 +51,8 @@ class ReplicationLayer(FederationClient, FederationServer): def __init__(self, hs, transport_layer): self.server_name = hs.hostname + self.keyring = hs.get_keyring() + self.transport_layer = transport_layer self.transport_layer.register_received_handler(self) self.transport_layer.register_request_handler(self) From 8d7accb28fdf2b8936eb865a716593a2ffcea0ed Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 23 Jan 2015 19:52:30 +0000 Subject: [PATCH 065/630] Initial minimal attempt at /user/:user_id/filter API - in-memory storage, no actual filter implementation --- synapse/rest/client/v2_alpha/__init__.py | 7 +- synapse/rest/client/v2_alpha/filter.py | 103 +++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/filter.py diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index bb740e280..4349579ab 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -14,6 +14,11 @@ # limitations under the License. +from . import ( + filter +) + + from synapse.http.server import JsonResource @@ -26,4 +31,4 @@ class ClientV2AlphaRestResource(JsonResource): @staticmethod def register_servlets(client_resource, hs): - pass + filter.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py new file mode 100644 index 000000000..a9a180ec0 --- /dev/null +++ b/synapse/rest/client/v2_alpha/filter.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import AuthError, SynapseError +from synapse.http.servlet import RestServlet +from synapse.types import UserID + +from ._base import client_v2_pattern + +import json +import logging + + +logger = logging.getLogger(__name__) + + +# TODO(paul) +_filters_for_user = {} + + +class GetFilterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter/(?P[^/]*)") + + def __init__(self, hs): + super(GetFilterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_GET(self, request, user_id, filter_id): + target_user = UserID.from_string(user_id) + auth_user = yield self.auth.get_user_by_req(request) + + if target_user != auth_user: + raise AuthError(403, "Cannot get filters for other users") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only get filters for local users") + + try: + filter_id = int(filter_id) + except: + raise SynapseError(400, "Invalid filter_id") + + filters = _filters_for_user.get(target_user.localpart, None) + + if not filters or filter_id >= len(filters): + raise SynapseError(400, "No such filter") + + defer.returnValue((200, filters[filter_id])) + + +class CreateFilterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter") + + def __init__(self, hs): + super(CreateFilterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_POST(self, request, user_id): + target_user = UserID.from_string(user_id) + auth_user = yield self.auth.get_user_by_req(request) + + if target_user != auth_user: + raise AuthError(403, "Cannot create filters for other users") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only create filters for local users") + + try: + content = json.loads(request.content.read()) + + # TODO(paul): check for required keys and invalid keys + except: + raise SynapseError(400, "Invalid filter definition") + + filters = _filters_for_user.setdefault(target_user.localpart, []) + + filter_id = len(filters) + filters.append(content) + + defer.returnValue((200, {"filter_id": str(filter_id)})) + + +def register_servlets(hs, http_server): + GetFilterRestServlet(hs).register(http_server) + CreateFilterRestServlet(hs).register(http_server) From efac71d6caeac9ef39dab521467932892b324e99 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 14:37:14 +0000 Subject: [PATCH 066/630] Pushers should only try & look for rejected devices in something that's a list or tuple. --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 53d331969..0c51d2dd8 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -173,7 +173,7 @@ class Pusher(object): processed = True else: rejected = yield self.dispatch_push(single_event, tweaks) - if not rejected is False: + if isinstance(rejected, list) or isinstance(rejected, tuple): processed = True for pk in rejected: if pk != self.pushkey: From 37b8a71f1086b394874fb13ee63dcbeb2b2334d2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 26 Jan 2015 15:27:40 +0000 Subject: [PATCH 067/630] Initial trivial REST test of v2_alpha filter API --- tests/rest/client/v2_alpha/test_filter.py | 74 +++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/rest/client/v2_alpha/test_filter.py diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py new file mode 100644 index 000000000..1d1273ab9 --- /dev/null +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tests import unittest +from twisted.internet import defer + +from mock import Mock + +from ....utils import MockHttpResource, MockKey + +from synapse.server import HomeServer +from synapse.rest.client.v2_alpha import filter +from synapse.types import UserID + + +myid = "@apple:test" +PATH_PREFIX = "/_matrix/client/v2_alpha" + + +class FilterTestCase(unittest.TestCase): + + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + + mock_config = Mock() + mock_config.signing_key = [MockKey()] + + hs = HomeServer("test", + db_pool=None, + datastore=Mock(spec=[ + "insert_client_ip", + ]), + http_client=None, + resource_for_client=self.mock_resource, + resource_for_federation=self.mock_resource, + config=mock_config, + ) + + def _get_user_by_token(token=None): + return { + "user": UserID.from_string(myid), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + filter.register_servlets(hs, self.mock_resource) + + @defer.inlineCallbacks + def test_filter(self): + (code, response) = yield self.mock_resource.trigger("POST", + "/user/%s/filter" % (myid), + '{"type": ["m.*"]}' + ) + self.assertEquals(200, code) + self.assertEquals({"filter_id": "0"}, response) + + (code, response) = yield self.mock_resource.trigger("GET", + "/user/%s/filter/0" % (myid), None + ) + self.assertEquals(200, code) + self.assertEquals({"type": ["m.*"]}, response) From 0cfb4591a7754bcc08edddd17629006b5096d94d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 15:46:31 +0000 Subject: [PATCH 068/630] Add handler for /sync API --- synapse/handlers/sync.py | 110 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 synapse/handlers/sync.py diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py new file mode 100644 index 000000000..ec20ea489 --- /dev/null +++ b/synapse/handlers/sync.py @@ -0,0 +1,110 @@ +import collections + + +SyncConfig = collections.namedtuple("SyncConfig", [ + "user", + "device", + "since", + "limit", + "gap", + "sort" + "backfill" + "filter", +) + + +RoomSyncResult = collections.namedtuple("RoomSyncResult", [ + "room_id", + "limited", + "published", + "prev_batch", + "events", + "state", + "event_map", +]) + + +class SyncResult(collections.namedtuple("SyncResult", [ + "next_batch", # Token for the next sync + "private_user_data", # List of private events for the user. + "public_user_data", # List of public events for all users. + "rooms", # RoomSyncResult for each room. +])): + __slots__ = [] + + def __nonzero__(self): + return self.private_user_data or self.public_user_data or self.rooms + + +class SyncHandler(BaseHandler): + + def __init__(self, hs): + super(SyncHandler, self).__init__(hs) + self.event_sources = hs.get_event_sources() + + def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): + if timeout == 0: + return self.current_sync_for_user(sync_config, since) + else: + def current_sync_callback(since_token): + return self.current_sync_for_user( + self, since_token, sync_config + ) + return self.notifier.wait_for_events( + sync_config.filter, since_token, current_sync_callback + ) + defer.returnValue(result) + + def current_sync_for_user(self, sync_config, since_token=None): + if since_token is None: + return self.inital_sync(sync_config) + else: + return self.incremental_sync(sync_config) + + @defer.inlineCallbacks + def initial_sync(self, sync_config): + now_token = yield self.event_sources.get_current_token() + + presence_stream = self.event_sources.sources["presence"] + # TODO (markjh): This looks wrong, shouldn't we be getting the presence + # UP to the present rather than after the present? + pagination_config = PaginationConfig(from_token=now_token) + presence, _ = yield presence_stream.get_pagination_rows( + user, pagination_config.get_source_config("presence"), None + ) + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=user_id, + membership_list=[Membership.INVITE, Membership.JOIN] + ) + + # TODO (markjh): Does public mean "published"? + published_rooms = yield self.store.get_rooms(is_public=True) + published_room_ids = set(r["room_id"] for r in public_rooms) + + for event in room_list: + + messages, token = yield self.store.get_recent_events_for_room( + event.room_id, + limit=sync_config.limit, + end_token=now_token.room_key, + ) + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state = yield self.state_handler.get_current_state( + event.room_id + ) + + rooms.append(RoomSyncResult( + room_id=event.room_id, + published=event.room_id in published_room_ids, + + + + + + @defer.inlineCallbacks + def incremental_sync(self, sync_config): + + + + + From 7f6f3f9d6247076493c4e9d48c8282e25892f8b5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 16:11:28 +0000 Subject: [PATCH 069/630] Pass the current time to serialize event, rather than passing an HS and getting a clock from it and calling time_msec on the clock. Remove the serialize_event method from the HS since it is no longer needed. --- synapse/events/utils.py | 12 ++++++------ synapse/handlers/events.py | 5 ++++- synapse/handlers/message.py | 21 +++++++++++++++------ synapse/handlers/room.py | 7 +++++-- synapse/rest/client/v1/events.py | 8 +++++++- synapse/rest/client/v1/room.py | 9 ++++++++- synapse/server.py | 3 --- 7 files changed, 45 insertions(+), 20 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index bcb545727..e391aca4c 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,31 +89,31 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(hs, e, client_event=True): +def serialize_event(e, time_now_ms, client_event=True): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e + time_now_ms = int(time_now_ms) + # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} if not client_event: # set the age and keep all other keys if "age_ts" in d["unsigned"]: - now = int(hs.get_clock().time_msec()) - d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] + d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] return d if "age_ts" in d["unsigned"]: - now = int(hs.get_clock().time_msec()) - d["age"] = now - d["unsigned"]["age_ts"] + d["age"] = time_now_ms - d["unsigned"]["age_ts"] del d["unsigned"]["age_ts"] d["user_id"] = d.pop("sender", None) if "redacted_because" in e.unsigned: d["redacted_because"] = serialize_event( - hs, e.unsigned["redacted_because"] + e.unsigned["redacted_because"], time_now_ms ) del d["unsigned"]["redacted_because"] diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 01e67b081..d997917cd 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logcontext import PreserveLoggingContext from synapse.util.logutils import log_function from synapse.types import UserID +from synapse.events.utils import serialize_event from ._base import BaseHandler @@ -78,8 +79,10 @@ class EventStreamHandler(BaseHandler): auth_user, room_ids, pagin_config, timeout ) + time_now = self.clock.time_msec() + chunks = [ - self.hs.serialize_event(e, as_client_event) for e in events + serialize_event(e, time_now, as_client_event) for e in events ] chunk = { diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6a1104a89..9c3271fe8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig +from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator from synapse.util.logcontext import PreserveLoggingContext from synapse.types import UserID @@ -100,9 +101,11 @@ class MessageHandler(BaseHandler): "room_key", next_key ) + time_now = self.clock.time_msec() + chunk = { "chunk": [ - self.hs.serialize_event(e, as_client_event) for e in events + serialize_event(e, time_now, as_client_event) for e in events ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), @@ -211,7 +214,8 @@ class MessageHandler(BaseHandler): # TODO: This is duplicating logic from snapshot_all_rooms current_state = yield self.state_handler.get_current_state(room_id) - defer.returnValue([self.hs.serialize_event(c) for c in current_state]) + now = self.clock.time_msec() + defer.returnValue([serialize_event(c, now) for c in current_state]) @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, @@ -283,10 +287,11 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) + time_now = self.clock.time_msec() d["messages"] = { "chunk": [ - self.hs.serialize_event(m, as_client_event) + serialize_event(m, time_now, as_client_event) for m in messages ], "start": start_token.to_string(), @@ -297,7 +302,8 @@ class MessageHandler(BaseHandler): event.room_id ) d["state"] = [ - self.hs.serialize_event(c) for c in current_state + serialize_event(c, time_now, as_client_event) + for c in current_state ] except: logger.exception("Failed to get snapshot") @@ -320,8 +326,9 @@ class MessageHandler(BaseHandler): auth_user = UserID.from_string(user_id) # TODO: These concurrently + time_now = self.clock.time_msec() state_tuples = yield self.state_handler.get_current_state(room_id) - state = [self.hs.serialize_event(x) for x in state_tuples] + state = [serialize_event(x, time_now) for x in state_tuples] member_event = (yield self.store.get_room_member( user_id=user_id, @@ -360,11 +367,13 @@ class MessageHandler(BaseHandler): "Failed to get member presence of %r", m.user_id ) + time_now = self.clock.time_msec() + defer.returnValue({ "membership": member_event.membership, "room_id": room_id, "messages": { - "chunk": [self.hs.serialize_event(m) for m in messages], + "chunk": [serialize_event(m, time_now) for m in messages], "start": start_token.to_string(), "end": end_token.to_string(), }, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index edb96cec8..23821d321 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -16,12 +16,14 @@ """Contains functions for performing events on rooms.""" from twisted.internet import defer +from ._base import BaseHandler + from synapse.types import UserID, RoomAlias, RoomID from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import StoreError, SynapseError from synapse.util import stringutils from synapse.util.async import run_on_reactor -from ._base import BaseHandler +from synapse.events.utils import serialize_event import logging @@ -293,8 +295,9 @@ class RoomMemberHandler(BaseHandler): yield self.auth.check_joined_room(room_id, user_id) member_list = yield self.store.get_room_members(room_id=room_id) + time_now = self.clock.time_msec() event_list = [ - self.hs.serialize_event(entry) + serialize_event(entry, time_now) for entry in member_list ] chunk_data = { diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index c69de5686..a0d051227 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig from .base import ClientV1RestServlet, client_path_pattern +from synapse.events.utils import serialize_event import logging @@ -64,14 +65,19 @@ class EventStreamRestServlet(ClientV1RestServlet): class EventRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/events/(?P[^/]*)$") + def __init__(self, hs): + super(EventRestServlet, self).__init__(hs) + self.clock = hs.get_clock() + @defer.inlineCallbacks def on_GET(self, request, event_id): auth_user = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) + time_now = self.clock.time_msec() if event: - defer.returnValue((200, self.hs.serialize_event(event))) + defer.returnValue((200, serialize_event(event, time_now))) else: defer.returnValue((404, "Event not found.")) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f06e3ddb9..58b09b6fc 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -21,6 +21,7 @@ from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID, RoomAlias +from synapse.events.utils import serialize_event import json import logging @@ -363,6 +364,10 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): class RoomTriggerBackfill(ClientV1RestServlet): PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") + def __init__(self, hs): + super(RoomTriggerBackfill, self).__init__(hs) + self.clock = hs.get_clock() + @defer.inlineCallbacks def on_GET(self, request, room_id): remote_server = urllib.unquote( @@ -374,7 +379,9 @@ class RoomTriggerBackfill(ClientV1RestServlet): handler = self.handlers.federation_handler events = yield handler.backfill(remote_server, room_id, limit) - res = [self.hs.serialize_event(event) for event in events] + time_now = self.clock.time_msec() + + res = [serialize_event(event, time_now) for event in events] defer.returnValue((200, res)) diff --git a/synapse/server.py b/synapse/server.py index 92ed2c5e3..c478f812e 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -124,9 +124,6 @@ class BaseHomeServer(object): setattr(BaseHomeServer, "get_%s" % (depname), _get) - def serialize_event(self, e, as_client_event=True): - return serialize_event(self, e, as_client_event) - def get_ip_from_request(self, request): # May be an X-Forwarding-For header depending on config ip_addr = request.getClientIP() From e5725eb3b950d7580672ed98b1366a175e41e916 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 16:16:50 +0000 Subject: [PATCH 070/630] Remove unused import from server.py --- synapse/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index c478f812e..f09d5d581 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -20,7 +20,6 @@ # Imports required for the default HomeServer() implementation from synapse.federation import initialize_http_replication -from synapse.events.utils import serialize_event from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers From 69a75b7ebebb393c1ce84ff949f3480a6af0a782 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 16:52:47 +0000 Subject: [PATCH 071/630] Add brackets to make get room name / alias work --- synapse/storage/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8f56d90d9..2534d109f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -402,8 +402,8 @@ class DataStore(RoomMemberStore, RoomStore, "redacted": del_sql, } - sql += " AND (s.type = 'm.room.name' AND s.state_key = '')" - sql += " OR s.type = 'm.room.aliases'" + sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases')" args = (room_id,) results = yield self._execute_and_decode(sql, *args) From b481889117ceb8b31dd2e491bca53b914d306312 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 26 Jan 2015 17:27:28 +0000 Subject: [PATCH 072/630] Support membership events and more camelcase/underscores --- synapse/push/__init__.py | 4 ++++ synapse/push/httppusher.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 0c51d2dd8..b6d01a82a 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -65,6 +65,10 @@ class Pusher(object): # let's assume you probably know about messages you sent yourself defer.returnValue(['dont_notify']) + if ev['type'] == 'm.room.member': + if ev['state_key'] != self.user_name: + defer.returnValue(['dont_notify']) + rules = yield self.store.get_push_rules_for_user_name(self.user_name) for r in rules: diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 25db1dded..22532fcc6 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -65,8 +65,6 @@ class HttpPusher(Pusher): d = { 'notification': { - 'transition': 'new', - # everything is new for now: we don't have read receipts 'id': event['event_id'], 'type': event['type'], 'from': event['user_id'], @@ -89,11 +87,13 @@ class HttpPusher(Pusher): ] } } + if event['type'] == 'm.room.member': + d['notification']['membership'] = event['content']['membership'] if len(ctx['aliases']): - d['notification']['roomAlias'] = ctx['aliases'][0] + d['notification']['room_alias'] = ctx['aliases'][0] if 'name' in ctx: - d['notification']['roomName'] = ctx['name'] + d['notification']['room_name'] = ctx['name'] defer.returnValue(d) From 436513068de73ab47d9ba9a32046420be3d86588 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 26 Jan 2015 18:53:31 +0000 Subject: [PATCH 073/630] Start implementing the non-incremental sync portion of the v2 /sync API --- synapse/events/utils.py | 6 +- synapse/handlers/__init__.py | 2 + synapse/handlers/sync.py | 87 +++++++++++++++------- synapse/rest/client/v2_alpha/sync.py | 103 ++++++++++++++++++++------- 4 files changed, 145 insertions(+), 53 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index e391aca4c..b7f1ad4b4 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,7 +89,7 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(e, time_now_ms, client_event=True): +def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -138,4 +138,8 @@ def serialize_event(e, time_now_ms, client_event=True): d.pop("unsigned", None) d.pop("origin", None) + if strip_ids: + d.pop("room_id", None) + d.pop("event_id", None) + return d diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index fe071a4bc..a32eab931 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -26,6 +26,7 @@ from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler from .admin import AdminHandler +from .sync import SyncHandler class Handlers(object): @@ -51,3 +52,4 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ec20ea489..bbabaf3df 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1,26 +1,49 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import BaseHandler + +from synapse.streams.config import PaginationConfig +from synapse.api.constants import Membership + +from twisted.internet import defer + import collections +import logging + +logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", "device", - "since", "limit", "gap", - "sort" - "backfill" + "sort", + "backfill", "filter", -) +]) RoomSyncResult = collections.namedtuple("RoomSyncResult", [ "room_id", "limited", "published", - "prev_batch", - "events", + "events", # dict of event "state", - "event_map", + "prev_batch", ]) @@ -41,10 +64,11 @@ class SyncHandler(BaseHandler): def __init__(self, hs): super(SyncHandler, self).__init__(hs) self.event_sources = hs.get_event_sources() + self.clock = hs.get_clock() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): if timeout == 0: - return self.current_sync_for_user(sync_config, since) + return self.current_sync_for_user(sync_config, since_token) else: def current_sync_callback(since_token): return self.current_sync_for_user( @@ -53,58 +77,71 @@ class SyncHandler(BaseHandler): return self.notifier.wait_for_events( sync_config.filter, since_token, current_sync_callback ) - defer.returnValue(result) def current_sync_for_user(self, sync_config, since_token=None): if since_token is None: - return self.inital_sync(sync_config) + return self.initial_sync(sync_config) else: return self.incremental_sync(sync_config) @defer.inlineCallbacks def initial_sync(self, sync_config): + if sync_config.sort == "timeline,desc": + # TODO(mjark): Handle going through events in reverse order?. + # What does "most recent events" mean when applying the limits mean + # in this case? + raise NotImplementedError() + now_token = yield self.event_sources.get_current_token() presence_stream = self.event_sources.sources["presence"] - # TODO (markjh): This looks wrong, shouldn't we be getting the presence + # TODO (mjark): This looks wrong, shouldn't we be getting the presence # UP to the present rather than after the present? pagination_config = PaginationConfig(from_token=now_token) presence, _ = yield presence_stream.get_pagination_rows( - user, pagination_config.get_source_config("presence"), None + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None ) room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=user_id, + user_id=sync_config.user.to_string(), membership_list=[Membership.INVITE, Membership.JOIN] ) - # TODO (markjh): Does public mean "published"? + # TODO (mjark): Does public mean "published"? published_rooms = yield self.store.get_rooms(is_public=True) - published_room_ids = set(r["room_id"] for r in public_rooms) + published_room_ids = set(r["room_id"] for r in published_rooms) + rooms = [] for event in room_list: - - messages, token = yield self.store.get_recent_events_for_room( + #TODO (mjark): Apply the event filter in sync_config. + recent_events, token = yield self.store.get_recent_events_for_room( event.room_id, limit=sync_config.limit, end_token=now_token.room_key, ) prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - current_state = yield self.state_handler.get_current_state( + current_state_events = yield self.state_handler.get_current_state( event.room_id ) rooms.append(RoomSyncResult( room_id=event.room_id, published=event.room_id in published_room_ids, + events=recent_events, + prev_batch=prev_batch_token, + state=current_state_events, + limited=True, + )) - - + defer.returnValue(SyncResult( + public_user_data=presence, + private_user_data=[], + rooms=rooms, + next_batch=now_token, + )) @defer.inlineCallbacks def incremental_sync(self, sync_config): - - - - - + pass diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 39bb5ec8e..cc667ebaf 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,9 @@ from twisted.internet import defer from synapse.http.servlet import RestServlet +from synapse.handlers.sync import SyncConfig +from synapse.types import StreamToken +from synapse.events.utils import serialize_event from ._base import client_v2_pattern import logging @@ -73,14 +76,15 @@ class SyncRestServlet(RestServlet): def __init__(self, hs): super(SyncRestServlet, self).__init__() self.auth = hs.get_auth() - #self.sync_handler = hs.get_handlers().sync_hanlder + self.sync_handler = hs.get_handlers().sync_handler + self.clock = hs.get_clock() @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) timeout = self.parse_integer(request, "timeout", default=0) - limit = self.parse_integer(request, "limit", default=None) + limit = self.parse_integer(request, "limit", required=True) gap = self.parse_boolean(request, "gap", default=True) sort = self.parse_string( request, "sort", default="timeline,asc", @@ -91,7 +95,7 @@ class SyncRestServlet(RestServlet): request, "set_presence", default="online", allowed_values=self.ALLOWED_PRESENCE ) - backfill = self.parse_boolean(request, "backfill", default=True) + backfill = self.parse_boolean(request, "backfill", default=False) filter_id = self.parse_string(request, "filter", default=None) logger.info( @@ -108,36 +112,81 @@ class SyncRestServlet(RestServlet): # if filter.matches(event): # # stuff - # if timeout != 0: - # register for updates from the event stream + sync_config = SyncConfig( + user=user, + device="TODO", # TODO(mjark) Get the device_id from access_token + gap=gap, + limit=limit, + sort=sort, + backfill=backfill, + filter="TODO", # TODO(mjark) Add the filter to the config. + ) - #rooms = [] - - if gap: - pass - # now_stream_token = get_current_stream_token - # for room_id in get_rooms_for_user(user, filter=filter): - # state, events, start, end, limited, published = updates_for_room( - # from=since, to=now_stream_token, limit=limit, - # anchor_to_start=False - # ) - # rooms[room_id] = (state, events, start, limited, published) - # next_stream_token = now. + if since is not None: + since_token = StreamToken.from_string(since) else: - pass - # now_stream_token = get_current_stream_token - # for room_id in get_rooms_for_user(user, filter=filter) - # state, events, start, end, limited, published = updates_for_room( - # from=since, to=now_stream_token, limit=limit, - # anchor_to_start=False - # ) - # next_stream_token = min(next_stream_token, end) + since_token = None + sync_result = yield self.sync_handler.wait_for_sync_for_user( + sync_config, since_token=since_token, timeout=timeout + ) - response_content = {} + time_now = self.clock.time_msec() + + response_content = { + "public_user_data": self.encode_events( + sync_result.public_user_data, filter, time_now + ), + "private_user_data": self.encode_events( + sync_result.private_user_data, filter, time_now + ), + "rooms": self.encode_rooms(sync_result.rooms, filter, time_now), + "next_batch": sync_result.next_batch.to_string(), + } defer.returnValue((200, response_content)) + def encode_events(self, events, filter, time_now): + return [self.encode_event(event, filter, time_now) for event in events] + + @staticmethod + def encode_event(event, filter, time_now): + # TODO(mjark): Respect formatting requirements in the filter. + return serialize_event(event, time_now) + + def encode_rooms(self, rooms, filter, time_now): + return [self.encode_room(room, filter, time_now) for room in rooms] + + @staticmethod + def encode_room(room, filter, time_now): + event_map = {} + state_event_ids = [] + recent_event_ids = [] + for event in room.state: + # TODO(mjark): Respect formatting requirements in the filter. + event_map[event.event_id] = serialize_event( + event, time_now, strip_ids=True + ) + state_event_ids.append(event.event_id) + + for event in room.events: + # TODO(mjark): Respect formatting requirements in the filter. + event_map[event.event_id] = serialize_event( + event, time_now, strip_ids=True + ) + recent_event_ids.append(event.event_id) + return { + "room_id": room.room_id, + "event_map": event_map, + "events": { + "batch": recent_event_ids, + "prev_batch": room.prev_batch.to_string(), + }, + "state": state_event_ids, + "limited": room.limited, + "published": room.published, + } + def register_servlets(hs, http_server): SyncRestServlet(hs).register(http_server) From 39c1892b22012e20ce6f43e92b01f6fad780081d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 13:03:31 +0000 Subject: [PATCH 074/630] Minor changes to v2_alpha filter REST test to allow the setUp method to be shareable --- tests/rest/client/v2_alpha/test_filter.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 1d1273ab9..91b19e88f 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -25,11 +25,12 @@ from synapse.rest.client.v2_alpha import filter from synapse.types import UserID -myid = "@apple:test" PATH_PREFIX = "/_matrix/client/v2_alpha" class FilterTestCase(unittest.TestCase): + USER_ID = "@apple:test" + TO_REGISTER = [filter] def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) @@ -50,25 +51,26 @@ class FilterTestCase(unittest.TestCase): def _get_user_by_token(token=None): return { - "user": UserID.from_string(myid), + "user": UserID.from_string(self.USER_ID), "admin": False, "device_id": None, } hs.get_auth().get_user_by_token = _get_user_by_token - filter.register_servlets(hs, self.mock_resource) + for r in self.TO_REGISTER: + r.register_servlets(hs, self.mock_resource) @defer.inlineCallbacks def test_filter(self): (code, response) = yield self.mock_resource.trigger("POST", - "/user/%s/filter" % (myid), + "/user/%s/filter" % (self.USER_ID), '{"type": ["m.*"]}' ) self.assertEquals(200, code) self.assertEquals({"filter_id": "0"}, response) (code, response) = yield self.mock_resource.trigger("GET", - "/user/%s/filter/0" % (myid), None + "/user/%s/filter/0" % (self.USER_ID), None ) self.assertEquals(200, code) self.assertEquals({"type": ["m.*"]}, response) From 57d2bfca3fc52d9e7fc80577854333b482eeae02 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 13:09:57 +0000 Subject: [PATCH 075/630] Initial cut of a shared base class for REST unit tests --- tests/rest/client/v2_alpha/__init__.py | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index 803f97ea4..f59745e13 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -13,3 +13,48 @@ # See the License for the specific language governing permissions and # limitations under the License. +from tests import unittest + +from mock import Mock + +from ....utils import MockHttpResource, MockKey + +from synapse.server import HomeServer +from synapse.types import UserID + + +PATH_PREFIX = "/_matrix/client/v2_alpha" + + +class V2AlphaRestTestCase(unittest.TestCase): + # Consumer must define + # USER_ID = + # TO_REGISTER = [] + + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + + mock_config = Mock() + mock_config.signing_key = [MockKey()] + + hs = HomeServer("test", + db_pool=None, + datastore=Mock(spec=[ + "insert_client_ip", + ]), + http_client=None, + resource_for_client=self.mock_resource, + resource_for_federation=self.mock_resource, + config=mock_config, + ) + + def _get_user_by_token(token=None): + return { + "user": UserID.from_string(self.USER_ID), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + for r in self.TO_REGISTER: + r.register_servlets(hs, self.mock_resource) From f9958f34043bf4fdf5923f471f193b40188e67bb Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 13:17:25 +0000 Subject: [PATCH 076/630] Use new V2AlphaRestTestCase --- tests/rest/client/v2_alpha/test_filter.py | 40 ++--------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 91b19e88f..8629a1aed 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -13,53 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from tests import unittest from twisted.internet import defer -from mock import Mock +from . import V2AlphaRestTestCase -from ....utils import MockHttpResource, MockKey - -from synapse.server import HomeServer from synapse.rest.client.v2_alpha import filter -from synapse.types import UserID -PATH_PREFIX = "/_matrix/client/v2_alpha" - - -class FilterTestCase(unittest.TestCase): +class FilterTestCase(V2AlphaRestTestCase): USER_ID = "@apple:test" TO_REGISTER = [filter] - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - - mock_config = Mock() - mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - db_pool=None, - datastore=Mock(spec=[ - "insert_client_ip", - ]), - http_client=None, - resource_for_client=self.mock_resource, - resource_for_federation=self.mock_resource, - config=mock_config, - ) - - def _get_user_by_token(token=None): - return { - "user": UserID.from_string(self.USER_ID), - "admin": False, - "device_id": None, - } - hs.get_auth().get_user_by_token = _get_user_by_token - - for r in self.TO_REGISTER: - r.register_servlets(hs, self.mock_resource) - @defer.inlineCallbacks def test_filter(self): (code, response) = yield self.mock_resource.trigger("POST", From fa8e6ff90074708ec19fcdcdfd79c1dce03cb128 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 14:01:51 +0000 Subject: [PATCH 077/630] Add stub application services REST API. --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 7 +++- synapse/rest/appservice/__init__.py | 14 ++++++++ synapse/rest/appservice/v1/__init__.py | 29 ++++++++++++++++ synapse/rest/appservice/v1/base.py | 47 ++++++++++++++++++++++++++ synapse/rest/appservice/v1/register.py | 37 ++++++++++++++++++++ synapse/server.py | 1 + 7 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/appservice/__init__.py create mode 100644 synapse/rest/appservice/v1/__init__.py create mode 100644 synapse/rest/appservice/v1/base.py create mode 100644 synapse/rest/appservice/v1/register.py diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 693c0efda..948571933 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -22,3 +22,4 @@ WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" MEDIA_PREFIX = "/_matrix/media/v1" +APP_SERVICE_PREFIX = "/_matrix/appservice/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 40d28dcbd..f7e24d0cc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,13 +26,14 @@ from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect +from synapse.rest.appservice.v1 import AppServiceRestResource from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, APP_SERVICE_PREFIX ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -69,6 +70,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_federation(self): return JsonResource() + def build_resource_for_app_services(self): + return AppServiceRestResource(self) + def build_resource_for_web_client(self): syweb_path = os.path.dirname(syweb.__file__) webclient_path = os.path.join(syweb_path, "webclient") @@ -114,6 +118,7 @@ class SynapseHomeServer(HomeServer): (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), + (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), ] if web_client: logger.info("Adding the web client.") diff --git a/synapse/rest/appservice/__init__.py b/synapse/rest/appservice/__init__.py new file mode 100644 index 000000000..1a84d94cd --- /dev/null +++ b/synapse/rest/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py new file mode 100644 index 000000000..bf243b618 --- /dev/null +++ b/synapse/rest/appservice/v1/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from . import register + +from synapse.http.server import JsonResource + + +class AppServiceRestResource(JsonResource): + """A resource for version 1 of the matrix application service API.""" + + def __init__(self, hs): + JsonResource.__init__(self) + self.register_servlets(self, hs) + + @staticmethod + def register_servlets(appservice_resource, hs): + register.register_servlets(hs, appservice_resource) \ No newline at end of file diff --git a/synapse/rest/appservice/v1/base.py b/synapse/rest/appservice/v1/base.py new file mode 100644 index 000000000..46c9a444c --- /dev/null +++ b/synapse/rest/appservice/v1/base.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +from synapse.http.servlet import RestServlet +from synapse.api.urls import APP_SERVICE_PREFIX +import re + +import logging + + +logger = logging.getLogger(__name__) + + +def as_path_pattern(path_regex): + """Creates a regex compiled appservice path with the correct path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + SRE_Pattern + """ + return re.compile("^" + APP_SERVICE_PREFIX + path_regex) + + +class AppServiceRestServlet(RestServlet): + """A base Synapse REST Servlet for the application services version 1 API. + """ + + def __init__(self, hs): + self.hs = hs \ No newline at end of file diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py new file mode 100644 index 000000000..17b6dc566 --- /dev/null +++ b/synapse/rest/appservice/v1/register.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains REST servlets to do with registration: /register""" + +from base import AppServiceRestServlet, as_path_pattern +from synapse.api.errors import CodeMessageException + +import logging + +logger = logging.getLogger(__name__) + + +class RegisterRestServlet(AppServiceRestServlet): + """Handles AS registration with the home server. + """ + + PATTERN = as_path_pattern("/register$") + + def on_POST(self, request): + raise CodeMessageException(500, "Not implemented") + + +def register_servlets(hs, http_server): + RegisterRestServlet(hs).register(http_server) \ No newline at end of file diff --git a/synapse/server.py b/synapse/server.py index f09d5d581..891c5aa13 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -75,6 +75,7 @@ class BaseHomeServer(object): 'resource_for_content_repo', 'resource_for_server_key', 'resource_for_media_repository', + 'resource_for_app_services', 'event_sources', 'ratelimiter', 'keyring', From 05c7cba73a050f19cc52129b65b0183eaa832a42 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 14:28:56 +0000 Subject: [PATCH 078/630] Initial trivial implementation of an actual 'Filtering' object; move storage of user filters into there --- synapse/api/filtering.py | 41 ++++++++++++++++++++++++++ synapse/rest/client/v2_alpha/filter.py | 25 ++++++++-------- synapse/server.py | 5 ++++ 3 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 synapse/api/filtering.py diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py new file mode 100644 index 000000000..922c40004 --- /dev/null +++ b/synapse/api/filtering.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# TODO(paul) +_filters_for_user = {} + + +class Filtering(object): + + def __init__(self, hs): + super(Filtering, self).__init__() + self.hs = hs + + def get_user_filter(self, user_localpart, filter_id): + filters = _filters_for_user.get(user_localpart, None) + + if not filters or filter_id >= len(filters): + raise KeyError() + + return filters[filter_id] + + def add_user_filter(self, user_localpart, definition): + filters = _filters_for_user.setdefault(user_localpart, []) + + filter_id = len(filters) + filters.append(definition) + + return filter_id diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index a9a180ec0..585c8e02e 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -28,10 +28,6 @@ import logging logger = logging.getLogger(__name__) -# TODO(paul) -_filters_for_user = {} - - class GetFilterRestServlet(RestServlet): PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter/(?P[^/]*)") @@ -39,6 +35,7 @@ class GetFilterRestServlet(RestServlet): super(GetFilterRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_GET(self, request, user_id, filter_id): @@ -56,13 +53,14 @@ class GetFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter_id") - filters = _filters_for_user.get(target_user.localpart, None) - - if not filters or filter_id >= len(filters): + try: + defer.returnValue((200, self.filtering.get_user_filter( + user_localpart=target_user.localpart, + filter_id=filter_id, + ))) + except KeyError: raise SynapseError(400, "No such filter") - defer.returnValue((200, filters[filter_id])) - class CreateFilterRestServlet(RestServlet): PATTERN = client_v2_pattern("/user/(?P[^/]*)/filter") @@ -71,6 +69,7 @@ class CreateFilterRestServlet(RestServlet): super(CreateFilterRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_POST(self, request, user_id): @@ -90,10 +89,10 @@ class CreateFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter definition") - filters = _filters_for_user.setdefault(target_user.localpart, []) - - filter_id = len(filters) - filters.append(content) + filter_id = self.filtering.add_user_filter( + user_localpart=target_user.localpart, + definition=content, + ) defer.returnValue((200, {"filter_id": str(filter_id)})) diff --git a/synapse/server.py b/synapse/server.py index f09d5d581..9b42079e0 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -32,6 +32,7 @@ from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter from synapse.crypto.keyring import Keyring from synapse.events.builder import EventBuilderFactory +from synapse.api.filtering import Filtering class BaseHomeServer(object): @@ -79,6 +80,7 @@ class BaseHomeServer(object): 'ratelimiter', 'keyring', 'event_builder_factory', + 'filtering', ] def __init__(self, hostname, **kwargs): @@ -197,3 +199,6 @@ class HomeServer(BaseHomeServer): clock=self.get_clock(), hostname=self.hostname, ) + + def build_filtering(self): + return Filtering(self) From 6efdc11cc811fedd5c989dc06df64fd334b4b033 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 15:03:19 +0000 Subject: [PATCH 079/630] Parse /register and /unregister request JSON. --- synapse/rest/appservice/v1/register.py | 80 +++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 17b6dc566..779447ac6 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -16,8 +16,9 @@ """This module contains REST servlets to do with registration: /register""" from base import AppServiceRestServlet, as_path_pattern -from synapse.api.errors import CodeMessageException +from synapse.api.errors import CodeMessageException, SynapseError +import json import logging logger = logging.getLogger(__name__) @@ -30,8 +31,83 @@ class RegisterRestServlet(AppServiceRestServlet): PATTERN = as_path_pattern("/register$") def on_POST(self, request): + params = _parse_json(request) + + # sanity check required params + try: + as_token = params["as_token"] + as_url = params["url"] + if (not isinstance(as_token, basestring) or + not isinstance(as_url, basestring)): + raise ValueError + except (KeyError, ValueError): + raise SynapseError( + 400, "Missed required keys: as_token(str) / url(str)." + ) + + namespaces = { + "users": [], + "rooms": [], + "aliases": [] + } + + if "namespaces" in params: + self._parse_namespace(namespaces, params["namespaces"], "users") + self._parse_namespace(namespaces, params["namespaces"], "rooms") + self._parse_namespace(namespaces, params["namespaces"], "aliases") + + # TODO: pass to the appservice handler + + raise CodeMessageException(500, "Not implemented.") + + def _parse_namespace(self, target_ns, origin_ns, ns): + if ns not in target_ns or ns not in origin_ns: + return # nothing to parse / map through to. + + possible_regex_list = origin_ns[ns] + if not type(possible_regex_list) == list: + raise SynapseError(400, "Namespace %s isn't an array." % ns) + + for regex in possible_regex_list: + if not isinstance(regex, basestring): + raise SynapseError( + 400, "Regex '%s' isn't a string in namespace %s" % + (regex, ns) + ) + + target_ns[ns] = origin_ns[ns] + + +class UnregisterRestServlet(AppServiceRestServlet): + """Handles AS registration with the home server. + """ + + PATTERN = as_path_pattern("/unregister$") + + def on_POST(self, request): + params = _parse_json(request) + try: + as_token = params["as_token"] + if not isinstance(as_token, basestring): + raise ValueError + except (KeyError, ValueError): + raise SynapseError(400, "Missing required key: as_token(str)") + + # TODO: pass to the appservice handler + raise CodeMessageException(500, "Not implemented") +def _parse_json(request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except ValueError: + raise SynapseError(400, "Content not JSON.") + + def register_servlets(hs, http_server): - RegisterRestServlet(hs).register(http_server) \ No newline at end of file + RegisterRestServlet(hs).register(http_server) + UnregisterRestServlet(hs).register(http_server) From 51449e06654c4af7a645124dc64e1f0cc1678b24 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 15:50:28 +0000 Subject: [PATCH 080/630] Add appservice handler and store. Glue together rest > handler > store. --- synapse/handlers/__init__.py | 2 ++ synapse/handlers/appservice.py | 49 ++++++++++++++++++++++++++ synapse/rest/appservice/v1/base.py | 3 +- synapse/rest/appservice/v1/register.py | 8 +++-- synapse/storage/__init__.py | 5 +-- synapse/storage/appservice.py | 45 +++++++++++++++++++++++ 6 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 synapse/handlers/appservice.py create mode 100644 synapse/storage/appservice.py diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index fe071a4bc..96a9b143c 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -26,6 +26,7 @@ from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler from .admin import AdminHandler +from .appservice import ApplicationServicesHandler class Handlers(object): @@ -51,3 +52,4 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.appservice_handler = ApplicationServicesHandler(hs) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py new file mode 100644 index 000000000..55a653476 --- /dev/null +++ b/synapse/handlers/appservice.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import BaseHandler +from synapse.api.errors import StoreError, SynapseError + +import logging + + +logger = logging.getLogger(__name__) + + +class ApplicationServicesHandler(BaseHandler): + + def __init__(self, hs): + super(ApplicationServicesHandler, self).__init__(hs) + + @defer.inlineCallbacks + def register(self, base_url, token, namespaces): + # check the token is recognised + try: + app_service = yield self.store.get_app_service(token) + if not app_service: + raise StoreError + except StoreError: + raise SynapseError( + 403, "Unrecognised application services token. " + "Consult the home server admin." + ) + + # update AS entry with base URL + + # store namespaces for this AS + + defer.returnValue("not_implemented_yet") diff --git a/synapse/rest/appservice/v1/base.py b/synapse/rest/appservice/v1/base.py index 46c9a444c..65d5bcf9b 100644 --- a/synapse/rest/appservice/v1/base.py +++ b/synapse/rest/appservice/v1/base.py @@ -44,4 +44,5 @@ class AppServiceRestServlet(RestServlet): """ def __init__(self, hs): - self.hs = hs \ No newline at end of file + self.hs = hs + self.handler = hs.get_handlers().appservice_handler diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 779447ac6..142f09a63 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -14,6 +14,7 @@ # limitations under the License. """This module contains REST servlets to do with registration: /register""" +from twisted.internet import defer from base import AppServiceRestServlet, as_path_pattern from synapse.api.errors import CodeMessageException, SynapseError @@ -30,6 +31,7 @@ class RegisterRestServlet(AppServiceRestServlet): PATTERN = as_path_pattern("/register$") + @defer.inlineCallbacks def on_POST(self, request): params = _parse_json(request) @@ -56,9 +58,11 @@ class RegisterRestServlet(AppServiceRestServlet): self._parse_namespace(namespaces, params["namespaces"], "rooms") self._parse_namespace(namespaces, params["namespaces"], "aliases") - # TODO: pass to the appservice handler + hs_token = yield self.handler.register(as_url, as_token, namespaces) - raise CodeMessageException(500, "Not implemented.") + defer.returnValue({ + "hs_token": hs_token + }) def _parse_namespace(self, target_ns, origin_ns, ns): if ns not in target_ns or ns not in origin_ns: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9..9431c1a32 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.api.constants import EventTypes +from .appservice import ApplicationServiceStore from .directory import DirectoryStore from .feedback import FeedbackStore from .presence import PresenceStore @@ -80,8 +81,8 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, - EventFederationStore, - MediaRepositoryStore, + EventFederationStore, MediaRepositoryStore, + ApplicationServiceStore ): def __init__(self, hs): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py new file mode 100644 index 000000000..99f58b4c6 --- /dev/null +++ b/synapse/storage/appservice.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import StoreError + +from ._base import SQLBaseStore + + +class ApplicationServiceStore(SQLBaseStore): + + def __init__(self, hs): + super(ApplicationServiceStore, self).__init__(hs) + + self.clock = hs.get_clock() + + @defer.inlineCallbacks + def get_app_service(self, as_token): + """Get the application service with the given token. + + Args: + token (str): The application service token. + Raises: + StoreError if there was a problem retrieving this. + """ + row = self._simple_select_one( + "application_services", {"token": as_token}, + ["url", "token"] + ) + if not row: + raise StoreError(400, "Bad application services token supplied.") + defer.returnValue(row) From b1503112ce77e573aa8cfb7581ca4a916c7d018c Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 15:56:14 +0000 Subject: [PATCH 081/630] Initial trivial unittest of Filtering object --- tests/api/test_filtering.py | 67 +++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/api/test_filtering.py diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py new file mode 100644 index 000000000..c6c531769 --- /dev/null +++ b/tests/api/test_filtering.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from mock import Mock, NonCallableMock +from tests.utils import ( + MockHttpResource, MockClock, DeferredMockCallable, SQLiteMemoryDbPool, + MockKey +) + +from synapse.server import HomeServer + + +user_localpart = "test_user" + +class FilteringTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + self.mock_federation_resource = MockHttpResource() + + self.mock_http_client = Mock(spec=[]) + self.mock_http_client.put_json = DeferredMockCallable() + + hs = HomeServer("test", + db_pool=db_pool, + handlers=None, + http_client=self.mock_http_client, + config=self.mock_config, + keyring=Mock(), + ) + + self.filtering = hs.get_filtering() + + def test_filter(self): + filter_id = self.filtering.add_user_filter( + user_localpart=user_localpart, + definition={"type": ["m.*"]}, + ) + self.assertEquals(filter_id, 0) + + filter = self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, + ) + self.assertEquals(filter, {"type": ["m.*"]}) From 1d779691248be2fcdd43e97352d13c2cef239c10 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 15:58:27 +0000 Subject: [PATCH 082/630] Unbreak bad presence merge - don't add these blocks together with an and: they're different things. --- synapse/handlers/events.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 75fb94100..f1a3e4a4a 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -51,19 +51,21 @@ class EventStreamHandler(BaseHandler): auth_user = self.hs.parse_userid(auth_user_id) try: - if affect_presence and auth_user not in self._streams_per_user: - self._streams_per_user[auth_user] = 0 - if auth_user in self._stop_timer_per_user: - try: - self.clock.cancel_call_later( - self._stop_timer_per_user.pop(auth_user) + if affect_presence: + if auth_user not in self._streams_per_user: + self._streams_per_user[auth_user] = 0 + if auth_user in self._stop_timer_per_user: + try: + print "cancel",auth_user + self.clock.cancel_call_later( + self._stop_timer_per_user.pop(auth_user) + ) + except: + logger.exception("Failed to cancel event timer") + else: + yield self.distributor.fire( + "started_user_eventstream", auth_user ) - except: - logger.exception("Failed to cancel event timer") - else: - yield self.distributor.fire( - "started_user_eventstream", auth_user - ) self._streams_per_user[auth_user] += 1 if pagin_config.from_token is None: From eba89f093f0376b8499953b53bbd85f9b12b7852 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:00:07 +0000 Subject: [PATCH 083/630] Need a defer.inlineCallbacks here as we yield in it: otherwise nothing in the cb gets executed. --- synapse/handlers/events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index f1a3e4a4a..851eebf60 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -99,6 +99,7 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone + @defer.inlineCallbacks def _later(): logger.debug( "_later stopped_user_eventstream %s", auth_user From 5eacaeb4a73aa4c4ff450d7b23acabe79124171f Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:05:23 +0000 Subject: [PATCH 084/630] or of course we could just return the deferred --- synapse/handlers/events.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 851eebf60..48de3630e 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -99,7 +99,6 @@ class EventStreamHandler(BaseHandler): # 10 seconds of grace to allow the client to reconnect again # before we think they're gone - @defer.inlineCallbacks def _later(): logger.debug( "_later stopped_user_eventstream %s", auth_user @@ -107,7 +106,7 @@ class EventStreamHandler(BaseHandler): self._stop_timer_per_user.pop(auth_user, None) - yield self.distributor.fire( + return self.distributor.fire( "stopped_user_eventstream", auth_user ) From f7c4daa8f94e5968277ba438f6d3da1f0f27ba4c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Jan 2015 16:08:47 +0000 Subject: [PATCH 085/630] Oops, remove debugging --- synapse/handlers/events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 48de3630e..52ef07eaa 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -56,7 +56,6 @@ class EventStreamHandler(BaseHandler): self._streams_per_user[auth_user] = 0 if auth_user in self._stop_timer_per_user: try: - print "cancel",auth_user self.clock.cancel_call_later( self._stop_timer_per_user.pop(auth_user) ) From 059651efa19a88eb0823bce1d5beff2d95cb01c2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 16:17:56 +0000 Subject: [PATCH 086/630] Have the Filtering API return Deferreds, so we can do the Datastore implementation nicely --- synapse/api/filtering.py | 16 ++++++++++++++-- synapse/rest/client/v2_alpha/filter.py | 8 +++++--- tests/api/test_filtering.py | 5 +++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 922c40004..014e2e1fc 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + # TODO(paul) _filters_for_user = {} @@ -24,18 +26,28 @@ class Filtering(object): super(Filtering, self).__init__() self.hs = hs + @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): filters = _filters_for_user.get(user_localpart, None) if not filters or filter_id >= len(filters): raise KeyError() - return filters[filter_id] + # trivial yield to make it a generator so d.iC works + yield + defer.returnValue(filters[filter_id]) + @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): filters = _filters_for_user.setdefault(user_localpart, []) filter_id = len(filters) filters.append(definition) - return filter_id + # trivial yield, see above + yield + defer.returnValue(filter_id) + + # TODO(paul): surely we should probably add a delete_user_filter or + # replace_user_filter at some point? There's no REST API specified for + # them however diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 585c8e02e..09e44e8ae 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -54,10 +54,12 @@ class GetFilterRestServlet(RestServlet): raise SynapseError(400, "Invalid filter_id") try: - defer.returnValue((200, self.filtering.get_user_filter( + filter = yield self.filtering.get_user_filter( user_localpart=target_user.localpart, filter_id=filter_id, - ))) + ) + + defer.returnValue((200, filter)) except KeyError: raise SynapseError(400, "No such filter") @@ -89,7 +91,7 @@ class CreateFilterRestServlet(RestServlet): except: raise SynapseError(400, "Invalid filter definition") - filter_id = self.filtering.add_user_filter( + filter_id = yield self.filtering.add_user_filter( user_localpart=target_user.localpart, definition=content, ) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index c6c531769..fecadd105 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -53,14 +53,15 @@ class FilteringTestCase(unittest.TestCase): self.filtering = hs.get_filtering() + @defer.inlineCallbacks def test_filter(self): - filter_id = self.filtering.add_user_filter( + filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, definition={"type": ["m.*"]}, ) self.assertEquals(filter_id, 0) - filter = self.filtering.get_user_filter( + filter = yield self.filtering.get_user_filter( user_localpart=user_localpart, filter_id=filter_id, ) From 7331d34839dca468b9e396e1d2952b0bb32011bf Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 16:23:46 +0000 Subject: [PATCH 087/630] Add AS specific classes with docstrings. --- synapse/storage/appservice.py | 60 ++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 99f58b4c6..4c11191fe 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -20,11 +20,69 @@ from synapse.api.errors import StoreError from ._base import SQLBaseStore +# XXX: This feels like it should belong in a "models" module, not storage. +class ApplicationService(object): + """Defines an application service. + + Provides methods to check if this service is "interested" in events. + """ + + def __init__(self, token, url=None, namespaces=None): + self.token = token + if url: + self.url = url + if namespaces: + self.namespaces = namespaces + + def is_interested(self, event): + """Check if this service is interested in this event. + + Args: + event(Event): The event to check. + Returns: + bool: True if this service would like to know about this event. + """ + # NB: This does not check room alias regex matches because that requires + # more context that an Event can provide. Room alias matches are checked + # in the ApplicationServiceHandler. + + # TODO check if event.room_id regex matches + # TODO check if event.user_id regex matches (or m.room.member state_key) + + return True + + +class ApplicationServiceCache(object): + """Caches ApplicationServices and provides utility functions on top. + + This class is designed to be invoked on incoming events in order to avoid + hammering the database every time to extract a list of application service + regexes. + """ + + def __init__(self): + self.services = [] + + def get_services_for_event(self, event): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. + Returns: + list: A list of services interested in this + event based on the service regex. + """ + interested_list = [ + s for s in self.services if s.is_event_claimed(event) + ] + return interested_list + + class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) - + self.cache = ApplicationServiceCache() self.clock = hs.get_clock() @defer.inlineCallbacks From a56008842b43089433768f569f35b2d14523ac39 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 16:24:22 +0000 Subject: [PATCH 088/630] Start implementing incremental initial sync --- synapse/events/utils.py | 1 + synapse/handlers/sync.py | 233 ++++++++++++++++++++++++++++++++++---- synapse/storage/stream.py | 41 +++++-- 3 files changed, 241 insertions(+), 34 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index b7f1ad4b4..42fb0371e 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -137,6 +137,7 @@ def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): d.pop("depth", None) d.pop("unsigned", None) d.pop("origin", None) + d.pop("prev_state", None) if strip_ids: d.pop("room_id", None) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bbabaf3df..f8629a588 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,14 +37,18 @@ SyncConfig = collections.namedtuple("SyncConfig", [ ]) -RoomSyncResult = collections.namedtuple("RoomSyncResult", [ +class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "room_id", "limited", "published", - "events", # dict of event + "events", "state", "prev_batch", -]) +])): + __slots__ = [] + + def __nonzero__(self): + return bool(self.events or self.state) class SyncResult(collections.namedtuple("SyncResult", [ @@ -56,7 +60,9 @@ class SyncResult(collections.namedtuple("SyncResult", [ __slots__ = [] def __nonzero__(self): - return self.private_user_data or self.public_user_data or self.rooms + return bool( + self.private_user_data or self.public_user_data or self.rooms + ) class SyncHandler(BaseHandler): @@ -67,7 +73,13 @@ class SyncHandler(BaseHandler): self.clock = hs.get_clock() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): - if timeout == 0: + """Get the sync for a client if we have new data for it now. Otherwise + wait for new data to arrive on the server. If the timeout expires, then + return an empty sync result. + Returns: + A Deferred SyncResult. + """ + if timeout == 0 or since_token is None: return self.current_sync_for_user(sync_config, since_token) else: def current_sync_callback(since_token): @@ -79,13 +91,25 @@ class SyncHandler(BaseHandler): ) def current_sync_for_user(self, sync_config, since_token=None): + """Get the sync for client needed to match what the server has now. + Returns: + A Deferred SyncResult. + """ if since_token is None: return self.initial_sync(sync_config) else: - return self.incremental_sync(sync_config) + if sync_config.gap: + return self.incremental_sync_with_gap(sync_config, since_token) + else: + #TODO(mjark): Handle gapless sync + pass @defer.inlineCallbacks def initial_sync(self, sync_config): + """Get a sync for a client which is starting without any state + Returns: + A Deferred SyncResult. + """ if sync_config.sort == "timeline,desc": # TODO(mjark): Handle going through events in reverse order?. # What does "most recent events" mean when applying the limits mean @@ -114,25 +138,86 @@ class SyncHandler(BaseHandler): rooms = [] for event in room_list: - #TODO (mjark): Apply the event filter in sync_config. - recent_events, token = yield self.store.get_recent_events_for_room( - event.room_id, - limit=sync_config.limit, - end_token=now_token.room_key, - ) - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - current_state_events = yield self.state_handler.get_current_state( - event.room_id + room_sync = yield self.initial_sync_for_room( + event.room_id, sync_config, now_token, published_room_ids ) + rooms.append(room_sync) - rooms.append(RoomSyncResult( - room_id=event.room_id, - published=event.room_id in published_room_ids, - events=recent_events, - prev_batch=prev_batch_token, - state=current_state_events, - limited=True, - )) + defer.returnValue(SyncResult( + public_user_data=presence, + private_user_data=[], + rooms=rooms, + next_batch=now_token, + )) + + @defer.inlineCallbacks + def intial_sync_for_room(self, room_id, sync_config, now_token, + published_room_ids): + """Sync a room for a client which is starting without any state + Returns: + A Deferred RoomSyncResult. + """ + recent_events, token = yield self.store.get_recent_events_for_room( + room_id, + limit=sync_config.limit, + end_token=now_token.room_key, + ) + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state_events = yield self.state_handler.get_current_state( + room_id + ) + + defer.returnValue(RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recent_events, + prev_batch=prev_batch_token, + state=current_state_events, + limited=True, + )) + + + @defer.inlineCallbacks + def incremental_sync_with_gap(self, sync_config, since_token): + """ Get the incremental delta needed to bring the client up to + date with the server. + Returns: + A Deferred SyncResult. + """ + if sync_config.sort == "timeline,desc": + # TODO(mjark): Handle going through events in reverse order?. + # What does "most recent events" mean when applying the limits mean + # in this case? + raise NotImplementedError() + + now_token = yield self.event_sources.get_current_token() + + presence_stream = self.event_sources.sources["presence"] + pagination_config = PaginationConfig( + from_token=since_token, to_token=now_token + ) + presence, _ = yield presence_stream.get_pagination_rows( + user=sync_config.user, + pagination_config=pagination_config.get_source_config("presence"), + key=None + ) + room_list = yield self.store.get_rooms_for_user_where_membership_is( + user_id=sync_config.user.to_string(), + membership_list=[Membership.INVITE, Membership.JOIN] + ) + + # TODO (mjark): Does public mean "published"? + published_rooms = yield self.store.get_rooms(is_public=True) + published_room_ids = set(r["room_id"] for r in published_rooms) + + rooms = [] + for event in room_list: + room_sync = yield self.incremental_sync_with_gap_for_room( + event.room_id, sync_config, since_token, now_token, + published_room_ids + ) + if room_sync: + rooms.append(room_sync) defer.returnValue(SyncResult( public_user_data=presence, @@ -143,5 +228,103 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks - def incremental_sync(self, sync_config): - pass + def incremental_sync_with_gap_for_room(self, room_id, sync_config, + since_token, now_token, + published_room_ids): + """ Get the incremental delta needed to bring the client up to date for + the room. Gives the client the most recent events and the changes to + state. + Returns: + A Deferred RoomSyncResult + """ + # TODO(mjark): Check if they have joined the room between + # the previous sync and this one. + # TODO(mjark): Apply the event filter in sync_config + # TODO(mjark): Check for redactions we might have missed. + # TODO(mjark): Typing notifications. + recents, token = yield self.store.get_recent_events_for_room( + room_id, + limit=sync_config.limit + 1, + from_token=since_token.room_key, + end_token=now_token.room_key, + ) + + logging.debug("Recents %r", recents) + + if len(recents) > sync_config.limit: + limited = True + recents = recents[1:] + else: + limited = False + + prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + + # TODO(mjark): This seems racy since this isn't being passed a + # token to indicate what point in the stream this is + current_state_events = yield self.state_handler.get_current_state( + room_id + ) + + state_at_previous_sync = yield self.get_state_at_previous_sync( + room_id, since_token=since_token + ) + + state_events_delta = yield self.compute_state_delta( + since_token=since_token, + previous_state=state_at_previous_sync, + current_state=current_state_events, + ) + + room_sync = RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recents, + prev_batch=prev_batch_token, + state=state_events_delta, + limited=limited, + ) + + logging.debug("Room sync: %r", room_sync) + + defer.returnValue(room_sync) + + @defer.inlineCallbacks + def get_state_at_previous_sync(self, room_id, since_token): + """ Get the room state at the previous sync the client made. + Returns: + A Deferred list of Events. + """ + last_events, token = yield self.store.get_recent_events_for_room( + room_id, end_token=since_token.room_key, limit=1, + ) + + if last_events: + last_event = last_events[0] + last_context = yield self.state_handler.compute_event_context( + last_event + ) + if last_event.is_state(): + state = [last_event] + last_context.current_state.values() + else: + state = last_context.current_state.values() + else: + state = () + defer.returnValue(state) + + + def compute_state_delta(self, since_token, previous_state, current_state): + """ Works out the differnce in state between the current state and the + state the client got when it last performed a sync. + Returns: + A list of events. + """ + # TODO(mjark) Check if the state events were received by the server + # after the previous sync, since we need to include those state + # updates even if they occured logically before the previous event. + # TODO(mjark) Check for new redactions in the state events. + previous_dict = {event.event_id:event for event in previous_state} + state_delta = [] + for event in current_state: + if event.event_id not in previous_dict: + state_delta.append(event) + return state_delta diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8ac2adab0..06aca1a4e 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -265,17 +265,38 @@ class StreamStore(SQLBaseStore): return self.runInteraction("paginate_room_events", f) def get_recent_events_for_room(self, room_id, limit, end_token, - with_feedback=False): + with_feedback=False, from_token=None): # TODO (erikj): Handle compressed feedback - sql = ( - "SELECT stream_ordering, topological_ordering, event_id FROM events " - "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " - "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) + end_token = _StreamToken.parse_stream_token(end_token) - def f(txn): - txn.execute(sql, (room_id, end_token, limit,)) + if from_token is None: + sql = ( + "SELECT stream_ordering, topological_ordering, event_id" + " FROM events" + " WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0" + " ORDER BY topological_ordering DESC, stream_ordering DESC" + " LIMIT ?" + ) + else: + from_token = _StreamToken.parse_stream_token(from_token) + sql = ( + "SELECT stream_ordering, topological_ordering, event_id" + " FROM events" + " WHERE room_id = ? AND stream_ordering > ?" + " AND stream_ordering <= ? AND outlier = 0" + " ORDER BY topological_ordering DESC, stream_ordering DESC" + " LIMIT ?" + ) + + + def get_recent_events_for_room_txn(txn): + if from_token is None: + txn.execute(sql, (room_id, end_token.stream, limit,)) + else: + txn.execute(sql, ( + room_id, from_token.stream, end_token.stream, limit + )) rows = self.cursor_to_dict(txn) @@ -303,7 +324,9 @@ class StreamStore(SQLBaseStore): return events, token - return self.runInteraction("get_recent_events_for_room", f) + return self.runInteraction( + "get_recent_events_for_room", get_recent_events_for_room_txn + ) def get_room_events_max_id(self): return self.runInteraction( From 92171f9dd1ecac24aeae2f46729f3cbbbe94f91e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 16:53:59 +0000 Subject: [PATCH 089/630] Add stub methods, TODOs and docstrings for application services. --- synapse/handlers/appservice.py | 25 ++++++++++++++-- synapse/storage/appservice.py | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 55a653476..25e1cece5 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -42,8 +42,27 @@ class ApplicationServicesHandler(BaseHandler): "Consult the home server admin." ) - # update AS entry with base URL - - # store namespaces for this AS + # store this AS defer.returnValue("not_implemented_yet") + + def unregister(self, token): + yield self.store.unregister_app_service(token) + + def notify_interested_services(self, event): + """Notifies (pushes) all application services interested in this event. + + Pushing is done asynchronously, so this method won't block for any + prolonged length of time. + + Args: + event(Event): The event to push out to interested services. + """ + # TODO: Gather interested services + # get_services_for_event(event) <-- room IDs and user IDs + # Get a list of room aliases. Check regex. + # TODO: If unknown user: poke User Query API. + # TODO: If unknown room alias: poke Room Alias Query API. + + # TODO: Fork off pushes to these services - XXX First cut, best effort + pass diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 4c11191fe..fbad17cb9 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -84,16 +84,60 @@ class ApplicationServiceStore(SQLBaseStore): super(ApplicationServiceStore, self).__init__(hs) self.cache = ApplicationServiceCache() self.clock = hs.get_clock() + self._populate_cache() + + def unregister_app_service(self, token): + """Unregisters this service. + + This removes all AS specific regex and the base URL. The token is the + only thing preserved for future registration attempts. + """ + # TODO: DELETE FROM application_services_regex WHERE id=this service + # TODO: SET url=NULL WHERE token=token + # TODO: Update cache + pass + + def update_app_service(self, service): + """Update an application service, clobbering what was previously there. + + Args: + service(ApplicationService): The updated service. + """ + # NB: There is no "insert" since we provide no public-facing API to + # allocate new ASes. It relies on the server admin inserting the AS + # token into the database manually. + + # TODO: UPDATE application_services, SET url WHERE token=service.token + # TODO: DELETE FROM application_services_regex WHERE id=this service + # TODO: INSERT INTO application_services_regex + # TODO: Update cache + pass + + def get_services_for_event(self, event): + return self.cache.get_services_for_event(event) @defer.inlineCallbacks - def get_app_service(self, as_token): + def get_app_service(self, as_token, from_cache=True): """Get the application service with the given token. Args: token (str): The application service token. + from_cache (bool): True to get this service from the cache, False to + check the database. Raises: - StoreError if there was a problem retrieving this. + StoreError if there was a problem retrieving this service. """ + + if from_cache: + for service in self.cache.services: + if service.token == as_token: + defer.returnValue(service) + return + defer.returnValue(None) + return + + + # TODO: This should be JOINed with the application_services_regex table. row = self._simple_select_one( "application_services", {"token": as_token}, ["url", "token"] @@ -101,3 +145,7 @@ class ApplicationServiceStore(SQLBaseStore): if not row: raise StoreError(400, "Bad application services token supplied.") defer.returnValue(row) + + def _populate_cache(self): + """Populates the ApplicationServiceCache from the database.""" + pass From ec3719b583c6fbbc56dbd313b858054e535ae733 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 17:15:06 +0000 Subject: [PATCH 090/630] Use ApplicationService when registering. --- synapse/handlers/appservice.py | 13 +++++-------- synapse/rest/appservice/v1/register.py | 8 ++++++-- synapse/storage/appservice.py | 16 ++++++---------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 25e1cece5..1890ca06a 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -30,21 +30,18 @@ class ApplicationServicesHandler(BaseHandler): super(ApplicationServicesHandler, self).__init__(hs) @defer.inlineCallbacks - def register(self, base_url, token, namespaces): + def register(self, app_service): # check the token is recognised try: - app_service = yield self.store.get_app_service(token) - if not app_service: - raise StoreError + stored_service = yield self.store.get_app_service(app_service.token) + if not stored_service: + raise StoreError(404, "Not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " "Consult the home server admin." ) - - # store this AS - - defer.returnValue("not_implemented_yet") + # TODO store this AS def unregister(self, token): yield self.store.unregister_app_service(token) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 142f09a63..5786cf873 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -18,6 +18,7 @@ from twisted.internet import defer from base import AppServiceRestServlet, as_path_pattern from synapse.api.errors import CodeMessageException, SynapseError +from synapse.storage.appservice import ApplicationService import json import logging @@ -58,7 +59,10 @@ class RegisterRestServlet(AppServiceRestServlet): self._parse_namespace(namespaces, params["namespaces"], "rooms") self._parse_namespace(namespaces, params["namespaces"], "aliases") - hs_token = yield self.handler.register(as_url, as_token, namespaces) + app_service = ApplicationService(as_token, as_url, namespaces) + + yield self.handler.register(app_service) + hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? defer.returnValue({ "hs_token": hs_token @@ -97,7 +101,7 @@ class UnregisterRestServlet(AppServiceRestServlet): except (KeyError, ValueError): raise SynapseError(400, "Missing required key: as_token(str)") - # TODO: pass to the appservice handler + yield self.handler.unregister(as_token) raise CodeMessageException(500, "Not implemented") diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index fbad17cb9..f84f026b7 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -116,8 +116,7 @@ class ApplicationServiceStore(SQLBaseStore): def get_services_for_event(self, event): return self.cache.get_services_for_event(event) - @defer.inlineCallbacks - def get_app_service(self, as_token, from_cache=True): + def get_app_service(self, token, from_cache=True): """Get the application service with the given token. Args: @@ -130,21 +129,18 @@ class ApplicationServiceStore(SQLBaseStore): if from_cache: for service in self.cache.services: - if service.token == as_token: - defer.returnValue(service) - return - defer.returnValue(None) - return - + if service.token == token: + return service + return None # TODO: This should be JOINed with the application_services_regex table. row = self._simple_select_one( - "application_services", {"token": as_token}, + "application_services", {"token": token}, ["url", "token"] ) if not row: raise StoreError(400, "Bad application services token supplied.") - defer.returnValue(row) + return row def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" From fbeaeb868960099c3682802275d5a222c0cc2d8b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 27 Jan 2015 17:34:40 +0000 Subject: [PATCH 091/630] Log when ASes are registered/unregistered. --- synapse/handlers/appservice.py | 2 ++ synapse/storage/appservice.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 1890ca06a..c9f56c41e 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -31,6 +31,7 @@ class ApplicationServicesHandler(BaseHandler): @defer.inlineCallbacks def register(self, app_service): + logger.info("Register -> %s", app_service) # check the token is recognised try: stored_service = yield self.store.get_app_service(app_service.token) @@ -44,6 +45,7 @@ class ApplicationServicesHandler(BaseHandler): # TODO store this AS def unregister(self, token): + logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) def notify_interested_services(self, event): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index f84f026b7..cd15843ba 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -51,6 +51,9 @@ class ApplicationService(object): return True + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) + class ApplicationServiceCache(object): """Caches ApplicationServices and provides utility functions on top. @@ -83,7 +86,6 @@ class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.cache = ApplicationServiceCache() - self.clock = hs.get_clock() self._populate_cache() def unregister_app_service(self, token): From 54e513b4e6b5c644b9a2aeb02cef8258e87ae26a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 17:48:13 +0000 Subject: [PATCH 092/630] Move storage of user filters into real datastore layer; now have to mock it out in the REST-level tests --- synapse/api/filtering.py | 27 ++----------- synapse/storage/__init__.py | 3 +- synapse/storage/filtering.py | 46 +++++++++++++++++++++++ tests/rest/client/v2_alpha/__init__.py | 9 +++-- tests/rest/client/v2_alpha/test_filter.py | 21 +++++++++++ 5 files changed, 79 insertions(+), 27 deletions(-) create mode 100644 synapse/storage/filtering.py diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 014e2e1fc..20b6951d4 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -16,37 +16,18 @@ from twisted.internet import defer -# TODO(paul) -_filters_for_user = {} - - class Filtering(object): def __init__(self, hs): super(Filtering, self).__init__() - self.hs = hs + self.store = hs.get_datastore() - @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): - filters = _filters_for_user.get(user_localpart, None) + return self.store.get_user_filter(user_localpart, filter_id) - if not filters or filter_id >= len(filters): - raise KeyError() - - # trivial yield to make it a generator so d.iC works - yield - defer.returnValue(filters[filter_id]) - - @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): - filters = _filters_for_user.setdefault(user_localpart, []) - - filter_id = len(filters) - filters.append(definition) - - # trivial yield, see above - yield - defer.returnValue(filter_id) + # TODO(paul): implement sanity checking of the definition + return self.store.add_user_filter(user_localpart, definition) # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4beb951b9..efa63031b 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,9 +30,9 @@ from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore from .media_repository import MediaRepositoryStore - from .state import StateStore from .signatures import SignatureStore +from .filtering import FilteringStore from syutil.base64util import decode_base64 from syutil.jsonutil import encode_canonical_json @@ -82,6 +82,7 @@ class DataStore(RoomMemberStore, RoomStore, DirectoryStore, KeyStore, StateStore, SignatureStore, EventFederationStore, MediaRepositoryStore, + FilteringStore, ): def __init__(self, hs): diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py new file mode 100644 index 000000000..18e0e7c29 --- /dev/null +++ b/synapse/storage/filtering.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import SQLBaseStore + + +# TODO(paul) +_filters_for_user = {} + + +class FilteringStore(SQLBaseStore): + @defer.inlineCallbacks + def get_user_filter(self, user_localpart, filter_id): + filters = _filters_for_user.get(user_localpart, None) + + if not filters or filter_id >= len(filters): + raise KeyError() + + # trivial yield to make it a generator so d.iC works + yield + defer.returnValue(filters[filter_id]) + + @defer.inlineCallbacks + def add_user_filter(self, user_localpart, definition): + filters = _filters_for_user.setdefault(user_localpart, []) + + filter_id = len(filters) + filters.append(definition) + + # trivial yield, see above + yield + defer.returnValue(filter_id) diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index f59745e13..3fe62d5ac 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -39,9 +39,7 @@ class V2AlphaRestTestCase(unittest.TestCase): hs = HomeServer("test", db_pool=None, - datastore=Mock(spec=[ - "insert_client_ip", - ]), + datastore=self.make_datastore_mock(), http_client=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource, @@ -58,3 +56,8 @@ class V2AlphaRestTestCase(unittest.TestCase): for r in self.TO_REGISTER: r.register_servlets(hs, self.mock_resource) + + def make_datastore_mock(self): + return Mock(spec=[ + "insert_client_ip", + ]) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 8629a1aed..1add727e6 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from mock import Mock + from . import V2AlphaRestTestCase from synapse.rest.client.v2_alpha import filter @@ -24,6 +26,25 @@ class FilterTestCase(V2AlphaRestTestCase): USER_ID = "@apple:test" TO_REGISTER = [filter] + def make_datastore_mock(self): + datastore = super(FilterTestCase, self).make_datastore_mock() + + self._user_filters = {} + + def add_user_filter(user_localpart, definition): + filters = self._user_filters.setdefault(user_localpart, []) + filter_id = len(filters) + filters.append(definition) + return defer.succeed(filter_id) + datastore.add_user_filter = add_user_filter + + def get_user_filter(user_localpart, filter_id): + filters = self._user_filters[user_localpart] + return defer.succeed(filters[filter_id]) + datastore.get_user_filter = get_user_filter + + return datastore + @defer.inlineCallbacks def test_filter(self): (code, response) = yield self.mock_resource.trigger("POST", From 0c14a699bb4e103a1845b0808821138cfea99552 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 18:07:21 +0000 Subject: [PATCH 093/630] More unit-testing of REST errors --- tests/rest/client/v2_alpha/test_filter.py | 36 ++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 1add727e6..80ddabf81 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -21,6 +21,8 @@ from . import V2AlphaRestTestCase from synapse.rest.client.v2_alpha import filter +from synapse.api.errors import StoreError + class FilterTestCase(V2AlphaRestTestCase): USER_ID = "@apple:test" @@ -39,14 +41,18 @@ class FilterTestCase(V2AlphaRestTestCase): datastore.add_user_filter = add_user_filter def get_user_filter(user_localpart, filter_id): + if user_localpart not in self._user_filters: + raise StoreError(404, "No user") filters = self._user_filters[user_localpart] + if filter_id >= len(filters): + raise StoreError(404, "No filter") return defer.succeed(filters[filter_id]) datastore.get_user_filter = get_user_filter return datastore @defer.inlineCallbacks - def test_filter(self): + def test_add_filter(self): (code, response) = yield self.mock_resource.trigger("POST", "/user/%s/filter" % (self.USER_ID), '{"type": ["m.*"]}' @@ -54,8 +60,36 @@ class FilterTestCase(V2AlphaRestTestCase): self.assertEquals(200, code) self.assertEquals({"filter_id": "0"}, response) + self.assertIn("apple", self._user_filters) + self.assertEquals(len(self._user_filters["apple"]), 1) + self.assertEquals({"type": ["m.*"]}, self._user_filters["apple"][0]) + + @defer.inlineCallbacks + def test_get_filter(self): + self._user_filters["apple"] = [ + {"type": ["m.*"]} + ] + (code, response) = yield self.mock_resource.trigger("GET", "/user/%s/filter/0" % (self.USER_ID), None ) self.assertEquals(200, code) self.assertEquals({"type": ["m.*"]}, response) + + @defer.inlineCallbacks + def test_get_filter_no_id(self): + self._user_filters["apple"] = [ + {"type": ["m.*"]} + ] + + (code, response) = yield self.mock_resource.trigger("GET", + "/user/%s/filter/2" % (self.USER_ID), None + ) + self.assertEquals(404, code) + + @defer.inlineCallbacks + def test_get_filter_no_user(self): + (code, response) = yield self.mock_resource.trigger("GET", + "/user/%s/filter/0" % (self.USER_ID), None + ) + self.assertEquals(404, code) From 06cc1470129d443f71bfc81ba716f63b9505467d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 18:46:03 +0000 Subject: [PATCH 094/630] Initial stab at real SQL storage implementation of user filter definitions --- synapse/storage/__init__.py | 1 + synapse/storage/filtering.py | 49 ++++++++++++++++++++-------- synapse/storage/schema/filtering.sql | 24 ++++++++++++++ tests/api/test_filtering.py | 19 ++++++++++- 4 files changed, 78 insertions(+), 15 deletions(-) create mode 100644 synapse/storage/schema/filtering.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index efa63031b..7c5631d01 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -61,6 +61,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "media_repository", + "filtering", ] diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 18e0e7c29..e98eaf803 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -17,6 +17,8 @@ from twisted.internet import defer from ._base import SQLBaseStore +import json + # TODO(paul) _filters_for_user = {} @@ -25,22 +27,41 @@ _filters_for_user = {} class FilteringStore(SQLBaseStore): @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): - filters = _filters_for_user.get(user_localpart, None) + def_json = yield self._simple_select_one_onecol( + table="user_filters", + keyvalues={ + "user_id": user_localpart, + "filter_id": filter_id, + }, + retcol="definition", + allow_none=False, + ) - if not filters or filter_id >= len(filters): - raise KeyError() + defer.returnValue(json.loads(def_json)) - # trivial yield to make it a generator so d.iC works - yield - defer.returnValue(filters[filter_id]) - - @defer.inlineCallbacks def add_user_filter(self, user_localpart, definition): - filters = _filters_for_user.setdefault(user_localpart, []) + def_json = json.dumps(definition) - filter_id = len(filters) - filters.append(definition) + # Need an atomic transaction to SELECT the maximal ID so far then + # INSERT a new one + def _do_txn(txn): + sql = ( + "SELECT MAX(filter_id) FROM user_filters " + "WHERE user_id = ?" + ) + txn.execute(sql, (user_localpart,)) + max_id = txn.fetchone()[0] + if max_id is None: + filter_id = 0 + else: + filter_id = max_id + 1 - # trivial yield, see above - yield - defer.returnValue(filter_id) + sql = ( + "INSERT INTO user_filters (user_id, filter_id, definition)" + "VALUES(?, ?, ?)" + ) + txn.execute(sql, (user_localpart, filter_id, def_json)) + + return filter_id + + return self.runInteraction("add_user_filter", _do_txn) diff --git a/synapse/storage/schema/filtering.sql b/synapse/storage/schema/filtering.sql new file mode 100644 index 000000000..795aca4af --- /dev/null +++ b/synapse/storage/schema/filtering.sql @@ -0,0 +1,24 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + definition TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index fecadd105..149948374 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -53,16 +53,33 @@ class FilteringTestCase(unittest.TestCase): self.filtering = hs.get_filtering() + self.datastore = hs.get_datastore() + @defer.inlineCallbacks - def test_filter(self): + def test_add_filter(self): filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, definition={"type": ["m.*"]}, ) + self.assertEquals(filter_id, 0) + self.assertEquals({"type": ["m.*"]}, + (yield self.datastore.get_user_filter( + user_localpart=user_localpart, + filter_id=0, + )) + ) + + @defer.inlineCallbacks + def test_get_filter(self): + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + definition={"type": ["m.*"]}, + ) filter = yield self.filtering.get_user_filter( user_localpart=user_localpart, filter_id=filter_id, ) + self.assertEquals(filter, {"type": ["m.*"]}) From 8398f19bcea8fb0134b37efa303dc65b017d75ce Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 27 Jan 2015 19:00:09 +0000 Subject: [PATCH 095/630] Created schema delta --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v12.sql | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/v12.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7c5631d01..00a04f565 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql new file mode 100644 index 000000000..795aca4af --- /dev/null +++ b/synapse/storage/schema/delta/v12.sql @@ -0,0 +1,24 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + definition TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); From b19cf6a105ad87954e15cf01e60e14fec280db6d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 20:09:52 +0000 Subject: [PATCH 096/630] Wait for events if the incremental sync is empty and a timeout is given --- demo/start.sh | 2 +- synapse/handlers/sync.py | 19 ++++++++++------ synapse/notifier.py | 48 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index bb2248770..a7502e7a8 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,7 +16,7 @@ if [ $# -eq 1 ]; then fi fi -for port in 8080 8081 8082; do +for port in 8080; do echo "Starting server on port $port... " https_port=$((port + 400)) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index f8629a588..9f5f73eab 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -72,6 +72,7 @@ class SyncHandler(BaseHandler): self.event_sources = hs.get_event_sources() self.clock = hs.get_clock() + @defer.inlineCallbacks def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0): """Get the sync for a client if we have new data for it now. Otherwise wait for new data to arrive on the server. If the timeout expires, then @@ -80,15 +81,19 @@ class SyncHandler(BaseHandler): A Deferred SyncResult. """ if timeout == 0 or since_token is None: - return self.current_sync_for_user(sync_config, since_token) + result = yield self.current_sync_for_user(sync_config, since_token) + defer.returnValue(result) else: - def current_sync_callback(since_token): - return self.current_sync_for_user( - self, since_token, sync_config - ) - return self.notifier.wait_for_events( - sync_config.filter, since_token, current_sync_callback + def current_sync_callback(): + return self.current_sync_for_user(sync_config, since_token) + + rm_handler = self.hs.get_handlers().room_member_handler + room_ids = yield rm_handler.get_rooms_for_user(sync_config.user) + result = yield self.notifier.wait_for_events( + sync_config.user, room_ids, + sync_config.filter, timeout, current_sync_callback ) + defer.returnValue(result) def current_sync_for_user(self, sync_config, since_token=None): """Get the sync for client needed to match what the server has now. diff --git a/synapse/notifier.py b/synapse/notifier.py index 3aec1d4af..922bf064d 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.util.async import run_on_reactor +from synapse.types import StreamToken import logging @@ -205,6 +206,53 @@ class Notifier(object): [notify(l).addErrback(eb) for l in listeners] ) + @defer.inlineCallbacks + def wait_for_events(self, user, rooms, filter, timeout, callback): + """Wait until the callback returns a non empty response or the + timeout fires. + """ + + deferred = defer.Deferred() + + from_token=StreamToken("s0","0","0") + + listener = [_NotificationListener( + user=user, + rooms=rooms, + from_token=from_token, + limit=1, + timeout=timeout, + deferred=deferred, + )] + + if timeout: + self._register_with_keys(listener[0]) + + result = yield callback() + if timeout: + timed_out = [False] + def _timeout_listener(): + timed_out[0] = True + listener[0].notify(self, [], from_token, from_token) + + self.clock.call_later(timeout/1000., _timeout_listener) + while not result and not timed_out[0]: + yield deferred + deferred = defer.Deferred() + listener[0] = _NotificationListener( + user=user, + rooms=rooms, + from_token=from_token, + limit=1, + timeout=timeout, + deferred=deferred, + ) + self._register_with_keys(listener[0]) + result = yield callback() + + defer.returnValue(result) + + def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any From e020574d65a994858ac53c45070ae5016090d2f3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jan 2015 20:19:36 +0000 Subject: [PATCH 097/630] Fix Formatting --- synapse/handlers/sync.py | 13 +++++-------- synapse/notifier.py | 4 ++-- synapse/rest/client/v2_alpha/sync.py | 5 ++--- synapse/storage/stream.py | 1 - 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9f5f73eab..82a2c6986 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -52,10 +52,10 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ class SyncResult(collections.namedtuple("SyncResult", [ - "next_batch", # Token for the next sync - "private_user_data", # List of private events for the user. - "public_user_data", # List of public events for all users. - "rooms", # RoomSyncResult for each room. + "next_batch", # Token for the next sync + "private_user_data", # List of private events for the user. + "public_user_data", # List of public events for all users. + "rooms", # RoomSyncResult for each room. ])): __slots__ = [] @@ -181,7 +181,6 @@ class SyncHandler(BaseHandler): limited=True, )) - @defer.inlineCallbacks def incremental_sync_with_gap(self, sync_config, since_token): """ Get the incremental delta needed to bring the client up to @@ -231,7 +230,6 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) - @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, @@ -316,7 +314,6 @@ class SyncHandler(BaseHandler): state = () defer.returnValue(state) - def compute_state_delta(self, since_token, previous_state, current_state): """ Works out the differnce in state between the current state and the state the client got when it last performed a sync. @@ -327,7 +324,7 @@ class SyncHandler(BaseHandler): # after the previous sync, since we need to include those state # updates even if they occured logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - previous_dict = {event.event_id:event for event in previous_state} + previous_dict = {event.event_id: event for event in previous_state} state_delta = [] for event in current_state: if event.event_id not in previous_dict: diff --git a/synapse/notifier.py b/synapse/notifier.py index 922bf064d..e3b6ead62 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -214,7 +214,7 @@ class Notifier(object): deferred = defer.Deferred() - from_token=StreamToken("s0","0","0") + from_token = StreamToken("s0", "0", "0") listener = [_NotificationListener( user=user, @@ -231,6 +231,7 @@ class Notifier(object): result = yield callback() if timeout: timed_out = [False] + def _timeout_listener(): timed_out[0] = True listener[0].notify(self, [], from_token, from_token) @@ -252,7 +253,6 @@ class Notifier(object): defer.returnValue(result) - def get_events_for(self, user, rooms, pagination_config, timeout): """ For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index cc667ebaf..0c17208cd 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -68,7 +68,6 @@ class SyncRestServlet(RestServlet): } """ - PATTERN = client_v2_pattern("/sync$") ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) ALLOWED_PRESENCE = set(["online", "offline", "idle"]) @@ -114,12 +113,12 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - device="TODO", # TODO(mjark) Get the device_id from access_token + device="TODO", # TODO(mjark) Get the device_id from access_token gap=gap, limit=limit, sort=sort, backfill=backfill, - filter="TODO", # TODO(mjark) Add the filter to the config. + filter="TODO", # TODO(mjark) Add the filter to the config. ) if since is not None: diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 06aca1a4e..db1816ea8 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -289,7 +289,6 @@ class StreamStore(SQLBaseStore): " LIMIT ?" ) - def get_recent_events_for_room_txn(txn): if from_token is None: txn.execute(sql, (room_id, end_token.stream, limit,)) From b46fa8603e8e0726ea12310a77ade5cea59c3ae2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 09:17:48 +0000 Subject: [PATCH 098/630] Remove unused import --- synapse/storage/appservice.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index cd15843ba..533fac497 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import StoreError from ._base import SQLBaseStore From e32ded7b3e290067f95497ceee0ac04842ba9cdf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 10:09:54 +0000 Subject: [PATCH 099/630] Add matrix.org as a trusted ID server because it's now passed through on ports 80/443 and the web client defaults to that now. Fixes email validation (including signing up with an email address). --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 732652c22..66a89c10b 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -163,7 +163,7 @@ class RegistrationHandler(BaseHandler): # each request httpCli = SimpleHttpClient(self.hs) # XXX: make this configurable! - trustedIdServers = ['matrix.org:8090'] + trustedIdServers = ['matrix.org:8090', 'matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) From 273b12729b99addf4474c9092f44ff300fd8153b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 11:55:49 +0000 Subject: [PATCH 100/630] Reset badge count to zero when last active time is bumped --- synapse/handlers/presence.py | 5 +++++ synapse/push/__init__.py | 19 ++++++++++++++++++ synapse/push/httppusher.py | 38 +++++++++++++++++++++++++++++++++--- synapse/push/pusherpool.py | 17 ++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8aeed9927..24d901b51 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -86,6 +86,10 @@ class PresenceHandler(BaseHandler): "changed_presencelike_data", self.changed_presencelike_data ) + # outbound signal from the presence module to advertise when a user's + # presence has changed + distributor.declare("user_presence_changed") + self.distributor = distributor self.federation = hs.get_replication_layer() @@ -603,6 +607,7 @@ class PresenceHandler(BaseHandler): room_ids=room_ids, statuscache=statuscache, ) + yield self.distributor.fire("user_presence_changed", user, statuscache) @defer.inlineCallbacks def _push_presence_remote(self, user, destination, state=None): diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index b6d01a82a..4862d0de2 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -54,6 +54,9 @@ class Pusher(object): self.failing_since = failing_since self.alive = True + # The last value of last_active_time that we saw + self.last_last_active_time = 0 + @defer.inlineCallbacks def _actions_for_event(self, ev): """ @@ -273,6 +276,22 @@ class Pusher(object): """ pass + def reset_badge_count(self): + pass + + def presence_changed(self, state): + """ + We clear badge counts whenever a user's last_active time is bumped + This is by no means perfect but I think it's the best we can do + without read receipts. + """ + if 'last_active' in state.state: + last_active = state.state['last_active'] + if last_active > self.last_last_active_time: + logger.info("Resetting badge count for %s", self.user_name) + self.reset_badge_count() + self.last_last_active_time = last_active + def _value_for_dotted_key(dotted_key, event): parts = dotted_key.split(".") diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 22532fcc6..d592bc2fd 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -71,11 +71,12 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'from_display_name': 'Steve Stevington' - #'counts': { -- we don't mark messages as read yet so + 'counts': { #-- we don't mark messages as read yet so # we have no way of knowing - # 'unread': 1, + # Just set the badge to 1 until we have read receipts + 'unread': 1, # 'missed_calls': 2 - # }, + }, 'devices': [ { 'app_id': self.app_id, @@ -111,3 +112,34 @@ class HttpPusher(Pusher): if 'rejected' in resp: rejected = resp['rejected'] defer.returnValue(rejected) + + @defer.inlineCallbacks + def reset_badge_count(self): + d = { + 'notification': { + 'id': '', + 'type': None, + 'from': '', + 'counts': { + 'unread': 0, + 'missed_calls': 0 + }, + 'devices': [ + { + 'app_id': self.app_id, + 'pushkey': self.pushkey, + 'pushkey_ts': long(self.pushkey_ts / 1000), + 'data': self.data_minus_url, + } + ] + } + } + try: + resp = yield self.httpCli.post_json_get_json(self.url, d) + except: + logger.exception("Failed to push %s ", self.url) + defer.returnValue(False) + rejected = [] + if 'rejected' in resp: + rejected = resp['rejected'] + defer.returnValue(rejected) \ No newline at end of file diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 2dfecf178..65ab4f46e 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -18,6 +18,7 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException +from synapse.api.constants import PresenceState import logging import json @@ -32,6 +33,22 @@ class PusherPool: self.pushers = {} self.last_pusher_started = -1 + distributor = self.hs.get_distributor() + distributor.observe( + "user_presence_changed", self.user_presence_changed + ) + + @defer.inlineCallbacks + def user_presence_changed(self, user, state): + user_name = user.to_string() + + # until we have read receipts, pushers use this to reset a user's + # badge counters to zero + for p in self.pushers.values(): + if p.user_name == user_name: + yield p.presence_changed(state) + + @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() From 42876969b99b6bad146b44a734e8d4a1a14d6835 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 11:59:38 +0000 Subject: [PATCH 101/630] Add basic application_services SQL, and hook up parts of the appservice store to read from it. --- synapse/handlers/appservice.py | 7 +- synapse/rest/appservice/v1/register.py | 4 +- synapse/storage/__init__.py | 1 + synapse/storage/appservice.py | 89 ++++++++++++++++--- .../storage/schema/application_services.sql | 32 +++++++ 5 files changed, 117 insertions(+), 16 deletions(-) create mode 100644 synapse/storage/schema/application_services.sql diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index c9f56c41e..8bd475cbf 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -16,7 +16,7 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import StoreError, SynapseError +from synapse.api.errors import Codes, StoreError, SynapseError import logging @@ -36,11 +36,12 @@ class ApplicationServicesHandler(BaseHandler): try: stored_service = yield self.store.get_app_service(app_service.token) if not stored_service: - raise StoreError(404, "Not found") + raise StoreError(404, "Application Service Not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " - "Consult the home server admin." + "Consult the home server admin.", + errcode=Codes.FORBIDDEN ) # TODO store this AS diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index 5786cf873..e374d538e 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -64,9 +64,9 @@ class RegisterRestServlet(AppServiceRestServlet): yield self.handler.register(app_service) hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? - defer.returnValue({ + defer.returnValue((200, { "hs_token": hs_token - }) + })) def _parse_namespace(self, target_ns, origin_ns, ns): if ns not in target_ns or ns not in origin_ns: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9431c1a32..e86b981b4 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -62,6 +62,7 @@ SCHEMAS = [ "event_edges", "event_signatures", "media_repository", + "application_services" ] diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 533fac497..5a0e47e0d 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -12,12 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from synapse.api.errors import StoreError +import logging +from twisted.internet import defer from ._base import SQLBaseStore +logger = logging.getLogger(__name__) + + # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): """Defines an application service. @@ -30,7 +33,22 @@ class ApplicationService(object): if url: self.url = url if namespaces: - self.namespaces = namespaces + self._set_namespaces(namespaces) + + def _set_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + for ns in ["users", "rooms", "aliases"]: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + self.namespaces = namespaces def is_interested(self, event): """Check if this service is interested in this event. @@ -133,15 +151,64 @@ class ApplicationServiceStore(SQLBaseStore): return service return None + # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. - row = self._simple_select_one( - "application_services", {"token": token}, - ["url", "token"] - ) - if not row: - raise StoreError(400, "Bad application services token supplied.") - return row + + @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" - pass + sql = ("SELECT * FROM application_services LEFT JOIN " + "application_services_regex ON application_services.id = " + "application_services_regex.as_id") + + namespace_enum = [ + "users", # 0 + "aliases", # 1 + "rooms" # 2 + ] + # SQL results in the form: + # [ + # { + # 'regex': "something", + # 'url': "something", + # 'namespace': enum, + # 'as_id': 0, + # 'token': "something", + # 'id': 0 + # } + # ] + services = {} + results = yield self._execute_and_decode(sql) + for res in results: + as_token = res["token"] + if as_token not in services: + # add the service + services[as_token] = { + "url": res["url"], + "token": as_token, + "namespaces": { + "users": [], + "aliases": [], + "rooms": [] + } + } + # add the namespace regex if one exists + ns_int = res["namespace"] + if ns_int is None: + continue + try: + services[as_token]["namespaces"][namespace_enum[ns_int]].append( + res["regex"] + ) + except IndexError: + logger.error("Bad namespace enum '%s'. %s", ns_int, res) + + for service in services.values(): + logger.info("Found application service: %s", service) + self.cache.services.append(ApplicationService( + service["token"], + service["url"], + service["namespaces"] + )) + diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql new file mode 100644 index 000000000..6d245fc80 --- /dev/null +++ b/synapse/storage/schema/application_services.sql @@ -0,0 +1,32 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + token TEXT, + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id INTEGER PRIMARY KEY AUTOINCREMENT, + as_id INTEGER NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + + + From fb532d84250e7d112461c62b3ec2443cc61d1348 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:06:09 +0000 Subject: [PATCH 102/630] Unused import --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 65ab4f46e..e69ee2cf1 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -18,7 +18,6 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException -from synapse.api.constants import PresenceState import logging import json From ca7240a2f06b6f6797c857b5cc4d8998a1a46557 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:17:55 +0000 Subject: [PATCH 103/630] Update copyright --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 4862d0de2..fd8d6d4bb 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 6df6f5e0840e6a2b878f62c1656834bfbd5bcd62 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:56:35 +0000 Subject: [PATCH 104/630] Redundant bracketing & missed space --- synapse/push/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index fd8d6d4bb..8ddc83357 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -187,8 +187,8 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we" + - "didn't send it"), (pk,) + ("Ignoring rejected pushkey %s because we " + + "didn't send it"), pk ) else: logger.info( From e1ca0f1396425db5c80a975bb83596c7384484ef Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 13:58:32 +0000 Subject: [PATCH 105/630] Brackets rather than slashes at end --- synapse/push/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8ddc83357..8a3e8fd2a 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -226,9 +226,9 @@ class Pusher(object): self.failing_since ) - if self.failing_since and \ - self.failing_since < \ - self.clock.time_msec() - Pusher.GIVE_UP_AFTER: + if (self.failing_since and + self.failing_since < + self.clock.time_msec() - Pusher.GIVE_UP_AFTER): # we really only give up so that if the URL gets # fixed, we don't suddenly deliver a load # of old notifications. From 03149ad23ace1f200993a74b998a6986d70c9420 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:01:24 +0000 Subject: [PATCH 106/630] More code style things --- synapse/push/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8a3e8fd2a..b79f2d4b2 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -187,7 +187,7 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we " + + ("Ignoring rejected pushkey %s because we " "didn't send it"), pk ) else: @@ -234,7 +234,8 @@ class Pusher(object): # of old notifications. logger.warn("Giving up on a notification to user %s, " "pushkey %s", - self.user_name, self.pushkey) + self.user_name, self.pushkey + ) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token( @@ -256,7 +257,7 @@ class Pusher(object): self.user_name, self.clock.time_msec() - self.failing_since, self.backoff_delay - ) + ) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: From 20c47383dca7d68608c903fe1f856756fd3da057 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:10:46 +0000 Subject: [PATCH 107/630] Oops, bad merge: needed to change the base class of the rest servlets too. --- synapse/push/__init__.py | 5 +++-- synapse/rest/client/v1/push_rule.py | 4 ++-- synapse/rest/client/v1/pusher.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index b79f2d4b2..1cac5fff4 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -269,8 +269,9 @@ class Pusher(object): def dispatch_push(self, p, tweaks): """ Overridden by implementing classes to actually deliver the notification - :param p: The event to notify for as a single event from the event stream - :return: If the notification was delivered, an array containing any + Args: + p The event to notify for as a single event from the event stream + Returns: If the notification was delivered, an array containing any pushkeys that were rejected by the push gateway. False if the notification could not be delivered (ie. should be retried). diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 50bf5b900..00fed42f4 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -17,13 +17,13 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ StoreError -from base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException import json -class PushRuleRestServlet(RestServlet): +class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { 'underride': 0, diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 4659c9b1d..80a11890a 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -17,12 +17,12 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.push import PusherConfigException -from base import RestServlet, client_path_pattern +from .base import ClientV1RestServlet, client_path_pattern import json -class PusherRestServlet(RestServlet): +class PusherRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushers/set$") @defer.inlineCallbacks From 5f2665320fbe4392c9f487820c697691463cf709 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:11:45 +0000 Subject: [PATCH 108/630] It is 2015 --- synapse/push/httppusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index d592bc2fd..e12b94672 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 6fde707addc29b6717fda9617a127093fb5b08ae Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:14:49 +0000 Subject: [PATCH 109/630] doc style fix --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 1cac5fff4..10ac89048 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -270,7 +270,7 @@ class Pusher(object): """ Overridden by implementing classes to actually deliver the notification Args: - p The event to notify for as a single event from the event stream + p: The event to notify for as a single event from the event stream Returns: If the notification was delivered, an array containing any pushkeys that were rejected by the push gateway. False if the notification could not be delivered (ie. From dd3abbd61fba7d3ea9fe300a309202f0eb4c5fbd Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:22:39 +0000 Subject: [PATCH 110/630] 2015 --- synapse/push/pusherpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index e69ee2cf1..716d9514c 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd +# Copyright 2015 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 30fbba168b0f9bf8b38594b3badb9af463167485 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:23:16 +0000 Subject: [PATCH 111/630] Easy on the newlines --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 716d9514c..856defeda 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -47,7 +47,6 @@ class PusherPool: if p.user_name == user_name: yield p.presence_changed(state) - @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() From 4fbf2328c29ca8f8a7918f6763b91787e55f37f8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:24:28 +0000 Subject: [PATCH 112/630] Unnecessary new line --- synapse/rest/client/v1/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 96a9a474f..d8d01cdd1 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from . import ( room, events, register, login, profile, presence, initial_sync, directory, voip, admin, pusher, push_rule From 6741c3dbd90b6406d7eba8445868cdd8fd2ec6a5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:26:03 +0000 Subject: [PATCH 113/630] Brackets are nicer --- synapse/rest/client/v1/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 00fed42f4..b03d804d8 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -33,8 +33,8 @@ class PushRuleRestServlet(ClientV1RestServlet): 'override': 4 } PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} - SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR =\ - "Unrecognised request: You probably wanted a trailing slash" + SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( + "Unrecognised request: You probably wanted a trailing slash") def rule_spec_from_path(self, path): if len(path) < 2: From d93ce29a86b79478bfb2011c9b8d3c8ec7d0bdd0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:27:01 +0000 Subject: [PATCH 114/630] Ah, the comma of doom. --- synapse/rest/client/v1/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index b03d804d8..c6085370a 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -101,7 +101,7 @@ class PushRuleRestServlet(ClientV1RestServlet): if pat.strip("*?[]") == pat: # no special glob characters so we assume the user means # 'contains this string' rather than 'is this string' - pat = "*%s*" % (pat) + pat = "*%s*" % (pat,) conditions = [{ 'kind': 'event_match', 'key': 'content.body', From 032f8d4ed3a5b4015087b7a97295da9b580a95b3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:33:15 +0000 Subject: [PATCH 115/630] Another superfluous newline --- synapse/rest/client/v1/push_rule.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index c6085370a..3a08bdd9a 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -213,7 +213,6 @@ class PushRuleRestServlet(ClientV1RestServlet): else: raise - @defer.inlineCallbacks def on_GET(self, request): user = yield self.auth.get_user_by_req(request) From 8807f4170eaf2515407b656b7eb6fe7e8fc93796 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:35:00 +0000 Subject: [PATCH 116/630] Better style --- synapse/rest/client/v1/push_rule.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 3a08bdd9a..52f2b19bb 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -242,8 +242,11 @@ class PushRuleRestServlet(ClientV1RestServlet): continue if instance_handle not in rules['device']: rules['device'][instance_handle] = {} - rules['device'][instance_handle] = \ - _add_empty_priority_class_arrays(rules['device'][instance_handle]) + rules['device'][instance_handle] = ( + _add_empty_priority_class_arrays( + rules['device'][instance_handle] + ) + ) rulearray = rules['device'][instance_handle][template_name] else: From 3cb5b73c0dd10ba9020547a81864846c70d4e709 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:37:55 +0000 Subject: [PATCH 117/630] Unnecessary newline. --- synapse/rest/client/v1/push_rule.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 52f2b19bb..550554c18 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -290,7 +290,6 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_OPTIONS(self, _): return 200, {} - def _add_empty_priority_class_arrays(d): for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): d[pc] = [] From 289a2498743caae7e0c3366a1c1f7855d48d9a8e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:39:03 +0000 Subject: [PATCH 118/630] Unnecessary newlines. --- synapse/rest/client/v1/push_rule.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 550554c18..dbcac3d4e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -295,7 +295,6 @@ def _add_empty_priority_class_arrays(d): d[pc] = [] return d - def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -326,7 +325,6 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError - def _priority_class_from_spec(spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) @@ -337,7 +335,6 @@ def _priority_class_from_spec(spec): return pc - def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -346,7 +343,6 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] - def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: @@ -363,7 +359,6 @@ def _rule_to_template(rule): ret["pattern"] = thecond["pattern"] return ret - def _strip_device_condition(rule): for i,c in enumerate(rule['conditions']): if c['kind'] == 'device': From 2cfdfee572dc037b65571ab72efcb3223c7d8d11 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:41:51 +0000 Subject: [PATCH 119/630] spaces --- synapse/rest/client/v1/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index dbcac3d4e..2b33bdac0 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -346,7 +346,7 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: - return {k:rule[k] for k in ["rule_id", "conditions", "actions"]} + return {k: rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: return {k: rule[k] for k in ["rule_id", "actions"]} elif template_name == 'content': @@ -355,7 +355,7 @@ def _rule_to_template(rule): thecond = rule["conditions"][0] if "pattern" not in thecond: return None - ret = {k:rule[k] for k in ["rule_id", "actions"]} + ret = {k: rule[k] for k in ["rule_id", "actions"]} ret["pattern"] = thecond["pattern"] return ret From 0cbb6b0f5235e4501a0fb360e881d152644a17cd Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:44:41 +0000 Subject: [PATCH 120/630] Google doc style --- synapse/storage/_base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4f172d396..809c81f47 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -195,10 +195,11 @@ class SQLBaseStore(object): def _simple_upsert(self, table, keyvalues, values): """ - :param table: The table to upsert into - :param keyvalues: Dict of the unique key tables and their new values - :param values: Dict of all the nonunique columns and their new values - :return: A deferred + Args: + table (str): The table to upsert into + keyvalues (dict): The unique key tables and their new values + values (dict): The nonunique columns and their new values + Returns: A deferred """ return self.runInteraction( "_simple_upsert", From fb0928097a0dc1606aebb9aed8f070bcea304178 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:48:07 +0000 Subject: [PATCH 121/630] More magic commas (including the place I copied it from...) --- synapse/storage/_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 809c81f47..9261c999c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -210,8 +210,8 @@ class SQLBaseStore(object): # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, - ", ".join("%s = ?" % (k) for k in values), - " AND ".join("%s = ?" % (k) for k in keyvalues) + ", ".join("%s = ?" % (k,) for k in values), + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) sqlargs = values.values() + keyvalues.values() logger.debug( @@ -390,8 +390,8 @@ class SQLBaseStore(object): if updatevalues: update_sql = "UPDATE %s SET %s WHERE %s" % ( table, - ", ".join("%s = ?" % (k) for k in updatevalues), - " AND ".join("%s = ?" % (k) for k in keyvalues) + ", ".join("%s = ?" % (k,) for k in updatevalues), + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) def func(txn): From 6d485dd1c727e7ecfe3991066bd058794ae05051 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:48:42 +0000 Subject: [PATCH 122/630] unnecessary newlines --- synapse/storage/_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9261c999c..4e8bd3faa 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -237,8 +237,6 @@ class SQLBaseStore(object): ) txn.execute(sql, allvalues.values()) - - def _simple_select_one(self, table, keyvalues, retcols, allow_none=False): """Executes a SELECT query on the named table, which is expected to From 445ad9941ea2e4038846aa6fed456e3250ae49b1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:49:59 +0000 Subject: [PATCH 123/630] Redundant parens --- synapse/storage/push_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index ca04f2cce..f5a736be4 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -92,7 +92,7 @@ class PushRuleStore(SQLBaseStore): res = txn.fetchall() if not res: raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) - (priority_class, base_rule_priority) = res[0] + priority_class, base_rule_priority = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: raise InconsistentRuleException( From 93aac9bb7b3023e6c82961b1cdd655a48ec567fb Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:51:01 +0000 Subject: [PATCH 124/630] Newline --- synapse/storage/push_rule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index f5a736be4..48105234f 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -184,6 +184,7 @@ class PushRuleStore(SQLBaseStore): } ) + class RuleNotFoundException(Exception): pass From e78dd332928c111c8a62985bce0a3c1c5631244e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 14:52:58 +0000 Subject: [PATCH 125/630] Use %s instead of + --- synapse/storage/push_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 48105234f..0342996ed 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -85,8 +85,8 @@ class PushRuleStore(SQLBaseStore): # get the priority of the rule we're inserting after/before sql = ( - "SELECT priority_class, priority FROM "+PushRuleTable.table_name+ - " WHERE user_name = ? and rule_id = ?" + "SELECT priority_class, priority FROM ? " + "WHERE user_name = ? and rule_id = ?" % (PushRuleTable.table_name,) ) txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() From c59bcabf0b5c0ab78c0f89da75b031993c4660d9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 15:36:21 +0000 Subject: [PATCH 126/630] Return the device_id from get_auth_by_req --- synapse/api/auth.py | 7 ++++-- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/directory.py | 4 ++-- synapse/rest/client/v1/events.py | 4 ++-- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/presence.py | 8 +++---- synapse/rest/client/v1/profile.py | 4 ++-- synapse/rest/client/v1/room.py | 24 ++++++++++----------- synapse/rest/client/v1/voip.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- tests/rest/client/v1/test_presence.py | 2 +- tests/rest/client/v1/test_profile.py | 2 +- 13 files changed, 34 insertions(+), 31 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a342a0e0d..292e9e2a8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -290,7 +290,9 @@ class Auth(object): Args: request - An HTTP request with an access_token query parameter. Returns: - UserID : User ID object of the user making the request + Tuple of UserID and device string: + User ID object of the user making the request + Device ID string of the device the user is using Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -299,6 +301,7 @@ class Auth(object): access_token = request.args["access_token"][0] user_info = yield self.get_user_by_token(access_token) user = user_info["user"] + device_id = user_info["device_id"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( @@ -314,7 +317,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue(user) + defer.returnValue((user, device_id)) except KeyError: raise AuthError(403, "Missing access token.") diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 1051d96f9..6cfce1a47 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,7 +31,7 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) if not is_admin and target_user != auth_user: diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 15ae8749b..ef853af41 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,7 +45,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) if not "room_id" in content: @@ -85,7 +85,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, room_alias): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index a0d051227..e58ee46fc 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,7 +34,7 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) try: handler = self.handlers.event_stream_handler pagin_config = PaginationConfig.from_request(request) @@ -71,7 +71,7 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 357fa845b..78d30abbf 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,7 +25,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index b6c207e66..74669274a 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,7 +32,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,7 +42,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -77,7 +77,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): @@ -97,7 +97,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 24f8d5695..f04abb2c2 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 58b09b6fc..c8c34b480 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -62,7 +62,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) info = yield self.make_room(room_config, auth_user, None) @@ -125,7 +125,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( @@ -143,7 +143,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, event_type, state_key): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -173,7 +173,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_type): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -216,7 +216,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_identifier): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) # the identifier could be a room alias or a room id. Try one then the # other if it fails to parse, without swallowing other valid @@ -283,7 +283,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( room_id=room_id, @@ -311,7 +311,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -335,7 +335,7 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( @@ -351,7 +351,7 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, @@ -396,7 +396,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, membership_action): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -445,7 +445,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, room_id, event_id): - user = yield self.auth.get_user_by_req(request) + user, device_id = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -483,7 +483,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 822d863ce..42d8e30ba 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 79ae0e3d7..311ab89ed 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,7 +66,7 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index b1718a630..6bed8a8ef 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -42,7 +42,7 @@ class UploadResource(BaseMediaResource): @defer.inlineCallbacks def _async_render_POST(self, request): try: - auth_user = yield self.auth.get_user_by_req(request) + auth_user, device_id = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 65d5cc491..a4f2abf21 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -282,7 +282,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.get_clock().time_msec.return_value = 1000000 def _get_user_by_req(req=None): - return UserID.from_string(myid) + return (UserID.from_string(myid), "") hs.get_auth().get_user_by_req = _get_user_by_req diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 39cd68d82..6a2085276 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -58,7 +58,7 @@ class ProfileTestCase(unittest.TestCase): ) def _get_user_by_req(request=None): - return UserID.from_string(myid) + return (UserID.from_string(myid), "") hs.get_auth().get_user_by_req = _get_user_by_req From 60b143a52e69751a406ea83cdab58f4045cdd9d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 15:48:28 +0000 Subject: [PATCH 127/630] Move pushers delta to v12 and bump schema version --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/{v10.sql => v12.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/delta/{v10.sql => v12.sql} (100%) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2534d109f..277581b4e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -69,7 +69,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 11 +SCHEMA_VERSION = 12 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v10.sql b/synapse/storage/schema/delta/v12.sql similarity index 100% rename from synapse/storage/schema/delta/v10.sql rename to synapse/storage/schema/delta/v12.sql From 0ef5bfd6a9eaaae14e199997658b3d0006abd854 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 28 Jan 2015 16:16:53 +0000 Subject: [PATCH 128/630] Start implementing auth conflict res --- synapse/api/auth.py | 38 +++-- synapse/api/constants.py | 6 + synapse/federation/federation_client.py | 39 +++++ synapse/handlers/federation.py | 211 ++++++++++++++++++++---- synapse/storage/rejections.py | 10 ++ synapse/storage/schema/im.sql | 1 + 6 files changed, 253 insertions(+), 52 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a342a0e0d..461faa8c7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -353,9 +353,23 @@ class Auth(object): def add_auth_events(self, builder, context): yield run_on_reactor() - if builder.type == EventTypes.Create: - builder.auth_events = [] - return + auth_ids = self.compute_auth_events(builder, context) + + auth_events_entries = yield self.store.add_event_hashes( + auth_ids + ) + + builder.auth_events = auth_events_entries + + context.auth_events = { + k: v + for k, v in context.current_state.items() + if v.event_id in auth_ids + } + + def compute_auth_events(self, event, context): + if event.type == EventTypes.Create: + return [] auth_ids = [] @@ -368,7 +382,7 @@ class Auth(object): key = (EventTypes.JoinRules, "", ) join_rule_event = context.current_state.get(key) - key = (EventTypes.Member, builder.user_id, ) + key = (EventTypes.Member, event.user_id, ) member_event = context.current_state.get(key) key = (EventTypes.Create, "", ) @@ -382,8 +396,8 @@ class Auth(object): else: is_public = False - if builder.type == EventTypes.Member: - e_type = builder.content["membership"] + if event.type == EventTypes.Member: + e_type = event.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: if join_rule_event: auth_ids.append(join_rule_event.event_id) @@ -398,17 +412,7 @@ class Auth(object): if member_event.content["membership"] == Membership.JOIN: auth_ids.append(member_event.event_id) - auth_events_entries = yield self.store.add_event_hashes( - auth_ids - ) - - builder.auth_events = auth_events_entries - - context.auth_events = { - k: v - for k, v in context.current_state.items() - if v.event_id in auth_ids - } + return auth_ids @log_function def _can_send_event(self, event, auth_events): diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7ee6dcc46..0d3fc629a 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -74,3 +74,9 @@ class EventTypes(object): Message = "m.room.message" Topic = "m.room.topic" Name = "m.room.name" + + +class RejectedReason(object): + AUTH_ERROR = "auth_error" + REPLACED = "replaced" + NOT_ANCESTOR = "not_ancestor" diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 91b44cd8b..ebcd59350 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -331,6 +331,45 @@ class FederationClient(object): defer.returnValue(pdu) + @defer.inlineCallbacks + def query_auth(self, destination, room_id, event_id, local_auth): + """ + Params: + destination (str) + event_it (str) + local_auth (list) + """ + time_now = self._clock.time_msec() + + send_content = { + "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], + } + + code, content = yield self.transport_layer.send_invite( + destination=destination, + room_id=room_id, + event_id=event_id, + content=send_content, + ) + + auth_chain = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content["auth_chain"] + ] + + missing = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content.get("missing", []) + ] + + ret = { + "auth_chain": auth_chain, + "rejects": content.get("rejects", []), + "missing": missing, + } + + defer.returnValue(ret) + def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( pdu_json diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index bcdcc90a1..97e3c503b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,19 +17,16 @@ from ._base import BaseHandler -from synapse.events.utils import prune_event from synapse.api.errors import ( - AuthError, FederationError, SynapseError, StoreError, + AuthError, FederationError, StoreError, ) -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import ( - compute_event_signature, check_event_content_hash, - add_hashes_and_signatures, + compute_event_signature, add_hashes_and_signatures, ) from synapse.types import UserID -from syutil.jsonutil import encode_canonical_json from twisted.internet import defer @@ -113,33 +110,6 @@ class FederationHandler(BaseHandler): logger.debug("Processing event: %s", event.event_id) - redacted_event = prune_event(event) - - redacted_pdu_json = redacted_event.get_pdu_json() - try: - yield self.keyring.verify_json_for_server( - event.origin, redacted_pdu_json - ) - except SynapseError as e: - logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), - ) - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) - - if not check_event_content_hash(event): - logger.warn( - "Event content has been tampered, redacting %s, %s", - event.event_id, encode_canonical_json(event.get_dict()) - ) - event = redacted_event - logger.debug("Event: %s", event) # FIXME (erikj): Awful hack to make the case where we are not currently @@ -180,7 +150,6 @@ class FederationHandler(BaseHandler): if state: for e in state: - logging.info("A :) %r", e) e.internal_metadata.outlier = True try: yield self._handle_new_event(e) @@ -747,7 +716,20 @@ class FederationHandler(BaseHandler): event.event_id, event.signatures, ) - self.auth.check(event, auth_events=context.auth_events) + try: + self.auth.check(event, auth_events=context.auth_events) + except AuthError: + # TODO: Store rejection. + context.rejected = RejectedReason.AUTH_ERROR + + yield self.store.persist_event( + event, + context=context, + backfilled=backfilled, + is_new_state=False, + current_state=current_state, + ) + raise logger.debug( "_handle_new_event: Before persist_event: %s, sigs: %s", @@ -768,3 +750,162 @@ class FederationHandler(BaseHandler): ) defer.returnValue(context) + + @defer.inlineCallbacks + def do_auth(self, origin, event, context): + for e_id, _ in event.auth_events: + pass + + auth_events = set(e_id for e_id, _ in event.auth_events) + current_state = set(e.event_id for e in context.auth_events.values()) + + missing_auth = auth_events - current_state + + if missing_auth: + # Do auth conflict res. + + # 1. Get what we think is the auth chain. + auth_ids = self.auth.compute_auth_events(event, context) + local_auth_chain = yield self.store.get_auth_chain(auth_ids) + + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) + + # 3. Process any remote auth chain events we haven't seen. + for e in result.get("missing", []): + # TODO. + pass + + # 4. Look at rejects and their proofs. + # TODO. + + try: + self.auth.check(event, auth_events=context.auth_events) + except AuthError: + raise + + @defer.inlineCallbacks + def construct_auth_difference(self, local_auth, remote_auth): + """ Given a local and remote auth chain, find the differences. This + assumes that we have already processed all events in remote_auth + + Params: + local_auth (list) + remote_auth (list) + + Returns: + dict + """ + + # TODO: Make sure we are OK with local_auth or remote_auth having more + # auth events in them than strictly necessary. + + def sort_fun(ev): + return ev.depth, ev.event_id + + # We find the differences by starting at the "bottom" of each list + # and iterating up on both lists. The lists are ordered by depth and + # then event_id, we iterate up both lists until we find the event ids + # don't match. Then we look at depth/event_id to see which side is + # missing that event, and iterate only up that list. Repeat. + + remote_list = list(remote_auth) + remote_list.sort(key=sort_fun) + + local_list = list(local_auth) + local_list.sort(key=sort_fun) + + local_iter = iter(local_list) + remote_iter = iter(remote_list) + + current_local = local_iter.next() + current_remote = remote_iter.next() + + def get_next(it, opt=None): + return it.next() if it.has_next() else opt + + missing_remotes = [] + missing_locals = [] + while current_local and current_remote: + if current_remote is None: + missing_locals.append(current_local) + current_local = get_next(local_iter) + continue + + if current_local is None: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + if current_local.event_id == current_remote.event_id: + current_local = get_next(local_iter) + current_remote = get_next(remote_iter) + continue + + if current_local.depth < current_remote.depth: + missing_locals.append(current_local) + current_local = get_next(local_iter) + continue + + if current_local.depth > current_remote.depth: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + # They have the same depth, so we fall back to the event_id order + if current_local.event_id < current_remote.event_id: + missing_locals.append(current_local) + current_local = get_next(local_iter) + + if current_local.event_id > current_remote.event_id: + missing_remotes.append(current_remote) + current_remote = get_next(remote_iter) + continue + + # missing locals should be sent to the server + # We should find why we are missing remotes, as they will have been + # rejected. + + # Remove events from missing_remotes if they are referencing a missing + # remote. We only care about the "root" rejected ones. + missing_remote_ids = [e.event_id for e in missing_remotes] + base_remote_rejected = list(missing_remotes) + for e in missing_remotes: + for e_id, _ in e.auth_events: + if e_id in missing_remote_ids: + base_remote_rejected.remove(e) + + reason_map = {} + + for e in base_remote_rejected: + reason = yield self.store.get_rejection_reason(e.event_id) + if reason is None: + # FIXME: ERRR?! + raise RuntimeError("") + + reason_map[e.event_id] = reason + + if reason == RejectedReason.AUTH_ERROR: + pass + elif reason == RejectedReason.REPLACED: + # TODO: Get proof + pass + elif reason == RejectedReason.NOT_ANCESTOR: + # TODO: Get proof. + pass + + defer.returnValue({ + "rejects": { + e.event_id: { + "reason": reason_map[e.event_id], + "proof": None, + } + for e in base_remote_rejected + }, + "missing": missing_locals, + }) diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index 7d38b31f4..b7249700d 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -31,3 +31,13 @@ class RejectionsStore(SQLBaseStore): "last_failure": self._clock.time_msec(), } ) + + def get_rejection_reason(self, event_id): + self._simple_select_one_onecol( + table="rejections", + retcol="reason", + keyvalues={ + "event_id": event_id, + }, + allow_none=True, + ) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index bc7c6b6ed..5866a387f 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -128,5 +128,6 @@ CREATE TABLE IF NOT EXISTS rejections( event_id TEXT NOT NULL, reason TEXT NOT NULL, last_check TEXT NOT NULL, + root_rejected TEXT, CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE ); From c23e3db544eb940d95a092b661e3872480f3bf30 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 16:45:18 +0000 Subject: [PATCH 129/630] Add filter JSON sanity checks. --- synapse/api/filtering.py | 109 ++++++++++++++++++++++++- synapse/rest/client/v2_alpha/filter.py | 2 +- synapse/storage/filtering.py | 4 +- tests/api/test_filtering.py | 24 +++++- 4 files changed, 128 insertions(+), 11 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 20b6951d4..6c7a73b6d 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer +from synapse.api.errors import SynapseError +from synapse.types import UserID, RoomID class Filtering(object): @@ -25,10 +26,110 @@ class Filtering(object): def get_user_filter(self, user_localpart, filter_id): return self.store.get_user_filter(user_localpart, filter_id) - def add_user_filter(self, user_localpart, definition): - # TODO(paul): implement sanity checking of the definition - return self.store.add_user_filter(user_localpart, definition) + def add_user_filter(self, user_localpart, user_filter): + self._check_valid_filter(user_filter) + return self.store.add_user_filter(user_localpart, user_filter) # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however + + def _check_valid_filter(self, user_filter): + """Check if the provided filter is valid. + + This inspects all definitions contained within the filter. + + Args: + user_filter(dict): The filter + Raises: + SynapseError: If the filter is not valid. + """ + # NB: Filters are the complete json blobs. "Definitions" are an + # individual top-level key e.g. public_user_data. Filters are made of + # many definitions. + + top_level_definitions = [ + "public_user_data", "private_user_data", "server_data" + ] + + room_level_definitions = [ + "state", "events", "ephemeral" + ] + + for key in top_level_definitions: + if key in user_filter: + self._check_definition(user_filter[key]) + + if "room" in user_filter: + for key in room_level_definitions: + if key in user_filter["room"]: + self._check_definition(user_filter["room"][key]) + + + def _check_definition(self, definition): + """Check if the provided definition is valid. + + This inspects not only the types but also the values to make sure they + make sense. + + Args: + definition(dict): The filter definition + Raises: + SynapseError: If there was a problem with this definition. + """ + # NB: Filters are the complete json blobs. "Definitions" are an + # individual top-level key e.g. public_user_data. Filters are made of + # many definitions. + if type(definition) != dict: + raise SynapseError( + 400, "Expected JSON object, not %s" % (definition,) + ) + + # check rooms are valid room IDs + room_id_keys = ["rooms", "not_rooms"] + for key in room_id_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for room_id in definition[key]: + RoomID.from_string(room_id) + + # check senders are valid user IDs + user_id_keys = ["senders", "not_senders"] + for key in user_id_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for user_id in definition[key]: + UserID.from_string(user_id) + + # TODO: We don't limit event type values but we probably should... + # check types are valid event types + event_keys = ["types", "not_types"] + for key in event_keys: + if key in definition: + if type(definition[key]) != list: + raise SynapseError(400, "Expected %s to be a list." % key) + for event_type in definition[key]: + if not isinstance(event_type, basestring): + raise SynapseError(400, "Event type should be a string") + + try: + event_format = definition["format"] + if event_format not in ["federation", "events"]: + raise SynapseError(400, "Invalid format: %s" % (event_format,)) + except KeyError: + pass # format is optional + + try: + event_select_list = definition["select"] + for select_key in event_select_list: + if select_key not in ["event_id", "origin_server_ts", + "thread_id", "content", "content.body"]: + raise SynapseError(400, "Bad select: %s" % (select_key,)) + except KeyError: + pass # select is optional + + if ("bundle_updates" in definition and + type(definition["bundle_updates"]) != bool): + raise SynapseError(400, "Bad bundle_updates: expected bool.") diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 09e44e8ae..81a3e9515 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -93,7 +93,7 @@ class CreateFilterRestServlet(RestServlet): filter_id = yield self.filtering.add_user_filter( user_localpart=target_user.localpart, - definition=content, + user_filter=content, ) defer.returnValue((200, {"filter_id": str(filter_id)})) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index e98eaf803..bab68a9ee 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -39,8 +39,8 @@ class FilteringStore(SQLBaseStore): defer.returnValue(json.loads(def_json)) - def add_user_filter(self, user_localpart, definition): - def_json = json.dumps(definition) + def add_user_filter(self, user_localpart, user_filter): + def_json = json.dumps(user_filter) # Need an atomic transaction to SELECT the maximal ID so far then # INSERT a new one diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 149948374..188fbfb91 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -57,13 +57,21 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_add_filter(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, - definition={"type": ["m.*"]}, + user_filter=user_filter, ) self.assertEquals(filter_id, 0) - self.assertEquals({"type": ["m.*"]}, + self.assertEquals(user_filter, (yield self.datastore.get_user_filter( user_localpart=user_localpart, filter_id=0, @@ -72,9 +80,17 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_filter(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - definition={"type": ["m.*"]}, + user_filter=user_filter, ) filter = yield self.filtering.get_user_filter( @@ -82,4 +98,4 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - self.assertEquals(filter, {"type": ["m.*"]}) + self.assertEquals(filter, user_filter) From 388581e087a3658c1b70d2aa1d17a132953350ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 16:58:23 +0000 Subject: [PATCH 130/630] Extract the id token of the token when authing users, include the token and device_id in the internal meta data for the event along with the transaction id when sending events --- synapse/api/auth.py | 8 ++- synapse/handlers/message.py | 12 +++- synapse/rest/client/v1/admin.py | 2 +- synapse/rest/client/v1/directory.py | 4 +- synapse/rest/client/v1/events.py | 4 +- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/presence.py | 8 +-- synapse/rest/client/v1/profile.py | 4 +- synapse/rest/client/v1/room.py | 64 ++++++++++++--------- synapse/rest/client/v1/voip.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- synapse/storage/registration.py | 3 +- synapse/types.py | 3 + tests/rest/client/v1/test_presence.py | 2 + tests/rest/client/v1/test_rooms.py | 7 +++ tests/rest/client/v1/test_typing.py | 1 + tests/storage/test_registration.py | 10 +++- 18 files changed, 92 insertions(+), 48 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 292e9e2a8..3959e06a8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import UserID +from synapse.types import UserID, ClientID import logging @@ -292,7 +292,7 @@ class Auth(object): Returns: Tuple of UserID and device string: User ID object of the user making the request - Device ID string of the device the user is using + Client ID object of the client instance the user is using Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -302,6 +302,7 @@ class Auth(object): user_info = yield self.get_user_by_token(access_token) user = user_info["user"] device_id = user_info["device_id"] + token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) user_agent = request.requestHeaders.getRawHeaders( @@ -317,7 +318,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue((user, device_id)) + defer.returnValue((user, ClientID(device_id, token_id))) except KeyError: raise AuthError(403, "Missing access token.") @@ -342,6 +343,7 @@ class Auth(object): "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), "user": UserID.from_string(ret.get("name")), + "token_id": ret.get("token_id", None), } defer.returnValue(user_info) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9c3271fe8..6fbd2af4a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -114,7 +114,8 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def create_and_send_event(self, event_dict, ratelimit=True): + def create_and_send_event(self, event_dict, ratelimit=True, + client=None, txn_id=None): """ Given a dict from a client, create and handle a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, @@ -148,6 +149,15 @@ class MessageHandler(BaseHandler): builder.content ) + if client is not None: + if client.token_id is not None: + builder.internal_metadata.token_id = client.token_id + if client.device_id is not None: + builder.internal_metadata.device_id = client.device_id + + if txn_id is not None: + builder.internal_metadata.txn_id = txn_id + event, context = yield self._create_new_client_event( builder=builder, ) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index 6cfce1a47..2ce754b02 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -31,7 +31,7 @@ class WhoisRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(auth_user) if not is_admin and target_user != auth_user: diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index ef853af41..8f65efec5 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,7 +45,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) if not "room_id" in content: @@ -85,7 +85,7 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, room_alias): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index e58ee46fc..77b7c25a0 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -34,7 +34,7 @@ class EventStreamRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) try: handler = self.handlers.event_stream_handler pagin_config = PaginationConfig.from_request(request) @@ -71,7 +71,7 @@ class EventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, event_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.event_handler event = yield handler.get_event(auth_user, event_id) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 78d30abbf..4a259bba6 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -25,7 +25,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 74669274a..7feb4aadb 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -32,7 +32,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = yield self.handlers.presence_handler.get_state( @@ -42,7 +42,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) state = {} @@ -77,7 +77,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): @@ -97,7 +97,7 @@ class PresenceListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index f04abb2c2..15d6f3fc6 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -37,7 +37,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: @@ -70,7 +70,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) try: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c8c34b480..410f19ccf 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -62,7 +62,7 @@ class RoomCreateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) room_config = self.get_room_config(request) info = yield self.make_room(room_config, auth_user, None) @@ -125,7 +125,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( @@ -142,8 +142,8 @@ class RoomStateEventRestServlet(ClientV1RestServlet): defer.returnValue((200, data.get_dict()["content"])) @defer.inlineCallbacks - def on_PUT(self, request, room_id, event_type, state_key): - user, device_id = yield self.auth.get_user_by_req(request) + def on_PUT(self, request, room_id, event_type, state_key, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -158,7 +158,9 @@ class RoomStateEventRestServlet(ClientV1RestServlet): event_dict["state_key"] = state_key msg_handler = self.handlers.message_handler - yield msg_handler.create_and_send_event(event_dict) + yield msg_handler.create_and_send_event( + event_dict, client=client, txn_id=txn_id, + ) defer.returnValue((200, {})) @@ -172,8 +174,8 @@ class RoomSendEventRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server, with_get=True) @defer.inlineCallbacks - def on_POST(self, request, room_id, event_type): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, event_type, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -183,7 +185,9 @@ class RoomSendEventRestServlet(ClientV1RestServlet): "content": content, "room_id": room_id, "sender": user.to_string(), - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"event_id": event.event_id})) @@ -200,7 +204,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, event_type) + response = yield self.on_POST(request, room_id, event_type, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -215,8 +219,8 @@ class JoinRoomAliasServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_identifier): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_identifier, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) # the identifier could be a room alias or a room id. Try one then the # other if it fails to parse, without swallowing other valid @@ -245,7 +249,9 @@ class JoinRoomAliasServlet(ClientV1RestServlet): "room_id": identifier.to_string(), "sender": user.to_string(), "state_key": user.to_string(), - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"room_id": identifier.to_string()})) @@ -259,7 +265,7 @@ class JoinRoomAliasServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_identifier) + response = yield self.on_POST(request, room_identifier, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -283,7 +289,7 @@ class RoomMemberListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( room_id=room_id, @@ -311,7 +317,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request( request, default_limit=10, ) @@ -335,7 +341,7 @@ class RoomStateRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( @@ -351,7 +357,7 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - user, device_id = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, @@ -395,8 +401,8 @@ class RoomMembershipRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_id, membership_action): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, membership_action, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -418,7 +424,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), "state_key": state_key, - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {})) @@ -432,7 +440,9 @@ class RoomMembershipRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, membership_action) + response = yield self.on_POST( + request, room_id, membership_action, txn_id + ) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -444,8 +454,8 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): register_txn_path(self, PATTERN, http_server) @defer.inlineCallbacks - def on_POST(self, request, room_id, event_id): - user, device_id = yield self.auth.get_user_by_req(request) + def on_POST(self, request, room_id, event_id, txn_id=None): + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) msg_handler = self.handlers.message_handler @@ -456,7 +466,9 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): "room_id": room_id, "sender": user.to_string(), "redacts": event_id, - } + }, + client=client, + txn_id=txn_id, ) defer.returnValue((200, {"event_id": event.event_id})) @@ -470,7 +482,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): except KeyError: pass - response = yield self.on_POST(request, room_id, event_id) + response = yield self.on_POST(request, room_id, event_id, txn_id) self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) @@ -483,7 +495,7 @@ class RoomTypingRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) room_id = urllib.unquote(room_id) target_user = UserID.from_string(urllib.unquote(user_id)) diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 42d8e30ba..11d08fbce 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) turnUris = self.hs.config.turn_uris turnSecret = self.hs.config.turn_shared_secret diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 311ab89ed..22e26e3cd 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -66,7 +66,7 @@ class ContentRepoResource(resource.Resource): @defer.inlineCallbacks def map_request_to_name(self, request): # auth the user - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) # namespace all file uploads on the user prefix = base64.urlsafe_b64encode( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 6bed8a8ef..b939a30e1 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -42,7 +42,7 @@ class UploadResource(BaseMediaResource): @defer.inlineCallbacks def _async_render_POST(self, request): try: - auth_user, device_id = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 75dffa4db..029b07cc6 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -122,7 +122,8 @@ class RegistrationStore(SQLBaseStore): def _query_for_auth(self, txn, token): sql = ( - "SELECT users.name, users.admin, access_tokens.device_id" + "SELECT users.name, users.admin," + " access_tokens.device_id, access_tokens.id as token_id" " FROM users" " INNER JOIN access_tokens on users.id = access_tokens.user_id" " WHERE token = ?" diff --git a/synapse/types.py b/synapse/types.py index faac729ff..46dbab537 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -119,3 +119,6 @@ class StreamToken( d = self._asdict() d[key] = new_value return StreamToken(**d) + + +ClientID = namedtuple("ClientID", ("device_id", "token_id")) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index a4f2abf21..f849120a3 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -75,6 +75,7 @@ class PresenceStateTestCase(unittest.TestCase): "user": UserID.from_string(myid), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -165,6 +166,7 @@ class PresenceListTestCase(unittest.TestCase): "user": UserID.from_string(myid), "admin": False, "device_id": None, + "token_id": 1, } hs.handlers.room_member_handler = Mock( diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 76ed550b7..81ead10e7 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -70,6 +70,7 @@ class RoomPermissionsTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -466,6 +467,7 @@ class RoomsMemberListTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -555,6 +557,7 @@ class RoomsCreateTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -657,6 +660,7 @@ class RoomTopicTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -773,6 +777,7 @@ class RoomMemberStateTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -909,6 +914,7 @@ class RoomMessagesTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token @@ -1013,6 +1019,7 @@ class RoomInitialSyncTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index c89b37d00..c5d5b06da 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -73,6 +73,7 @@ class RoomTypingTestCase(RestTestCase): "user": UserID.from_string(self.auth_user_id), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 84bfde756..6f8bea2f6 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -53,7 +53,10 @@ class RegistrationStoreTestCase(unittest.TestCase): ) self.assertEquals( - {"admin": 0, "device_id": None, "name": self.user_id}, + {"admin": 0, + "device_id": None, + "name": self.user_id, + "token_id": 1}, (yield self.store.get_user_by_token(self.tokens[0])) ) @@ -63,7 +66,10 @@ class RegistrationStoreTestCase(unittest.TestCase): yield self.store.add_access_token_to_user(self.user_id, self.tokens[1]) self.assertEquals( - {"admin": 0, "device_id": None, "name": self.user_id}, + {"admin": 0, + "device_id": None, + "name": self.user_id, + "token_id": 2}, (yield self.store.get_user_by_token(self.tokens[1])) ) From c18e551640994c8b2c509509bcf664748dd05724 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:08:53 +0000 Subject: [PATCH 131/630] Add a : to the doc string after the type of the return value --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3959e06a8..f08cb7615 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -290,7 +290,7 @@ class Auth(object): Args: request - An HTTP request with an access_token query parameter. Returns: - Tuple of UserID and device string: + tuple : of UserID and device string: User ID object of the user making the request Client ID object of the client instance the user is using Raises: From 3cca61e006d7e69b6643721c01ab7d81a8c2f373 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:16:12 +0000 Subject: [PATCH 132/630] Rename ClientID to ClientInfo since it is a pair of IDs rather than a single identifier --- synapse/api/auth.py | 4 ++-- synapse/types.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f08cb7615..9c0302451 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import UserID, ClientID +from synapse.types import UserID, ClientInfo import logging @@ -318,7 +318,7 @@ class Auth(object): user_agent=user_agent ) - defer.returnValue((user, ClientID(device_id, token_id))) + defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: raise AuthError(403, "Missing access token.") diff --git a/synapse/types.py b/synapse/types.py index 46dbab537..f6a1b0bbc 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -121,4 +121,4 @@ class StreamToken( return StreamToken(**d) -ClientID = namedtuple("ClientID", ("device_id", "token_id")) +ClientInfo = namedtuple("ClientInfo", ("device_id", "token_id")) From c81a19552f12a16591ae695fa1ef660f9e38730e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 28 Jan 2015 17:32:41 +0000 Subject: [PATCH 133/630] Add ports back to demo/start.sh --- demo/start.sh | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index a7502e7a8..bb2248770 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,7 +16,7 @@ if [ $# -eq 1 ]; then fi fi -for port in 8080; do +for port in 8080 8081 8082; do echo "Starting server on port $port... " https_port=$((port + 400)) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 0c17208cd..a0ab9839a 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -80,7 +80,7 @@ class SyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) timeout = self.parse_integer(request, "timeout", default=0) limit = self.parse_integer(request, "limit", required=True) From 11634017f47779d784325da5513517ad76b0dbc1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jan 2015 17:42:19 +0000 Subject: [PATCH 134/630] s/definition/filter_json/ since definition is now used to mean a component of the filter, rather than the complete json --- synapse/storage/filtering.py | 4 ++-- synapse/storage/schema/filtering.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index bab68a9ee..cb01c2040 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -33,7 +33,7 @@ class FilteringStore(SQLBaseStore): "user_id": user_localpart, "filter_id": filter_id, }, - retcol="definition", + retcol="filter_json", allow_none=False, ) @@ -57,7 +57,7 @@ class FilteringStore(SQLBaseStore): filter_id = max_id + 1 sql = ( - "INSERT INTO user_filters (user_id, filter_id, definition)" + "INSERT INTO user_filters (user_id, filter_id, filter_json)" "VALUES(?, ?, ?)" ) txn.execute(sql, (user_localpart, filter_id, def_json)) diff --git a/synapse/storage/schema/filtering.sql b/synapse/storage/schema/filtering.sql index 795aca4af..beb39ca20 100644 --- a/synapse/storage/schema/filtering.sql +++ b/synapse/storage/schema/filtering.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id TEXT, filter_id INTEGER, - definition TEXT, + filter_json TEXT, FOREIGN KEY(user_id) REFERENCES users(id) ); From 8552ed8df2990d79b0015e0e84dd98de25fd0a9d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 18:04:40 +0000 Subject: [PATCH 135/630] Change uses of get_user_by_req because it returns a tuple now. --- synapse/rest/client/v1/push_rule.py | 6 +++--- synapse/rest/client/v1/pusher.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2b33bdac0..64743a2f4 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -138,7 +138,7 @@ class PushRuleRestServlet(ClientV1RestServlet): except InvalidRuleException as e: raise SynapseError(400, e.message) - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -184,7 +184,7 @@ class PushRuleRestServlet(ClientV1RestServlet): except InvalidRuleException as e: raise SynapseError(400, e.message) - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) if 'device' in spec: rules = yield self.hs.get_datastore().get_push_rules_for_user_name( @@ -215,7 +215,7 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 80a11890a..72d5e9e47 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -27,7 +27,7 @@ class PusherRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - user = yield self.auth.get_user_by_req(request) + user, _ = yield self.auth.get_user_by_req(request) content = _parse_json(request) From d5bdf3c0c7958e6a080f9ec4b38a51428717d02a Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 28 Jan 2015 18:06:04 +0000 Subject: [PATCH 136/630] Allow the push rule delete method to take more specifiers. --- synapse/storage/push_rule.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 0342996ed..c7b553292 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -175,14 +175,17 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) @defer.inlineCallbacks - def delete_push_rule(self, user_name, rule_id): - yield self._simple_delete_one( - PushRuleTable.table_name, - { - 'user_name': user_name, - 'rule_id': rule_id - } - ) + def delete_push_rule(self, user_name, rule_id, **kwargs): + """ + Delete a push rule. Args specify the row to be deleted and can be + any of the columns in the push_rule table, but below are the + standard ones + + Args: + user_name (str): The matrix ID of the push rule owner + rule_id (str): The rule_id of the rule to be deleted + """ + yield self._simple_delete_one(PushRuleTable.table_name, kwargs) class RuleNotFoundException(Exception): From b0b80074e04dcb2b70dff56b0368060a19c065d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 01:48:36 +0000 Subject: [PATCH 137/630] SYN-252: Supply the stream and topological parts in the correct order to the constructor --- synapse/storage/stream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8ac2adab0..062ca06fb 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -82,10 +82,10 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): def parse(cls, string): try: if string[0] == 's': - return cls(None, int(string[1:])) + return cls(topological=None, stream=int(string[1:])) if string[0] == 't': parts = string[1:].split('-', 1) - return cls(int(parts[1]), int(parts[0])) + return cls(topological=int(parts[0]), stream=int(parts[1])) except: pass raise SynapseError(400, "Invalid token %r" % (string,)) @@ -94,7 +94,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): def parse_stream_token(cls, string): try: if string[0] == 's': - return cls(None, int(string[1:])) + return cls(topological=None, stream=int(string[1:])) except: pass raise SynapseError(400, "Invalid token %r" % (string,)) From 1b4a164c026ac027b7c84422f6c743c36896a0e7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 02:34:35 +0000 Subject: [PATCH 138/630] Add support for formatting events in the way a v2 client expects --- synapse/events/utils.py | 91 +++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 42fb0371e..dd7f3d6f4 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,7 +89,46 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): +def format_event_raw(d): + return d + + +def format_event_for_client_v1(d): + d["user_id"] = d.pop("sender", None) + + move_keys = ("age", "redacted_because", "replaces_state", "prev_content") + for key in move_keys: + if key in d["unsigned"]: + d[key] = d["unsigned"][key] + + drop_keys = ( + "auth_events", "prev_events", "hashes", "signatures", "depth", + "unsigned", "origin" + ) + for key in drop_keys: + d.pop(key, None) + return d + + +def format_event_for_client_v2(d): + drop_keys = ( + "auth_events", "prev_events", "hashes", "signatures", "depth", "origin" + ) + for key in drop_keys: + d.pop(key, None) + return d + + +def format_event_for_client_v2_without_event_id(d): + d = format_event_for_client_v2(d) + d.pop("room_id", None) + d.pop("event_id", None) + return d + + +def serialize_event(e, time_now_ms, as_client_event=True, + event_format=format_event_for_client_v1, + token_id=None): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -99,48 +138,22 @@ def serialize_event(e, time_now_ms, client_event=True, strip_ids=False): # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} - if not client_event: - # set the age and keep all other keys - if "age_ts" in d["unsigned"]: - d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] - return d - if "age_ts" in d["unsigned"]: - d["age"] = time_now_ms - d["unsigned"]["age_ts"] - del d["unsigned"]["age_ts"] - - d["user_id"] = d.pop("sender", None) + d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] + d["unsigned"]["age_ts"] if "redacted_because" in e.unsigned: - d["redacted_because"] = serialize_event( + d["unsigned"]["redacted_because"] = serialize_event( e.unsigned["redacted_because"], time_now_ms ) - del d["unsigned"]["redacted_because"] + if token_id is not None: + if token_id == e.internal_metadata["token_id"]: + txn_id = e.internal_metadata.get("txn_id", None) + if txn_id is not None: + d["unsigned"]["transaction_id"] = txn_id - if "redacted_by" in e.unsigned: - d["redacted_by"] = e.unsigned["redacted_by"] - del d["unsigned"]["redacted_by"] - - if "replaces_state" in e.unsigned: - d["replaces_state"] = e.unsigned["replaces_state"] - del d["unsigned"]["replaces_state"] - - if "prev_content" in e.unsigned: - d["prev_content"] = e.unsigned["prev_content"] - del d["unsigned"]["prev_content"] - - del d["auth_events"] - del d["prev_events"] - del d["hashes"] - del d["signatures"] - d.pop("depth", None) - d.pop("unsigned", None) - d.pop("origin", None) - d.pop("prev_state", None) - - if strip_ids: - d.pop("room_id", None) - d.pop("event_id", None) - - return d + if as_client_event: + return event_format(d) + else: + return d From b9c442c85c9f8c2aa7fcb57d6132dec9b85b4e60 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 02:45:33 +0000 Subject: [PATCH 139/630] Include transaction ids in unsigned section of events in the sync results for the clients that made those requests --- synapse/events/utils.py | 11 ++++++----- synapse/rest/client/v2_alpha/sync.py | 23 ++++++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index dd7f3d6f4..21316cc12 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -103,7 +103,7 @@ def format_event_for_client_v1(d): drop_keys = ( "auth_events", "prev_events", "hashes", "signatures", "depth", - "unsigned", "origin" + "unsigned", "origin", "prev_state" ) for key in drop_keys: d.pop(key, None) @@ -112,7 +112,8 @@ def format_event_for_client_v1(d): def format_event_for_client_v2(d): drop_keys = ( - "auth_events", "prev_events", "hashes", "signatures", "depth", "origin" + "auth_events", "prev_events", "hashes", "signatures", "depth", + "origin", "prev_state", ) for key in drop_keys: d.pop(key, None) @@ -140,7 +141,7 @@ def serialize_event(e, time_now_ms, as_client_event=True, if "age_ts" in d["unsigned"]: d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] - d["unsigned"]["age_ts"] + del d["unsigned"]["age_ts"] if "redacted_because" in e.unsigned: d["unsigned"]["redacted_because"] = serialize_event( @@ -148,8 +149,8 @@ def serialize_event(e, time_now_ms, as_client_event=True, ) if token_id is not None: - if token_id == e.internal_metadata["token_id"]: - txn_id = e.internal_metadata.get("txn_id", None) + if token_id == getattr(e.internal_metadata, "token_id", None): + txn_id = getattr(e.internal_metadata, "txn_id", None) if txn_id is not None: d["unsigned"]["transaction_id"] = txn_id diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index a0ab9839a..4d950f995 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -18,7 +18,9 @@ from twisted.internet import defer from synapse.http.servlet import RestServlet from synapse.handlers.sync import SyncConfig from synapse.types import StreamToken -from synapse.events.utils import serialize_event +from synapse.events.utils import ( + serialize_event, format_event_for_client_v2_without_event_id, +) from ._base import client_v2_pattern import logging @@ -139,7 +141,9 @@ class SyncRestServlet(RestServlet): "private_user_data": self.encode_events( sync_result.private_user_data, filter, time_now ), - "rooms": self.encode_rooms(sync_result.rooms, filter, time_now), + "rooms": self.encode_rooms( + sync_result.rooms, filter, time_now, client.token_id + ), "next_batch": sync_result.next_batch.to_string(), } @@ -153,25 +157,30 @@ class SyncRestServlet(RestServlet): # TODO(mjark): Respect formatting requirements in the filter. return serialize_event(event, time_now) - def encode_rooms(self, rooms, filter, time_now): - return [self.encode_room(room, filter, time_now) for room in rooms] + def encode_rooms(self, rooms, filter, time_now, token_id): + return [ + self.encode_room(room, filter, time_now, token_id) + for room in rooms + ] @staticmethod - def encode_room(room, filter, time_now): + def encode_room(room, filter, time_now, token_id): event_map = {} state_event_ids = [] recent_event_ids = [] for event in room.state: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( - event, time_now, strip_ids=True + event, time_now, token_id=token_id, + event_format=format_event_for_client_v2_without_event_id, ) state_event_ids.append(event.event_id) for event in room.events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( - event, time_now, strip_ids=True + event, time_now, token_id=token_id, + event_format=format_event_for_client_v2_without_event_id, ) recent_event_ids.append(event.event_id) return { From 3dbce6f4a59fde2a67e563ce338f510feda2dd1a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 03:33:51 +0000 Subject: [PATCH 140/630] Add typing notifications to sync --- synapse/handlers/sync.py | 32 +++++++++++++++++++--------- synapse/rest/client/v2_alpha/sync.py | 18 +++++++--------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 82a2c6986..b86e783e1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -44,11 +44,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "events", "state", "prev_batch", + "typing", ])): __slots__ = [] def __nonzero__(self): - return bool(self.events or self.state) + return bool(self.events or self.state or self.typing) class SyncResult(collections.namedtuple("SyncResult", [ @@ -196,15 +197,25 @@ class SyncHandler(BaseHandler): now_token = yield self.event_sources.get_current_token() - presence_stream = self.event_sources.sources["presence"] - pagination_config = PaginationConfig( - from_token=since_token, to_token=now_token - ) - presence, _ = yield presence_stream.get_pagination_rows( + presence_source = self.event_sources.sources["presence"] + presence, presence_key = yield presence_source.get_new_events_for_user( user=sync_config.user, - pagination_config=pagination_config.get_source_config("presence"), - key=None + from_key=since_token.presence_key, + limit=sync_config.limit, ) + now_token = now_token.copy_and_replace("presence_key", presence_key) + + typing_source = self.event_sources.sources["typing"] + typing, typing_key = yield typing_source.get_new_events_for_user( + user=sync_config.user, + from_key=since_token.typing_key, + limit=sync_config.limit, + ) + now_token = now_token.copy_and_replace("typing_key", typing_key) + + typing_by_room = {event["room_id"]: event for event in typing} + logger.debug("Typing %r", typing_by_room) + room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=sync_config.user.to_string(), membership_list=[Membership.INVITE, Membership.JOIN] @@ -218,7 +229,7 @@ class SyncHandler(BaseHandler): for event in room_list: room_sync = yield self.incremental_sync_with_gap_for_room( event.room_id, sync_config, since_token, now_token, - published_room_ids + published_room_ids, typing_by_room ) if room_sync: rooms.append(room_sync) @@ -233,7 +244,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, - published_room_ids): + published_room_ids, typing_by_room): """ Get the incremental delta needed to bring the client up to date for the room. Gives the client the most recent events and the changes to state. @@ -285,6 +296,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, + typing=typing_by_room.get(room_id, None) ) logging.debug("Room sync: %r", room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 4d950f995..76489e27c 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -135,10 +135,10 @@ class SyncRestServlet(RestServlet): time_now = self.clock.time_msec() response_content = { - "public_user_data": self.encode_events( + "public_user_data": self.encode_user_data( sync_result.public_user_data, filter, time_now ), - "private_user_data": self.encode_events( + "private_user_data": self.encode_user_data( sync_result.private_user_data, filter, time_now ), "rooms": self.encode_rooms( @@ -149,13 +149,8 @@ class SyncRestServlet(RestServlet): defer.returnValue((200, response_content)) - def encode_events(self, events, filter, time_now): - return [self.encode_event(event, filter, time_now) for event in events] - - @staticmethod - def encode_event(event, filter, time_now): - # TODO(mjark): Respect formatting requirements in the filter. - return serialize_event(event, time_now) + def encode_user_data(self, events, filter, time_now): + return events def encode_rooms(self, rooms, filter, time_now, token_id): return [ @@ -183,7 +178,7 @@ class SyncRestServlet(RestServlet): event_format=format_event_for_client_v2_without_event_id, ) recent_event_ids.append(event.event_id) - return { + result = { "room_id": room.room_id, "event_map": event_map, "events": { @@ -194,6 +189,9 @@ class SyncRestServlet(RestServlet): "limited": room.limited, "published": room.published, } + if room.typing is not None: + result["typing"] = room.typing + return result def register_servlets(hs, http_server): From e3e72b8c5c004c403929406fc952141fd0f1b8ae Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 03:35:25 +0000 Subject: [PATCH 141/630] Remove typing TODO --- synapse/handlers/sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b86e783e1..9e1188da5 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -255,7 +255,6 @@ class SyncHandler(BaseHandler): # the previous sync and this one. # TODO(mjark): Apply the event filter in sync_config # TODO(mjark): Check for redactions we might have missed. - # TODO(mjark): Typing notifications. recents, token = yield self.store.get_recent_events_for_room( room_id, limit=sync_config.limit + 1, From 3773759c0f1e25e6905e23368f770da99ceb3ea0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 09:15:33 +0000 Subject: [PATCH 142/630] Also edit the filter column on the delta SQL --- synapse/storage/schema/delta/v12.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index 795aca4af..beb39ca20 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id TEXT, filter_id INTEGER, - definition TEXT, + filter_json TEXT, FOREIGN KEY(user_id) REFERENCES users(id) ); From 2a4fda7b88cf91db8de2e524df162153d3f27094 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 09:27:16 +0000 Subject: [PATCH 143/630] Add filtering.filter_events function, with stub passes_filter function. --- synapse/api/filtering.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 6c7a73b6d..d7ba6510e 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -34,6 +34,21 @@ class Filtering(object): # replace_user_filter at some point? There's no REST API specified for # them however + def passes_filter(self, filter_json, event): + """Check if the event passes through the filter. + + Args: + filter_json(dict): The filter specification + event(Event): The event to check + Returns: + True if the event passes through the filter. + """ + return True + + def filter_events(self, events, user, filter_id): + filter_json = self.get_user_filter(user, filter_id) + return [e for e in events if self.passes_filter(filter_json, e)] + def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. From 50de1eaad94715a1dda470f44f379683e5fa552b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 10:24:57 +0000 Subject: [PATCH 144/630] Add filtering public API; outline filtering algorithm. --- synapse/api/filtering.py | 60 ++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index d7ba6510e..21fe72d6c 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -30,25 +30,69 @@ class Filtering(object): self._check_valid_filter(user_filter) return self.store.add_user_filter(user_localpart, user_filter) + def filter_public_user_data(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["public_user_data"] + ) + + def filter_private_user_data(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["private_user_data"] + ) + + def filter_room_state(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "state"] + ) + + def filter_room_events(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "events"] + ) + + def filter_room_ephemeral(self, events, user, filter_id): + return self._filter_on_key( + events, user, filter_id, ["room", "ephemeral"] + ) + # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however - def passes_filter(self, filter_json, event): - """Check if the event passes through the filter. + def _filter_on_key(self, events, user, filter_id, keys): + filter_json = self.get_user_filter(user.localpart, filter_id) + if not filter_json: + return events + + try: + # extract the right definition from the filter + definition = filter_json + for key in keys: + definition = definition[key] + return self._filter_with_definition(events, definition) + except KeyError: + return events # return all events if definition isn't specified. + + def _filter_with_definition(self, events, definition): + return [e for e in events if self._passes_definition(definition, e)] + + def _passes_definition(self, definition, event): + """Check if the event passes through the given definition. Args: - filter_json(dict): The filter specification - event(Event): The event to check + definition(dict): The definition to check against. + event(Event): The event to check. Returns: True if the event passes through the filter. """ + # Algorithm notes: + # For each key in the definition, check the event meets the criteria: + # * For types: Literal match or prefix match (if ends with wildcard) + # * For senders/rooms: Literal match only + # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' + # and 'not_types' then it is treated as only being in 'not_types') return True - def filter_events(self, events, user, filter_id): - filter_json = self.get_user_filter(user, filter_id) - return [e for e in events if self.passes_filter(filter_json, e)] - def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. From 777d9914b537d06ebba91948a26d74d3a04b7284 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 11:38:06 +0000 Subject: [PATCH 145/630] Implement filter algorithm. Add basic event type unit tests to assert it works. --- synapse/api/filtering.py | 49 +++++++++++++++++++++++++++++++++++++ tests/api/test_filtering.py | 45 +++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 21fe72d6c..8bc95aa39 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -91,8 +91,57 @@ class Filtering(object): # * For senders/rooms: Literal match only # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' # and 'not_types' then it is treated as only being in 'not_types') + + # room checks + if hasattr(event, "room_id"): + room_id = event.room_id + allow_rooms = definition["rooms"] if "rooms" in definition else None + reject_rooms = ( + definition["not_rooms"] if "not_rooms" in definition else None + ) + if reject_rooms and room_id in reject_rooms: + return False + if allow_rooms and room_id not in allow_rooms: + return False + + # sender checks + if hasattr(event, "sender"): + # Should we be including event.state_key for some event types? + sender = event.sender + allow_senders = ( + definition["senders"] if "senders" in definition else None + ) + reject_senders = ( + definition["not_senders"] if "not_senders" in definition else None + ) + if reject_senders and sender in reject_senders: + return False + if allow_senders and sender not in allow_senders: + return False + + # type checks + if "not_types" in definition: + for def_type in definition["not_types"]: + if self._event_matches_type(event, def_type): + return False + if "types" in definition: + included = False + for def_type in definition["types"]: + if self._event_matches_type(event, def_type): + included = True + break + if not included: + return False + return True + def _event_matches_type(self, event, def_type): + if def_type.endswith("*"): + type_prefix = def_type[:-1] + return event.type.startswith(type_prefix) + else: + return event.type == def_type + def _check_valid_filter(self, user_filter): """Check if the provided filter is valid. diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 188fbfb91..4d40d88b0 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from collections import namedtuple from tests import unittest from twisted.internet import defer @@ -27,6 +27,7 @@ from synapse.server import HomeServer user_localpart = "test_user" +MockEvent = namedtuple("MockEvent", "sender type room_id") class FilteringTestCase(unittest.TestCase): @@ -55,6 +56,48 @@ class FilteringTestCase(unittest.TestCase): self.datastore = hs.get_datastore() + def test_definition_include_literal_types(self): + definition = { + "types": ["m.room.message", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!foo:bar" + ) + + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_include_wildcard_types(self): + definition = { + "types": ["m.*", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!foo:bar" + ) + + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_exclude_unknown_types(self): + definition = { + "types": ["m.room.message", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="now.for.something.completely.different", + room_id="!foo:bar" + ) + + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + @defer.inlineCallbacks def test_add_filter(self): user_filter = { From 5561a879205316ae2c4cd0106cfd99d4fe35bceb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 12:06:16 +0000 Subject: [PATCH 146/630] Add more unit tests for the filter algorithm. --- tests/api/test_filtering.py | 264 +++++++++++++++++++++++++++++++++++- 1 file changed, 259 insertions(+), 5 deletions(-) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 4d40d88b0..380dd9793 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -56,7 +56,7 @@ class FilteringTestCase(unittest.TestCase): self.datastore = hs.get_datastore() - def test_definition_include_literal_types(self): + def test_definition_types_works_with_literals(self): definition = { "types": ["m.room.message", "org.matrix.foo.bar"] } @@ -65,12 +65,11 @@ class FilteringTestCase(unittest.TestCase): type="m.room.message", room_id="!foo:bar" ) - self.assertTrue( self.filtering._passes_definition(definition, event) ) - def test_definition_include_wildcard_types(self): + def test_definition_types_works_with_wildcards(self): definition = { "types": ["m.*", "org.matrix.foo.bar"] } @@ -79,12 +78,11 @@ class FilteringTestCase(unittest.TestCase): type="m.room.message", room_id="!foo:bar" ) - self.assertTrue( self.filtering._passes_definition(definition, event) ) - def test_definition_exclude_unknown_types(self): + def test_definition_types_works_with_unknowns(self): definition = { "types": ["m.room.message", "org.matrix.foo.bar"] } @@ -93,7 +91,263 @@ class FilteringTestCase(unittest.TestCase): type="now.for.something.completely.different", room_id="!foo:bar" ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + def test_definition_not_types_works_with_literals(self): + definition = { + "not_types": ["m.room.message", "org.matrix.foo.bar"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_types_works_with_wildcards(self): + definition = { + "not_types": ["m.room.message", "org.matrix.*"] + } + event = MockEvent( + sender="@foo:bar", + type="org.matrix.custom.event", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_types_works_with_unknowns(self): + definition = { + "not_types": ["m.*", "org.*"] + } + event = MockEvent( + sender="@foo:bar", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_types_takes_priority_over_types(self): + definition = { + "not_types": ["m.*", "org.*"], + "types": ["m.room.message", "m.room.topic"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.topic", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_senders_works_with_literals(self): + definition = { + "senders": ["@flibble:wibble"] + } + event = MockEvent( + sender="@flibble:wibble", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_senders_works_with_unknowns(self): + definition = { + "senders": ["@flibble:wibble"] + } + event = MockEvent( + sender="@challenger:appears", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_senders_works_with_literals(self): + definition = { + "not_senders": ["@flibble:wibble"] + } + event = MockEvent( + sender="@flibble:wibble", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_senders_works_with_unknowns(self): + definition = { + "not_senders": ["@flibble:wibble"] + } + event = MockEvent( + sender="@challenger:appears", + type="com.nom.nom.nom", + room_id="!foo:bar" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_senders_takes_priority_over_senders(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets", "@misspiggy:muppets"] + } + event = MockEvent( + sender="@misspiggy:muppets", + type="m.room.topic", + room_id="!foo:bar" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_rooms_works_with_literals(self): + definition = { + "rooms": ["!secretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!secretbase:unknown" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_rooms_works_with_unknowns(self): + definition = { + "rooms": ["!secretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!anothersecretbase:unknown" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_rooms_works_with_literals(self): + definition = { + "not_rooms": ["!anothersecretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!anothersecretbase:unknown" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_rooms_works_with_unknowns(self): + definition = { + "not_rooms": ["!secretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!anothersecretbase:unknown" + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_not_rooms_takes_priority_over_rooms(self): + definition = { + "not_rooms": ["!secretbase:unknown"], + "rooms": ["!secretbase:unknown"] + } + event = MockEvent( + sender="@foo:bar", + type="m.room.message", + room_id="!secretbase:unknown" + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_combined_event(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets"], + "rooms": ["!stage:unknown"], + "not_rooms": ["!piggyshouse:muppets"], + "types": ["m.room.message", "muppets.kermit.*"], + "not_types": ["muppets.misspiggy.*"] + } + event = MockEvent( + sender="@kermit:muppets", # yup + type="m.room.message", # yup + room_id="!stage:unknown" # yup + ) + self.assertTrue( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_combined_event_bad_sender(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets"], + "rooms": ["!stage:unknown"], + "not_rooms": ["!piggyshouse:muppets"], + "types": ["m.room.message", "muppets.kermit.*"], + "not_types": ["muppets.misspiggy.*"] + } + event = MockEvent( + sender="@misspiggy:muppets", # nope + type="m.room.message", # yup + room_id="!stage:unknown" # yup + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_combined_event_bad_room(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets"], + "rooms": ["!stage:unknown"], + "not_rooms": ["!piggyshouse:muppets"], + "types": ["m.room.message", "muppets.kermit.*"], + "not_types": ["muppets.misspiggy.*"] + } + event = MockEvent( + sender="@kermit:muppets", # yup + type="m.room.message", # yup + room_id="!piggyshouse:muppets" # nope + ) + self.assertFalse( + self.filtering._passes_definition(definition, event) + ) + + def test_definition_combined_event_bad_type(self): + definition = { + "not_senders": ["@misspiggy:muppets"], + "senders": ["@kermit:muppets"], + "rooms": ["!stage:unknown"], + "not_rooms": ["!piggyshouse:muppets"], + "types": ["m.room.message", "muppets.kermit.*"], + "not_types": ["muppets.misspiggy.*"] + } + event = MockEvent( + sender="@kermit:muppets", # yup + type="muppets.misspiggy.kisses", # nope + room_id="!stage:unknown" # yup + ) self.assertFalse( self.filtering._passes_definition(definition, event) ) From 83172487b05d7d99ccae0b353daee2f242445011 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 12:20:59 +0000 Subject: [PATCH 147/630] Add basic filtering public API unit tests. Use defers in the right places. --- synapse/api/filtering.py | 11 +++++--- tests/api/test_filtering.py | 54 ++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 8bc95aa39..7e239138b 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID @@ -59,19 +60,21 @@ class Filtering(object): # replace_user_filter at some point? There's no REST API specified for # them however + @defer.inlineCallbacks def _filter_on_key(self, events, user, filter_id, keys): - filter_json = self.get_user_filter(user.localpart, filter_id) + filter_json = yield self.get_user_filter(user.localpart, filter_id) if not filter_json: - return events + defer.returnValue(events) try: # extract the right definition from the filter definition = filter_json for key in keys: definition = definition[key] - return self._filter_with_definition(events, definition) + defer.returnValue(self._filter_with_definition(events, definition)) except KeyError: - return events # return all events if definition isn't specified. + # return all events if definition isn't specified. + defer.returnValue(events) def _filter_with_definition(self, events, definition): return [e for e in events if self._passes_definition(definition, e)] diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 380dd9793..97fb9758e 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -24,7 +24,7 @@ from tests.utils import ( ) from synapse.server import HomeServer - +from synapse.types import UserID user_localpart = "test_user" MockEvent = namedtuple("MockEvent", "sender type room_id") @@ -352,6 +352,58 @@ class FilteringTestCase(unittest.TestCase): self.filtering._passes_definition(definition, event) ) + @defer.inlineCallbacks + def test_filter_public_user_data_match(self): + user_filter = { + "public_user_data": { + "types": ["m.*"] + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="m.profile", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_public_user_data( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals(events, results) + + @defer.inlineCallbacks + def test_filter_public_user_data_no_match(self): + user_filter = { + "public_user_data": { + "types": ["m.*"] + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="custom.avatar.3d.crazy", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_public_user_data( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals([], results) + @defer.inlineCallbacks def test_add_filter(self): user_filter = { From c183cec8f61f5b3488973f01ca5203183a00e6d1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 13:44:52 +0000 Subject: [PATCH 148/630] Add post_json(...) method to federation client --- synapse/http/matrixfederationclient.py | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1dda3ba2c..b1b2916fd 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -244,6 +244,43 @@ class MatrixFederationHttpClient(object): defer.returnValue((response.code, body)) + @defer.inlineCallbacks + def post_json(self, destination, path, data={}): + """ Sends the specifed json data using POST + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + data (dict): A dict containing the data that will be used as + the request body. This will be encoded as JSON. + + Returns: + Deferred: Succeeds when we get a 2xx HTTP response. The result + will be the decoded JSON body. On a 4xx or 5xx error response a + CodeMessageException is raised. + """ + + def body_callback(method, url_bytes, headers_dict): + self.sign_request( + destination, method, url_bytes, headers_dict, data + ) + return None + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={"Content-Type": ["application/json"]}, + ) + + logger.debug("Getting resp body") + body = yield readBody(response) + logger.debug("Got resp body") + + defer.returnValue((response.code, body)) + @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): """ GETs some json from the given host homeserver and path From 5a3a15f5c186af6b818f2b2d3a4dafeee48b4e33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 13:58:22 +0000 Subject: [PATCH 149/630] Make post_json(...) actually send data. --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b1b2916fd..c7bf1b47b 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -265,7 +265,7 @@ class MatrixFederationHttpClient(object): self.sign_request( destination, method, url_bytes, headers_dict, data ) - return None + return _JsonProducer(data) response = yield self._create_request( destination.encode("ascii"), From 38b27bd2cbf38141938d6170c41e1d1dac9928cd Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 29 Jan 2015 14:28:34 +0000 Subject: [PATCH 150/630] Add filter_room_state unit tests. --- tests/api/test_filtering.py | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 97fb9758e..aa93616a9 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -404,6 +404,62 @@ class FilteringTestCase(unittest.TestCase): ) self.assertEquals([], results) + @defer.inlineCallbacks + def test_filter_room_state_match(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="m.room.topic", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_room_state( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals(events, results) + + @defer.inlineCallbacks + def test_filter_room_state_no_match(self): + user_filter = { + "room": { + "state": { + "types": ["m.*"] + } + } + } + user = UserID.from_string("@" + user_localpart + ":test") + filter_id = yield self.datastore.add_user_filter( + user_localpart=user_localpart, + user_filter=user_filter, + ) + event = MockEvent( + sender="@foo:bar", + type="org.matrix.custom.event", + room_id="!foo:bar" + ) + events = [event] + + results = yield self.filtering.filter_room_state( + events=events, + user=user, + filter_id=filter_id + ) + self.assertEquals([], results) + @defer.inlineCallbacks def test_add_filter(self): user_filter = { From e016f4043b81ffdedf71c4459772f66757386e44 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 14:40:28 +0000 Subject: [PATCH 151/630] Use get_room_events_stream to get changes to the rooms if the number of changes is small --- synapse/handlers/sync.py | 56 +++++++++++++++++++++++++++++++-------- synapse/storage/stream.py | 7 +++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9e1188da5..e93dfe005 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -216,23 +216,57 @@ class SyncHandler(BaseHandler): typing_by_room = {event["room_id"]: event for event in typing} logger.debug("Typing %r", typing_by_room) - room_list = yield self.store.get_rooms_for_user_where_membership_is( - user_id=sync_config.user.to_string(), - membership_list=[Membership.INVITE, Membership.JOIN] - ) + rm_handler = self.hs.get_handlers().room_member_handler + room_ids = yield rm_handler.get_rooms_for_user(sync_config.user) # TODO (mjark): Does public mean "published"? published_rooms = yield self.store.get_rooms(is_public=True) published_room_ids = set(r["room_id"] for r in published_rooms) + room_events, _ = yield self.store.get_room_events_stream( + sync_config.user.to_string(), + from_key=since_token.room_key, + to_key=now_token.room_key, + room_id=None, + limit=sync_config.limit + 1, + ) + rooms = [] - for event in room_list: - room_sync = yield self.incremental_sync_with_gap_for_room( - event.room_id, sync_config, since_token, now_token, - published_room_ids, typing_by_room - ) - if room_sync: - rooms.append(room_sync) + if len(room_events) <= sync_config.limit: + # There is no gap in any of the rooms. Therefore we can just + # partition the new events by room and return them. + events_by_room_id = {} + for event in room_events: + events_by_room_id.setdefault(event.room_id, []).append(event) + + for room_id in room_ids: + recents = events_by_room_id.get(room_id, []) + state = [event for event in recents if event.is_state()] + if recents: + prev_batch = now_token.copy_and_replace( + "room_key", recents[0].internal_metadata.before + ) + else: + prev_batch = now_token + room_sync = RoomSyncResult( + room_id=room_id, + published=room_id in published_room_ids, + events=recents, + prev_batch=prev_batch, + state=state, + limited=False, + typing=typing_by_room.get(room_id, None) + ) + if room_sync is not None: + rooms.append(room_sync) + else: + for room_id in room_ids: + room_sync = yield self.incremental_sync_with_gap_for_room( + room_id, sync_config, since_token, now_token, + published_room_ids, typing_by_room + ) + if room_sync: + rooms.append(room_sync) defer.returnValue(SyncResult( public_user_data=presence, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index db1816ea8..93ccfd8c1 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -181,6 +181,13 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + for event, row in zip(ret, rows): + stream = row["stream_ordering"] + topo = event.depth + internal = event.internal_metadata + internal.before = str(_StreamToken(topo, stream - 1)) + internal.after = str(_StreamToken(topo, stream)) + if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) else: From e4f50fa0aa3426a272b1526072c4c42802989ba4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 14:53:18 +0000 Subject: [PATCH 152/630] Move bump schema delta --- synapse/storage/schema/delta/{v12.sql => v13.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename synapse/storage/schema/delta/{v12.sql => v13.sql} (100%) diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v13.sql similarity index 100% rename from synapse/storage/schema/delta/v12.sql rename to synapse/storage/schema/delta/v13.sql From 33391db5f8d9d0d365607ca50ba59ce72c90cda0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 15:54:54 +0000 Subject: [PATCH 153/630] Merge in auth changes from develop --- synapse/rest/client/v2_alpha/filter.py | 4 ++-- tests/rest/client/v2_alpha/__init__.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 81a3e9515..cee06ccac 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -40,7 +40,7 @@ class GetFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, filter_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) if target_user != auth_user: raise AuthError(403, "Cannot get filters for other users") @@ -76,7 +76,7 @@ class CreateFilterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): target_user = UserID.from_string(user_id) - auth_user = yield self.auth.get_user_by_req(request) + auth_user, client = yield self.auth.get_user_by_req(request) if target_user != auth_user: raise AuthError(403, "Cannot create filters for other users") diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index 3fe62d5ac..fa70575c5 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -51,6 +51,7 @@ class V2AlphaRestTestCase(unittest.TestCase): "user": UserID.from_string(self.USER_ID), "admin": False, "device_id": None, + "token_id": 1, } hs.get_auth().get_user_by_token = _get_user_by_token From 9150a0d62ed4195b41834cea8a836332e74fb96b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:01:14 +0000 Subject: [PATCH 154/630] Fix code-style --- synapse/api/filtering.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 7e239138b..e16c0e559 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -74,7 +74,7 @@ class Filtering(object): defer.returnValue(self._filter_with_definition(events, definition)) except KeyError: # return all events if definition isn't specified. - defer.returnValue(events) + defer.returnValue(events) def _filter_with_definition(self, events, definition): return [e for e in events if self._passes_definition(definition, e)] @@ -94,14 +94,12 @@ class Filtering(object): # * For senders/rooms: Literal match only # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' # and 'not_types' then it is treated as only being in 'not_types') - + # room checks if hasattr(event, "room_id"): room_id = event.room_id - allow_rooms = definition["rooms"] if "rooms" in definition else None - reject_rooms = ( - definition["not_rooms"] if "not_rooms" in definition else None - ) + allow_rooms = definition.get("rooms", None) + reject_rooms = definition.get("not_rooms", None) if reject_rooms and room_id in reject_rooms: return False if allow_rooms and room_id not in allow_rooms: @@ -111,12 +109,8 @@ class Filtering(object): if hasattr(event, "sender"): # Should we be including event.state_key for some event types? sender = event.sender - allow_senders = ( - definition["senders"] if "senders" in definition else None - ) - reject_senders = ( - definition["not_senders"] if "not_senders" in definition else None - ) + allow_senders = definition.get("senders", None) + reject_senders = definition.get("not_senders", None) if reject_senders and sender in reject_senders: return False if allow_senders and sender not in allow_senders: @@ -176,7 +170,6 @@ class Filtering(object): if key in user_filter["room"]: self._check_definition(user_filter["room"][key]) - def _check_definition(self, definition): """Check if the provided definition is valid. From 8b1dd9f57f8afb2d602d3b533ab89dbe1df6b465 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 16:10:01 +0000 Subject: [PATCH 155/630] Only send a badge-reset if the user actually has unread notifications. --- synapse/push/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 10ac89048..fa967c5a5 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -56,6 +56,7 @@ class Pusher(object): # The last value of last_active_time that we saw self.last_last_active_time = 0 + self.has_unread = True @defer.inlineCallbacks def _actions_for_event(self, ev): @@ -180,6 +181,7 @@ class Pusher(object): processed = True else: rejected = yield self.dispatch_push(single_event, tweaks) + self.has_unread = True if isinstance(rejected, list) or isinstance(rejected, tuple): processed = True for pk in rejected: @@ -290,9 +292,12 @@ class Pusher(object): if 'last_active' in state.state: last_active = state.state['last_active'] if last_active > self.last_last_active_time: - logger.info("Resetting badge count for %s", self.user_name) - self.reset_badge_count() self.last_last_active_time = last_active + if self.has_unread: + logger.info("Resetting badge count for %s", self.user_name) + self.reset_badge_count() + self.has_unread = False + def _value_for_dotted_key(dotted_key, event): From acb68a39e02f405c116135400e33a3b1940a07f8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:10:35 +0000 Subject: [PATCH 156/630] Code style fixes. --- synapse/api/errors.py | 1 + synapse/push/__init__.py | 15 +++++++-------- synapse/push/httppusher.py | 8 ++++---- synapse/push/pusherpool.py | 2 +- synapse/rest/__init__.py | 2 +- synapse/rest/client/v1/push_rule.py | 29 ++++++++++++++++++++++------- synapse/storage/push_rule.py | 9 +++++---- synapse/storage/pusher.py | 2 +- 8 files changed, 42 insertions(+), 26 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5872e82d0..ad478aa6b 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -111,6 +111,7 @@ class NotFoundError(SynapseError): **kwargs ) + class AuthError(SynapseError): """An error raised when there was a problem authorising an event.""" diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index fa967c5a5..472ede548 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -189,8 +189,8 @@ class Pusher(object): # for sanity, we only remove the pushkey if it # was the one we actually sent... logger.warn( - ("Ignoring rejected pushkey %s because we " - "didn't send it"), pk + ("Ignoring rejected pushkey %s because we" + " didn't send it"), pk ) else: logger.info( @@ -236,8 +236,7 @@ class Pusher(object): # of old notifications. logger.warn("Giving up on a notification to user %s, " "pushkey %s", - self.user_name, self.pushkey - ) + self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token( @@ -258,8 +257,7 @@ class Pusher(object): "Trying again in %dms", self.user_name, self.clock.time_msec() - self.failing_since, - self.backoff_delay - ) + self.backoff_delay) yield synapse.util.async.sleep(self.backoff_delay / 1000.0) self.backoff_delay *= 2 if self.backoff_delay > Pusher.MAX_BACKOFF: @@ -299,7 +297,6 @@ class Pusher(object): self.has_unread = False - def _value_for_dotted_key(dotted_key, event): parts = dotted_key.split(".") val = event @@ -310,6 +307,7 @@ def _value_for_dotted_key(dotted_key, event): parts = parts[1:] return val + def _tweaks_for_actions(actions): tweaks = {} for a in actions: @@ -319,6 +317,7 @@ def _tweaks_for_actions(actions): tweaks['sound'] = a['set_sound'] return tweaks + class PusherConfigException(Exception): def __init__(self, msg): - super(PusherConfigException, self).__init__(msg) \ No newline at end of file + super(PusherConfigException, self).__init__(msg) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index e12b94672..ab128e31e 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -71,11 +71,11 @@ class HttpPusher(Pusher): # we may have to fetch this over federation and we # can't trust it anyway: is it worth it? #'from_display_name': 'Steve Stevington' - 'counts': { #-- we don't mark messages as read yet so - # we have no way of knowing + 'counts': { # -- we don't mark messages as read yet so + # we have no way of knowing # Just set the badge to 1 until we have read receipts 'unread': 1, - # 'missed_calls': 2 + # 'missed_calls': 2 }, 'devices': [ { @@ -142,4 +142,4 @@ class HttpPusher(Pusher): rejected = [] if 'rejected' in resp: rejected = resp['rejected'] - defer.returnValue(rejected) \ No newline at end of file + defer.returnValue(rejected) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 856defeda..4892c21e7 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -149,4 +149,4 @@ class PusherPool: logger.info("Stopping pusher %s", fullid) self.pushers[fullid].stop() del self.pushers[fullid] - yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) \ No newline at end of file + yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 90afd9333..1a84d94cd 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -11,4 +11,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 64743a2f4..2b1e93032 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -30,9 +30,9 @@ class PushRuleRestServlet(ClientV1RestServlet): 'sender': 1, 'room': 2, 'content': 3, - 'override': 4 + 'override': 4, } - PRIORITY_CLASS_INVERSE_MAP = {v: k for k,v in PRIORITY_CLASS_MAP.items()} + PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( "Unrecognised request: You probably wanted a trailing slash") @@ -260,7 +260,9 @@ class PushRuleRestServlet(ClientV1RestServlet): if path == []: # we're a reference impl: pedantry is our job. - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': defer.returnValue((200, rules)) @@ -271,7 +273,9 @@ class PushRuleRestServlet(ClientV1RestServlet): elif path[0] == 'device': path = path[1:] if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': defer.returnValue((200, rules['device'])) @@ -290,11 +294,13 @@ class PushRuleRestServlet(ClientV1RestServlet): def on_OPTIONS(self, _): return 200, {} + def _add_empty_priority_class_arrays(d): for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): d[pc] = [] return d + def _instance_handle_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the @@ -305,9 +311,12 @@ def _instance_handle_from_conditions(conditions): return c['instance_handle'] return None + def _filter_ruleset_with_path(ruleset, path): if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': return ruleset @@ -316,7 +325,9 @@ def _filter_ruleset_with_path(ruleset, path): raise UnrecognizedRequestError() path = path[1:] if path == []: - raise UnrecognizedRequestError(PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR) + raise UnrecognizedRequestError( + PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR + ) if path[0] == '': return ruleset[template_kind] rule_id = path[0] @@ -325,6 +336,7 @@ def _filter_ruleset_with_path(ruleset, path): return r raise NotFoundError + def _priority_class_from_spec(spec): if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) @@ -335,6 +347,7 @@ def _priority_class_from_spec(spec): return pc + def _priority_class_to_template_name(pc): if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device @@ -343,6 +356,7 @@ def _priority_class_to_template_name(pc): else: return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: @@ -359,8 +373,9 @@ def _rule_to_template(rule): ret["pattern"] = thecond["pattern"] return ret + def _strip_device_condition(rule): - for i,c in enumerate(rule['conditions']): + for i, c in enumerate(rule['conditions']): if c['kind'] == 'device': del rule['conditions'][i] return rule diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index c7b553292..27502d239 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -117,7 +117,7 @@ class PushRuleStore(SQLBaseStore): new_rule['priority'] = new_rule_priority sql = ( - "SELECT COUNT(*) FROM "+PushRuleTable.table_name+ + "SELECT COUNT(*) FROM " + PushRuleTable.table_name + " WHERE user_name = ? AND priority_class = ? AND priority = ?" ) txn.execute(sql, (user_name, priority_class, new_rule_priority)) @@ -146,10 +146,11 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) - def _add_push_rule_highest_priority_txn(self, txn, user_name, priority_class, **kwargs): + def _add_push_rule_highest_priority_txn(self, txn, user_name, + priority_class, **kwargs): # find the highest priority rule in that class sql = ( - "SELECT COUNT(*), MAX(priority) FROM "+PushRuleTable.table_name+ + "SELECT COUNT(*), MAX(priority) FROM " + PushRuleTable.table_name + " WHERE user_name = ? and priority_class = ?" ) txn.execute(sql, (user_name, priority_class)) @@ -209,4 +210,4 @@ class PushRuleTable(Table): "actions", ] - EntryType = collections.namedtuple("PushRuleEntry", fields) \ No newline at end of file + EntryType = collections.namedtuple("PushRuleEntry", fields) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 113cdc8a8..f253c9e2c 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -170,4 +170,4 @@ class PushersTable(Table): "failing_since" ] - EntryType = collections.namedtuple("PusherEntry", fields) \ No newline at end of file + EntryType = collections.namedtuple("PusherEntry", fields) From 4d9dd9bdc02ca7b524c90ae527687e7a838acee4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:23:03 +0000 Subject: [PATCH 157/630] Fix v2 initial sync --- synapse/handlers/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index e93dfe005..cfe294491 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -157,7 +157,7 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def intial_sync_for_room(self, room_id, sync_config, now_token, + def initial_sync_for_room(self, room_id, sync_config, now_token, published_room_ids): """Sync a room for a client which is starting without any state Returns: @@ -180,6 +180,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=current_state_events, limited=True, + typing=None, )) @defer.inlineCallbacks From cc42d3f9076db1815a989bab7ef6004796b31b3c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:27:38 +0000 Subject: [PATCH 158/630] Fix check for empty room update --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index cfe294491..ec8beb4c6 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -258,7 +258,7 @@ class SyncHandler(BaseHandler): limited=False, typing=typing_by_room.get(room_id, None) ) - if room_sync is not None: + if room_sync: rooms.append(room_sync) else: for room_id in room_ids: From 722b65f46131349c5afdcc7eb48297cdd9d9cbd6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:41:21 +0000 Subject: [PATCH 159/630] Move typing notifs to an "emphermal" event list on the room object --- synapse/handlers/sync.py | 12 +++++++----- synapse/rest/client/v2_alpha/sync.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index ec8beb4c6..3860c3c95 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -44,12 +44,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ "events", "state", "prev_batch", - "typing", + "ephemeral", ])): __slots__ = [] def __nonzero__(self): - return bool(self.events or self.state or self.typing) + return bool(self.events or self.state or self.ephemeral) class SyncResult(collections.namedtuple("SyncResult", [ @@ -180,7 +180,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=current_state_events, limited=True, - typing=None, + ephemeral=[], )) @defer.inlineCallbacks @@ -214,7 +214,9 @@ class SyncHandler(BaseHandler): ) now_token = now_token.copy_and_replace("typing_key", typing_key) - typing_by_room = {event["room_id"]: event for event in typing} + typing_by_room = {event["room_id"]: [event] for event in typing} + for event in typing: + event.pop("room_id") logger.debug("Typing %r", typing_by_room) rm_handler = self.hs.get_handlers().room_member_handler @@ -256,7 +258,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch, state=state, limited=False, - typing=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, []) ) if room_sync: rooms.append(room_sync) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 76489e27c..2ae2eec55 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -66,6 +66,7 @@ class SyncRestServlet(RestServlet): } "state": [] // list of EventIDs updating the current state to // be what it should be at the end of the batch. + "ephemeral": [] }] } """ @@ -188,9 +189,8 @@ class SyncRestServlet(RestServlet): "state": state_event_ids, "limited": room.limited, "published": room.published, + "ephemeral": room.ephemeral, } - if room.typing is not None: - result["typing"] = room.typing return result From 4ad45f258245344218c5ac9ffcf20052bfdb5ba2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 16:41:49 +0000 Subject: [PATCH 160/630] Fix indent --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3860c3c95..8c7681a48 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -158,7 +158,7 @@ class SyncHandler(BaseHandler): @defer.inlineCallbacks def initial_sync_for_room(self, room_id, sync_config, now_token, - published_room_ids): + published_room_ids): """Sync a room for a client which is starting without any state Returns: A Deferred RoomSyncResult. From 78015948a7febb18e000651f72f8f58830a55b93 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 29 Jan 2015 16:50:23 +0000 Subject: [PATCH 161/630] Initial implementation of auth conflict resolution --- synapse/events/utils.py | 6 +- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 33 ++++ synapse/federation/transport/client.py | 16 ++ synapse/federation/transport/server.py | 21 ++- synapse/handlers/federation.py | 209 +++++++++++++++--------- synapse/storage/rejections.py | 4 +- tests/handlers/test_federation.py | 2 + 8 files changed, 211 insertions(+), 82 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index bcb545727..10a6b9f26 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -45,12 +45,14 @@ def prune_event(event): "membership", ] + event_dict = event.get_dict() + new_content = {} def add_fields(*fields): for field in fields: if field in event.content: - new_content[field] = event.content[field] + new_content[field] = event_dict["content"][field] if event_type == EventTypes.Member: add_fields("membership") @@ -75,7 +77,7 @@ def prune_event(event): allowed_fields = { k: v - for k, v in event.get_dict().items() + for k, v in event_dict.items() if k in allowed_keys } diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index ebcd59350..1173ca817 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -345,7 +345,7 @@ class FederationClient(object): "auth_chain": [e.get_pdu_json(time_now) for e in local_auth], } - code, content = yield self.transport_layer.send_invite( + code, content = yield self.transport_layer.send_query_auth( destination=destination, room_id=room_id, event_id=event_id, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index fc5342afa..8cff4e647 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -230,6 +230,39 @@ class FederationServer(object): "auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus], })) + @defer.inlineCallbacks + def on_query_auth_request(self, origin, content, event_id): + auth_chain = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content["auth_chain"] + ] + + missing = [ + (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + for e in content.get("missing", []) + ] + + ret = yield self.handler.on_query_auth( + origin, event_id, auth_chain, content.get("rejects", []), missing + ) + + time_now = self._clock.time_msec() + send_content = { + "auth_chain": [ + e.get_pdu_json(time_now) + for e in ret["auth_chain"] + ], + "rejects": content.get("rejects", []), + "missing": [ + e.get_pdu_json(time_now) + for e in ret.get("missing", []) + ], + } + + defer.returnValue( + (200, send_content) + ) + @log_function def _get_persisted_pdu(self, origin, event_id, do_auth=True): """ Get a PDU from the database with given origin and id. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index e634a3a21..4cb1dea2d 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -213,3 +213,19 @@ class TransportLayerClient(object): ) defer.returnValue(response) + + @defer.inlineCallbacks + @log_function + def send_query_auth(self, destination, room_id, event_id, content): + path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) + + code, content = yield self.client.post_json( + destination=destination, + path=path, + data=content, + ) + + if not 200 <= code < 300: + raise RuntimeError("Got %d from send_invite", code) + + defer.returnValue(json.loads(content)) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a380a6910..9c9f8d525 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -42,7 +42,7 @@ class TransportLayerServer(object): content = None origin = None - if request.method == "PUT": + if request.method in ["PUT", "POST"]: # TODO: Handle other method types? other content types? try: content_bytes = request.content.read() @@ -234,6 +234,16 @@ class TransportLayerServer(object): ) ) ) + self.server.register_path( + "POST", + re.compile("^" + PREFIX + "/query_auth/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, event_id: + self._on_query_auth_request( + origin, content, event_id, + ) + ) + ) @defer.inlineCallbacks @log_function @@ -325,3 +335,12 @@ class TransportLayerServer(object): ) defer.returnValue((200, content)) + + @defer.inlineCallbacks + @log_function + def _on_query_auth_request(self, origin, content, event_id): + new_content = yield self.request_handler.on_query_auth_request( + origin, content, event_id + ) + + defer.returnValue((200, new_content)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 97e3c503b..14c26d8ce 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -126,7 +126,7 @@ class FederationHandler(BaseHandler): if not state: state, auth_chain = yield replication.get_state_for_room( - origin, context=event.room_id, event_id=event.event_id, + origin, room_id=event.room_id, event_id=event.event_id, ) if not auth_chain: @@ -139,7 +139,7 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e, fetch_auth_from=origin) + yield self._handle_new_event(origin, e) except: logger.exception( "Failed to handle auth event %s", @@ -152,7 +152,7 @@ class FederationHandler(BaseHandler): for e in state: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e) + yield self._handle_new_event(origin, e) except: logger.exception( "Failed to handle state event %s", @@ -161,6 +161,7 @@ class FederationHandler(BaseHandler): try: yield self._handle_new_event( + origin, event, state=state, backfilled=backfilled, @@ -363,7 +364,14 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + target_host, e, auth_events=auth + ) except: logger.exception( "Failed to handle auth event %s", @@ -374,8 +382,13 @@ class FederationHandler(BaseHandler): # FIXME: Auth these. e.internal_metadata.outlier = True try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } yield self._handle_new_event( - e, fetch_auth_from=target_host + target_host, e, auth_events=auth ) except: logger.exception( @@ -384,6 +397,7 @@ class FederationHandler(BaseHandler): ) yield self._handle_new_event( + target_host, new_event, state=state, current_state=state, @@ -450,7 +464,7 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - context = yield self._handle_new_event(event) + context = yield self._handle_new_event(origin, event) logger.debug( "on_send_join_request: After _handle_new_event: %s, sigs: %s", @@ -651,11 +665,12 @@ class FederationHandler(BaseHandler): waiters.pop().callback(None) @defer.inlineCallbacks - def _handle_new_event(self, event, state=None, backfilled=False, - current_state=None, fetch_auth_from=None): + @log_function + def _handle_new_event(self, origin, event, state=None, backfilled=False, + current_state=None, auth_events=None): logger.debug( - "_handle_new_event: Before annotate: %s, sigs: %s", + "_handle_new_event: %s, sigs: %s", event.event_id, event.signatures, ) @@ -663,62 +678,34 @@ class FederationHandler(BaseHandler): event, old_state=state ) + if not auth_events: + auth_events = context.auth_events + logger.debug( - "_handle_new_event: Before auth fetch: %s, sigs: %s", - event.event_id, event.signatures, + "_handle_new_event: %s, auth_events: %s", + event.event_id, auth_events, ) is_new_state = not event.internal_metadata.is_outlier() - known_ids = set( - [s.event_id for s in context.auth_events.values()] - ) - - for e_id, _ in event.auth_events: - if e_id not in known_ids: - e = yield self.store.get_event(e_id, allow_none=True) - - if not e and fetch_auth_from is not None: - # Grab the auth_chain over federation if we are missing - # auth events. - auth_chain = yield self.replication_layer.get_event_auth( - fetch_auth_from, event.event_id, event.room_id - ) - for auth_event in auth_chain: - yield self._handle_new_event(auth_event) - e = yield self.store.get_event(e_id, allow_none=True) - - if not e: - # TODO: Do some conflict res to make sure that we're - # not the ones who are wrong. - logger.info( - "Rejecting %s as %s not in db or %s", - event.event_id, e_id, known_ids, - ) - # FIXME: How does raising AuthError work with federation? - raise AuthError(403, "Cannot find auth event") - - context.auth_events[(e.type, e.state_key)] = e - - logger.debug( - "_handle_new_event: Before hack: %s, sigs: %s", - event.event_id, event.signatures, - ) - + # This is a hack to fix some old rooms where the initial join event + # didn't reference the create event in its auth events. if event.type == EventTypes.Member and not event.auth_events: if len(event.prev_events) == 1: c = yield self.store.get_event(event.prev_events[0][0]) if c.type == EventTypes.Create: - context.auth_events[(c.type, c.state_key)] = c - - logger.debug( - "_handle_new_event: Before auth check: %s, sigs: %s", - event.event_id, event.signatures, - ) + auth_events[(c.type, c.state_key)] = c try: - self.auth.check(event, auth_events=context.auth_events) - except AuthError: + yield self.do_auth( + origin, event, context, auth_events=auth_events + ) + except AuthError as e: + logger.warn( + "Rejecting %s because %s", + event.event_id, e.msg + ) + # TODO: Store rejection. context.rejected = RejectedReason.AUTH_ERROR @@ -731,11 +718,6 @@ class FederationHandler(BaseHandler): ) raise - logger.debug( - "_handle_new_event: Before persist_event: %s, sigs: %s", - event.event_id, event.signatures, - ) - yield self.store.persist_event( event, context=context, @@ -744,25 +726,73 @@ class FederationHandler(BaseHandler): current_state=current_state, ) - logger.debug( - "_handle_new_event: After persist_event: %s, sigs: %s", - event.event_id, event.signatures, - ) - defer.returnValue(context) @defer.inlineCallbacks - def do_auth(self, origin, event, context): - for e_id, _ in event.auth_events: - pass + def on_query_auth(self, origin, event_id, remote_auth_chain, rejects, + missing): + # Just go through and process each event in `remote_auth_chain`. We + # don't want to fall into the trap of `missing` being wrong. + for e in remote_auth_chain: + try: + yield self._handle_new_event(origin, e) + except AuthError: + pass - auth_events = set(e_id for e_id, _ in event.auth_events) - current_state = set(e.event_id for e in context.auth_events.values()) + # Now get the current auth_chain for the event. + local_auth_chain = yield self.store.get_auth_chain([event_id]) - missing_auth = auth_events - current_state + # TODO: Check if we would now reject event_id. If so we need to tell + # everyone. + + ret = yield self.construct_auth_difference( + local_auth_chain, remote_auth_chain + ) + + logger.debug("on_query_auth reutrning: %s", ret) + + defer.returnValue(ret) + + @defer.inlineCallbacks + @log_function + def do_auth(self, origin, event, context, auth_events): + # Check if we have all the auth events. + res = yield self.store.have_events( + [e_id for e_id, _ in event.auth_events] + ) + + event_auth_events = set(e_id for e_id, _ in event.auth_events) + seen_events = set(res.keys()) + + missing_auth = event_auth_events - seen_events if missing_auth: + logger.debug("Missing auth: %s", missing_auth) + # If we don't have all the auth events, we need to get them. + remote_auth_chain = yield self.replication_layer.get_event_auth( + origin, event.room_id, event.event_id + ) + + for e in remote_auth_chain: + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in remote_auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass + + current_state = set(e.event_id for e in auth_events.values()) + different_auth = event_auth_events - current_state + + if different_auth and not event.internal_metadata.is_outlier(): # Do auth conflict res. + logger.debug("Different auth: %s", different_auth) # 1. Get what we think is the auth chain. auth_ids = self.auth.compute_auth_events(event, context) @@ -778,14 +808,24 @@ class FederationHandler(BaseHandler): # 3. Process any remote auth chain events we haven't seen. for e in result.get("missing", []): - # TODO. - pass + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in result["auth_chain"] + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass # 4. Look at rejects and their proofs. # TODO. try: - self.auth.check(event, auth_events=context.auth_events) + self.auth.check(event, auth_events=auth_events) except AuthError: raise @@ -802,12 +842,16 @@ class FederationHandler(BaseHandler): dict """ + logger.debug("construct_auth_difference Start!") + # TODO: Make sure we are OK with local_auth or remote_auth having more # auth events in them than strictly necessary. def sort_fun(ev): return ev.depth, ev.event_id + logger.debug("construct_auth_difference after sort_fun!") + # We find the differences by starting at the "bottom" of each list # and iterating up on both lists. The lists are ordered by depth and # then event_id, we iterate up both lists until we find the event ids @@ -823,11 +867,18 @@ class FederationHandler(BaseHandler): local_iter = iter(local_list) remote_iter = iter(remote_list) - current_local = local_iter.next() - current_remote = remote_iter.next() + logger.debug("construct_auth_difference before get_next!") def get_next(it, opt=None): - return it.next() if it.has_next() else opt + try: + return it.next() + except: + return opt + + current_local = get_next(local_iter) + current_remote = get_next(remote_iter) + + logger.debug("construct_auth_difference before while") missing_remotes = [] missing_locals = [] @@ -867,6 +918,8 @@ class FederationHandler(BaseHandler): current_remote = get_next(remote_iter) continue + logger.debug("construct_auth_difference after while") + # missing locals should be sent to the server # We should find why we are missing remotes, as they will have been # rejected. @@ -886,6 +939,7 @@ class FederationHandler(BaseHandler): reason = yield self.store.get_rejection_reason(e.event_id) if reason is None: # FIXME: ERRR?! + logger.warn("Could not find reason for %s", e.event_id) raise RuntimeError("") reason_map[e.event_id] = reason @@ -899,7 +953,10 @@ class FederationHandler(BaseHandler): # TODO: Get proof. pass + logger.debug("construct_auth_difference returning") + defer.returnValue({ + "auth_chain": local_auth, "rejects": { e.event_id: { "reason": reason_map[e.event_id], diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index b7249700d..4e1a9a278 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -28,12 +28,12 @@ class RejectionsStore(SQLBaseStore): values={ "event_id": event_id, "reason": reason, - "last_failure": self._clock.time_msec(), + "last_check": self._clock.time_msec(), } ) def get_rejection_reason(self, event_id): - self._simple_select_one_onecol( + return self._simple_select_one_onecol( table="rejections", retcol="reason", keyvalues={ diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index ed21defd1..44dbce6be 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -52,6 +52,7 @@ class FederationTestCase(unittest.TestCase): "get_room", "get_destination_retry_timings", "set_destination_retry_timings", + "have_events", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -90,6 +91,7 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) + self.datastore.have_events.return_value = defer.succeed({}) def annotate(ev, old_state=None): context = Mock() From e0d2c6889bf31cf5f48e77334fed23352d19a75d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 17:04:31 +0000 Subject: [PATCH 162/630] Allow kind to be set to null to delete a pusher. --- synapse/rest/client/v1/pusher.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 72d5e9e47..353a4a658 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -31,6 +31,16 @@ class PusherRestServlet(ClientV1RestServlet): content = _parse_json(request) + pusher_pool = self.hs.get_pusherpool() + + if ('pushkey' in content and 'app_id' in content + and 'kind' in content and + content['kind'] is None): + yield pusher_pool.remove_pusher( + content['app_id'], content['pushkey'] + ) + defer.returnValue((200, {})) + reqd = ['instance_handle', 'kind', 'app_id', 'app_display_name', 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] @@ -41,7 +51,6 @@ class PusherRestServlet(ClientV1RestServlet): raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) - pusher_pool = self.hs.get_pusherpool() try: yield pusher_pool.add_pusher( user_name=user.to_string(), From 4bdfce30d70dddaa7c6de551fe3c9eed4a899d49 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 17:12:11 +0000 Subject: [PATCH 163/630] Renumber priority classes so we can use 0 for defaults. --- synapse/rest/client/v1/push_rule.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 2b1e93032..0f78fa667 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -26,11 +26,11 @@ import json class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { - 'underride': 0, - 'sender': 1, - 'room': 2, - 'content': 3, - 'override': 4, + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, } PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( From 93ed31dda2e23742c3d7f3eee6ac6839682f0ce9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 17:41:48 +0000 Subject: [PATCH 164/630] Create a separate filter object to do the actual filtering, so that we can split the storage and management of filters from the actual filter code and don't have to load a filter from the db each time we filter an event --- synapse/api/filtering.py | 220 ++++++++++++------------- synapse/rest/client/v2_alpha/filter.py | 2 +- tests/api/test_filtering.py | 108 ++++++------ 3 files changed, 166 insertions(+), 164 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index e16c0e559..b7e5d3222 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -25,127 +25,25 @@ class Filtering(object): self.store = hs.get_datastore() def get_user_filter(self, user_localpart, filter_id): - return self.store.get_user_filter(user_localpart, filter_id) + result = self.store.get_user_filter(user_localpart, filter_id) + result.addCallback(Filter) + return result def add_user_filter(self, user_localpart, user_filter): self._check_valid_filter(user_filter) return self.store.add_user_filter(user_localpart, user_filter) - def filter_public_user_data(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["public_user_data"] - ) - - def filter_private_user_data(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["private_user_data"] - ) - - def filter_room_state(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "state"] - ) - - def filter_room_events(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "events"] - ) - - def filter_room_ephemeral(self, events, user, filter_id): - return self._filter_on_key( - events, user, filter_id, ["room", "ephemeral"] - ) - # TODO(paul): surely we should probably add a delete_user_filter or # replace_user_filter at some point? There's no REST API specified for # them however - @defer.inlineCallbacks - def _filter_on_key(self, events, user, filter_id, keys): - filter_json = yield self.get_user_filter(user.localpart, filter_id) - if not filter_json: - defer.returnValue(events) - - try: - # extract the right definition from the filter - definition = filter_json - for key in keys: - definition = definition[key] - defer.returnValue(self._filter_with_definition(events, definition)) - except KeyError: - # return all events if definition isn't specified. - defer.returnValue(events) - - def _filter_with_definition(self, events, definition): - return [e for e in events if self._passes_definition(definition, e)] - - def _passes_definition(self, definition, event): - """Check if the event passes through the given definition. - - Args: - definition(dict): The definition to check against. - event(Event): The event to check. - Returns: - True if the event passes through the filter. - """ - # Algorithm notes: - # For each key in the definition, check the event meets the criteria: - # * For types: Literal match or prefix match (if ends with wildcard) - # * For senders/rooms: Literal match only - # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' - # and 'not_types' then it is treated as only being in 'not_types') - - # room checks - if hasattr(event, "room_id"): - room_id = event.room_id - allow_rooms = definition.get("rooms", None) - reject_rooms = definition.get("not_rooms", None) - if reject_rooms and room_id in reject_rooms: - return False - if allow_rooms and room_id not in allow_rooms: - return False - - # sender checks - if hasattr(event, "sender"): - # Should we be including event.state_key for some event types? - sender = event.sender - allow_senders = definition.get("senders", None) - reject_senders = definition.get("not_senders", None) - if reject_senders and sender in reject_senders: - return False - if allow_senders and sender not in allow_senders: - return False - - # type checks - if "not_types" in definition: - for def_type in definition["not_types"]: - if self._event_matches_type(event, def_type): - return False - if "types" in definition: - included = False - for def_type in definition["types"]: - if self._event_matches_type(event, def_type): - included = True - break - if not included: - return False - - return True - - def _event_matches_type(self, event, def_type): - if def_type.endswith("*"): - type_prefix = def_type[:-1] - return event.type.startswith(type_prefix) - else: - return event.type == def_type - - def _check_valid_filter(self, user_filter): + def _check_valid_filter(self, user_filter_json): """Check if the provided filter is valid. This inspects all definitions contained within the filter. Args: - user_filter(dict): The filter + user_filter_json(dict): The filter Raises: SynapseError: If the filter is not valid. """ @@ -162,13 +60,13 @@ class Filtering(object): ] for key in top_level_definitions: - if key in user_filter: - self._check_definition(user_filter[key]) + if key in user_filter_json: + self._check_definition(user_filter_json[key]) - if "room" in user_filter: + if "room" in user_filter_json: for key in room_level_definitions: - if key in user_filter["room"]: - self._check_definition(user_filter["room"][key]) + if key in user_filter_json["room"]: + self._check_definition(user_filter_json["room"][key]) def _check_definition(self, definition): """Check if the provided definition is valid. @@ -237,3 +135,101 @@ class Filtering(object): if ("bundle_updates" in definition and type(definition["bundle_updates"]) != bool): raise SynapseError(400, "Bad bundle_updates: expected bool.") + + +class Filter(object): + def __init__(self, filter_json): + self.filter_json = filter_json + + def filter_public_user_data(self, events): + return self._filter_on_key(events, ["public_user_data"]) + + def filter_private_user_data(self, events): + return self._filter_on_key(events, ["private_user_data"]) + + def filter_room_state(self, events): + return self._filter_on_key(events, ["room", "state"]) + + def filter_room_events(self, events): + return self._filter_on_key(events, ["room", "events"]) + + def filter_room_ephemeral(self, events): + return self._filter_on_key(events, ["room", "ephemeral"]) + + def _filter_on_key(self, events, keys): + filter_json = self.filter_json + if not filter_json: + return events + + try: + # extract the right definition from the filter + definition = filter_json + for key in keys: + definition = definition[key] + return self._filter_with_definition(events, definition) + except KeyError: + # return all events if definition isn't specified. + return events + + def _filter_with_definition(self, events, definition): + return [e for e in events if self._passes_definition(definition, e)] + + def _passes_definition(self, definition, event): + """Check if the event passes through the given definition. + + Args: + definition(dict): The definition to check against. + event(Event): The event to check. + Returns: + True if the event passes through the filter. + """ + # Algorithm notes: + # For each key in the definition, check the event meets the criteria: + # * For types: Literal match or prefix match (if ends with wildcard) + # * For senders/rooms: Literal match only + # * "not_" checks take presedence (e.g. if "m.*" is in both 'types' + # and 'not_types' then it is treated as only being in 'not_types') + + # room checks + if hasattr(event, "room_id"): + room_id = event.room_id + allow_rooms = definition.get("rooms", None) + reject_rooms = definition.get("not_rooms", None) + if reject_rooms and room_id in reject_rooms: + return False + if allow_rooms and room_id not in allow_rooms: + return False + + # sender checks + if hasattr(event, "sender"): + # Should we be including event.state_key for some event types? + sender = event.sender + allow_senders = definition.get("senders", None) + reject_senders = definition.get("not_senders", None) + if reject_senders and sender in reject_senders: + return False + if allow_senders and sender not in allow_senders: + return False + + # type checks + if "not_types" in definition: + for def_type in definition["not_types"]: + if self._event_matches_type(event, def_type): + return False + if "types" in definition: + included = False + for def_type in definition["types"]: + if self._event_matches_type(event, def_type): + included = True + break + if not included: + return False + + return True + + def _event_matches_type(self, event, def_type): + if def_type.endswith("*"): + type_prefix = def_type[:-1] + return event.type.startswith(type_prefix) + else: + return event.type == def_type diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index cee06ccac..6ddc495d2 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -59,7 +59,7 @@ class GetFilterRestServlet(RestServlet): filter_id=filter_id, ) - defer.returnValue((200, filter)) + defer.returnValue((200, filter.filter_json)) except KeyError: raise SynapseError(400, "No such filter") diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index aa93616a9..babf4c37f 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -25,6 +25,7 @@ from tests.utils import ( from synapse.server import HomeServer from synapse.types import UserID +from synapse.api.filtering import Filter user_localpart = "test_user" MockEvent = namedtuple("MockEvent", "sender type room_id") @@ -53,6 +54,7 @@ class FilteringTestCase(unittest.TestCase): ) self.filtering = hs.get_filtering() + self.filter = Filter({}) self.datastore = hs.get_datastore() @@ -66,7 +68,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_types_works_with_wildcards(self): @@ -79,7 +81,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_types_works_with_unknowns(self): @@ -92,7 +94,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_literals(self): @@ -105,7 +107,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_wildcards(self): @@ -118,7 +120,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_works_with_unknowns(self): @@ -131,7 +133,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_types_takes_priority_over_types(self): @@ -145,7 +147,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_senders_works_with_literals(self): @@ -158,7 +160,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_senders_works_with_unknowns(self): @@ -171,7 +173,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_works_with_literals(self): @@ -184,7 +186,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_works_with_unknowns(self): @@ -197,7 +199,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_senders_takes_priority_over_senders(self): @@ -211,7 +213,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!foo:bar" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_rooms_works_with_literals(self): @@ -224,7 +226,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!secretbase:unknown" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_rooms_works_with_unknowns(self): @@ -237,7 +239,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_works_with_literals(self): @@ -250,7 +252,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_works_with_unknowns(self): @@ -263,7 +265,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!anothersecretbase:unknown" ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_not_rooms_takes_priority_over_rooms(self): @@ -277,7 +279,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!secretbase:unknown" ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event(self): @@ -295,7 +297,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertTrue( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_sender(self): @@ -313,7 +315,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_room(self): @@ -331,7 +333,7 @@ class FilteringTestCase(unittest.TestCase): room_id="!piggyshouse:muppets" # nope ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) def test_definition_combined_event_bad_type(self): @@ -349,12 +351,12 @@ class FilteringTestCase(unittest.TestCase): room_id="!stage:unknown" # yup ) self.assertFalse( - self.filtering._passes_definition(definition, event) + self.filter._passes_definition(definition, event) ) @defer.inlineCallbacks def test_filter_public_user_data_match(self): - user_filter = { + user_filter_json = { "public_user_data": { "types": ["m.*"] } @@ -362,7 +364,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -371,16 +373,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_public_user_data( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_public_user_data(events=events) self.assertEquals(events, results) @defer.inlineCallbacks def test_filter_public_user_data_no_match(self): - user_filter = { + user_filter_json = { "public_user_data": { "types": ["m.*"] } @@ -388,7 +391,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -397,16 +400,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_public_user_data( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_public_user_data(events=events) self.assertEquals([], results) @defer.inlineCallbacks def test_filter_room_state_match(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -416,7 +420,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -425,16 +429,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_room_state( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_room_state(events=events) self.assertEquals(events, results) @defer.inlineCallbacks def test_filter_room_state_no_match(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -444,7 +449,7 @@ class FilteringTestCase(unittest.TestCase): user = UserID.from_string("@" + user_localpart + ":test") filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) event = MockEvent( sender="@foo:bar", @@ -453,16 +458,17 @@ class FilteringTestCase(unittest.TestCase): ) events = [event] - results = yield self.filtering.filter_room_state( - events=events, - user=user, - filter_id=filter_id + user_filter = yield self.filtering.get_user_filter( + user_localpart=user_localpart, + filter_id=filter_id, ) + + results = user_filter.filter_room_state(events) self.assertEquals([], results) @defer.inlineCallbacks def test_add_filter(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -472,11 +478,11 @@ class FilteringTestCase(unittest.TestCase): filter_id = yield self.filtering.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) self.assertEquals(filter_id, 0) - self.assertEquals(user_filter, + self.assertEquals(user_filter_json, (yield self.datastore.get_user_filter( user_localpart=user_localpart, filter_id=0, @@ -485,7 +491,7 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_filter(self): - user_filter = { + user_filter_json = { "room": { "state": { "types": ["m.*"] @@ -495,7 +501,7 @@ class FilteringTestCase(unittest.TestCase): filter_id = yield self.datastore.add_user_filter( user_localpart=user_localpart, - user_filter=user_filter, + user_filter=user_filter_json, ) filter = yield self.filtering.get_user_filter( @@ -503,4 +509,4 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - self.assertEquals(filter, user_filter) + self.assertEquals(filter.filter_json, user_filter_json) From 365a1867293b8f1d34a1f100d1bc8d2d39bb3d94 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 18:11:28 +0000 Subject: [PATCH 165/630] Add basic filtering support --- synapse/rest/client/v2_alpha/sync.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 2ae2eec55..c1277d267 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -21,6 +21,7 @@ from synapse.types import StreamToken from synapse.events.utils import ( serialize_event, format_event_for_client_v2_without_event_id, ) +from synapse.api.filtering import Filter from ._base import client_v2_pattern import logging @@ -80,6 +81,7 @@ class SyncRestServlet(RestServlet): self.auth = hs.get_auth() self.sync_handler = hs.get_handlers().sync_handler self.clock = hs.get_clock() + self.filtering = hs.get_filtering() @defer.inlineCallbacks def on_GET(self, request): @@ -109,9 +111,14 @@ class SyncRestServlet(RestServlet): ) # TODO(mjark): Load filter and apply overrides. - # filter = self.filters.load_fitler(filter_id_str) + try: + filter = yield self.filtering.get_user_filter( + user.localpart, filter_id + ) + except: + filter = Filter({}) # filter = filter.apply_overrides(http_request) - # if filter.matches(event): + #if filter.matches(event): # # stuff sync_config = SyncConfig( @@ -121,7 +128,7 @@ class SyncRestServlet(RestServlet): limit=limit, sort=sort, backfill=backfill, - filter="TODO", # TODO(mjark) Add the filter to the config. + filter=filter, ) if since is not None: @@ -162,9 +169,11 @@ class SyncRestServlet(RestServlet): @staticmethod def encode_room(room, filter, time_now, token_id): event_map = {} + state_events = filter.filter_room_state(room.state) + recent_events = filter.filter_room_events(room.events) state_event_ids = [] recent_event_ids = [] - for event in room.state: + for event in state_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( event, time_now, token_id=token_id, @@ -172,7 +181,7 @@ class SyncRestServlet(RestServlet): ) state_event_ids.append(event.event_id) - for event in room.events: + for event in recent_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( event, time_now, token_id=token_id, From ece828a7b72a25502d8daa15f6429c73a620b53c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 29 Jan 2015 18:15:24 +0000 Subject: [PATCH 166/630] Update todo for the filtering on sync --- synapse/handlers/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 8c7681a48..576870219 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -290,7 +290,8 @@ class SyncHandler(BaseHandler): """ # TODO(mjark): Check if they have joined the room between # the previous sync and this one. - # TODO(mjark): Apply the event filter in sync_config + # TODO(mjark): Apply the event filter in sync_config taking care to get + # enough events to reach the limit # TODO(mjark): Check for redactions we might have missed. recents, token = yield self.store.get_recent_events_for_room( room_id, From 1235f7f383617a91b4b96e287cc1ad205b80a5de Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 18:38:22 +0000 Subject: [PATCH 167/630] Add default push rules including setting a sound for messages mentioning your username / display name --- synapse/push/__init__.py | 47 ++++++++++++++++++++++++++++++++------- synapse/push/baserules.py | 35 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 synapse/push/baserules.py diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 472ede548..d19e13d64 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -16,9 +16,10 @@ from twisted.internet import defer from synapse.streams.config import PaginationConfig -from synapse.types import StreamToken +from synapse.types import StreamToken, UserID import synapse.util.async +import baserules import logging import fnmatch @@ -75,14 +76,34 @@ class Pusher(object): rules = yield self.store.get_push_rules_for_user_name(self.user_name) + for r in rules: + r['conditions'] = json.loads(r['conditions']) + r['actions'] = json.loads(r['actions']) + + user_name_localpart = UserID.from_string(self.user_name).localpart + + rules.extend(baserules.make_base_rules(user_name_localpart)) + + # get *our* member event for display name matching + member_events_for_room = yield self.store.get_current_state( + room_id=ev['room_id'], + event_type='m.room.member', + state_key=self.user_name + ) + my_display_name = None + if len(member_events_for_room) > 0: + my_display_name = member_events_for_room[0].content['displayname'] + for r in rules: matches = True - conditions = json.loads(r['conditions']) - actions = json.loads(r['actions']) + conditions = r['conditions'] + actions = r['actions'] for c in conditions: - matches &= self._event_fulfills_condition(ev, c) + matches &= self._event_fulfills_condition( + ev, c, display_name=my_display_name + ) # ignore rules with no actions (we have an explict 'dont_notify' if len(actions) == 0: logger.warn( @@ -95,7 +116,7 @@ class Pusher(object): defer.returnValue(Pusher.DEFAULT_ACTIONS) - def _event_fulfills_condition(self, ev, condition): + def _event_fulfills_condition(self, ev, condition, display_name): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") @@ -103,13 +124,23 @@ class Pusher(object): pat = condition['pattern'] val = _value_for_dotted_key(condition['key'], ev) - if fnmatch.fnmatch(val, pat): - return True - return False + if val is None: + return False + return fnmatch.fnmatch(val.upper(), pat.upper()) elif condition['kind'] == 'device': if 'instance_handle' not in condition: return True return condition['instance_handle'] == self.instance_handle + elif condition['kind'] == 'contains_display_name': + # This is special because display names can be different + # between rooms and so you can't really hard code it in a rule. + # Optimisation: we should cache these names and update them from + # the event stream. + if 'content' not in ev or 'body' not in ev['content']: + return False + return fnmatch.fnmatch( + ev['content']['body'].upper(), "*%s*" % (display_name.upper(),) + ) else: return True diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py new file mode 100644 index 000000000..4caf7beed --- /dev/null +++ b/synapse/push/baserules.py @@ -0,0 +1,35 @@ +def make_base_rules(user_name): + """ + Nominally we reserve priority class 0 for these rules, although + in practice we just append them to the end so we don't actually need it. + """ + return [ + { + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.body', + 'pattern': '*%s*' % (user_name,), # Matrix ID match + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + }, + { + 'conditions': [ + { + 'kind': 'contains_display_name' + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + }, + ] \ No newline at end of file From 0b1688639750e2401571263c127817d0b0a43644 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 18:51:22 +0000 Subject: [PATCH 168/630] Change 'from' in notification pokes to 'sender' to match client API v2. Send sender display names where they exist. --- synapse/push/__init__.py | 10 ++++++++++ synapse/push/httppusher.py | 9 ++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index d19e13d64..19478c72a 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -154,6 +154,16 @@ class Pusher(object): if name_aliases[0] is not None: ctx['name'] = name_aliases[0] + their_member_events_for_room = yield self.store.get_current_state( + room_id=ev['room_id'], + event_type='m.room.member', + state_key=ev['user_id'] + ) + if len(their_member_events_for_room) > 0: + dn = their_member_events_for_room[0].content['displayname'] + if dn is not None: + ctx['sender_display_name'] = dn + defer.returnValue(ctx) @defer.inlineCallbacks diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index ab128e31e..ac7c3148d 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -67,10 +67,7 @@ class HttpPusher(Pusher): 'notification': { 'id': event['event_id'], 'type': event['type'], - 'from': event['user_id'], - # we may have to fetch this over federation and we - # can't trust it anyway: is it worth it? - #'from_display_name': 'Steve Stevington' + 'sender': event['user_id'], 'counts': { # -- we don't mark messages as read yet so # we have no way of knowing # Just set the badge to 1 until we have read receipts @@ -93,6 +90,8 @@ class HttpPusher(Pusher): if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] + if 'sender_display_name' in ctx: + d['notification']['sender_display_name'] = ctx['sender_display_name'] if 'name' in ctx: d['notification']['room_name'] = ctx['name'] @@ -119,7 +118,7 @@ class HttpPusher(Pusher): 'notification': { 'id': '', 'type': None, - 'from': '', + 'sender': '', 'counts': { 'unread': 0, 'missed_calls': 0 From fc946f3b8da8c7f71a9c25bf542c04472147bc5b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Jan 2015 21:59:17 +0000 Subject: [PATCH 169/630] Include content in notification pokes --- synapse/push/httppusher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index ac7c3148d..d4c5f03b0 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -87,6 +87,8 @@ class HttpPusher(Pusher): } if event['type'] == 'm.room.member': d['notification']['membership'] = event['content']['membership'] + if 'content' in event: + d['notification']['content'] = event['content'] if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] From c1c7b398270cc19fcf9410b9159f741092e0510b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 10:30:54 +0000 Subject: [PATCH 170/630] Fix bug where we changes in outlier in metadata dict propogated to other events --- synapse/events/__init__.py | 2 +- synapse/events/builder.py | 5 +++-- synapse/events/utils.py | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 4252e5ab5..bf0795102 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -18,7 +18,7 @@ from synapse.util.frozenutils import freeze, unfreeze class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): - self.__dict__ = internal_metadata_dict + self.__dict__ = dict(internal_metadata_dict) def get_dict(self): return dict(self.__dict__) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index a9b1b99a1..9d45bdb89 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -23,14 +23,15 @@ import copy class EventBuilder(EventBase): - def __init__(self, key_values={}): + def __init__(self, key_values={}, internal_metadata_dict={}): signatures = copy.deepcopy(key_values.pop("signatures", {})) unsigned = copy.deepcopy(key_values.pop("unsigned", {})) super(EventBuilder, self).__init__( key_values, signatures=signatures, - unsigned=unsigned + unsigned=unsigned, + internal_metadata_dict=internal_metadata_dict, ) def build(self): diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 10a6b9f26..08d6d6fa4 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -88,7 +88,10 @@ def prune_event(event): if "age_ts" in event.unsigned: allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] - return type(event)(allowed_fields) + return type(event)( + allowed_fields, + internal_metadata_dict=event.internal_metadata.get_dict() + ) def serialize_event(hs, e, client_event=True): From c1d860870b418b9583078022d0babe557b73760f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 10:48:47 +0000 Subject: [PATCH 171/630] Fix regression where we no longer correctly handled the case of gaps in our event graph --- synapse/federation/federation_server.py | 3 +++ synapse/handlers/federation.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 8cff4e647..845a07a3a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -366,10 +366,13 @@ class FederationServer(object): logger.debug("Processed pdu %s", event_id) else: logger.warn("Failed to get PDU %s", event_id) + fetch_state = True except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") fetch_state = True + else: + fetch_state = True else: fetch_state = True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 14c26d8ce..de47a97e6 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -119,7 +119,7 @@ class FederationHandler(BaseHandler): event.room_id, self.server_name ) - if not is_in_room and not event.internal_metadata.outlier: + if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") replication = self.replication_layer @@ -780,6 +780,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in remote_auth_chain if e.event_id in auth_ids } + e.internal_metadata.outlier = True yield self._handle_new_event( origin, e, auth_events=auth ) @@ -787,6 +788,8 @@ class FederationHandler(BaseHandler): except AuthError: pass + # FIXME: Assumes we have and stored all the state for all the + # prev_events current_state = set(e.event_id for e in auth_events.values()) different_auth = event_auth_events - current_state @@ -814,6 +817,7 @@ class FederationHandler(BaseHandler): (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } + e.internal_metadata.outlier = True yield self._handle_new_event( origin, e, auth_events=auth ) @@ -882,7 +886,7 @@ class FederationHandler(BaseHandler): missing_remotes = [] missing_locals = [] - while current_local and current_remote: + while current_local or current_remote: if current_remote is None: missing_locals.append(current_local) current_local = get_next(local_iter) From 823999716eb506740b34d87fcdaaf3ebbe4f760c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:07:57 +0000 Subject: [PATCH 172/630] Fix bug in timeout handling in keyclient --- synapse/crypto/keyclient.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 9c910fa3f..cdb627976 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -61,9 +61,11 @@ class SynapseKeyClientProtocol(HTTPClient): def __init__(self): self.remote_key = defer.Deferred() + self.host = None def connectionMade(self): - logger.debug("Connected to %s", self.transport.getHost()) + self.host = self.transport.getHost() + logger.debug("Connected to %s", self.host) self.sendCommand(b"GET", b"/_matrix/key/v1/") self.endHeaders() self.timer = reactor.callLater( @@ -92,8 +94,7 @@ class SynapseKeyClientProtocol(HTTPClient): self.timer.cancel() def on_timeout(self): - logger.debug("Timeout waiting for response from %s", - self.transport.getHost()) + logger.debug("Timeout waiting for response from %s", self.host) self.remote_key.errback(IOError("Timeout waiting for response")) self.transport.abortConnection() From 0c2d245fdf5f24f99138f1c60c4e6bed52cf5d5a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:08:52 +0000 Subject: [PATCH 173/630] Update the current state of an event if we update auth events. --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index de47a97e6..cc22f21cd 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -706,7 +706,6 @@ class FederationHandler(BaseHandler): event.event_id, e.msg ) - # TODO: Store rejection. context.rejected = RejectedReason.AUTH_ERROR yield self.store.persist_event( @@ -828,6 +827,9 @@ class FederationHandler(BaseHandler): # 4. Look at rejects and their proofs. # TODO. + context.current_state.update(auth_events) + context.state_group = None + try: self.auth.check(event, auth_events=auth_events) except AuthError: From bd03947c05f23e4adb6ce1a745d725a106fed66c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 11:13:42 +0000 Subject: [PATCH 174/630] We do need Twisted 14, not 15: we use internal Twisted things that have been removed in 15. --- setup.py | 2 +- synapse/python_dependencies.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 043cd044a..3249e87a9 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( install_requires=[ "syutil==0.0.2", "matrix_angular_sdk==0.6.0", - "Twisted>=14.0.0", + "Twisted==14.0.2", "service_identity>=1.0.0", "pyopenssl>=0.14", "pyyaml", diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 4182ad990..826a36f20 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted>=14.0.0": ["twisted>=14.0.0"], + "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], From 2c9e136d5721d1797e153268f4562c27887a6201 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:14:33 +0000 Subject: [PATCH 175/630] Fix bad merge fo python_dependencies.py --- synapse/python_dependencies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ba9308803..4182ad990 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,6 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "matrix_angular_sdk>=0.6.0": ["syweb>=0.6.0"], "Twisted>=14.0.0": ["twisted>=14.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], From 0adf3e54454d499045d2bfd8f6bcfece4e8fe6ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 11:16:41 +0000 Subject: [PATCH 176/630] Revert accidental bumping of angluar_sdk dep --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 733cfa831..043cd044a 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.6.1", + "matrix_angular_sdk==0.6.0", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", @@ -47,7 +47,7 @@ setup( dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.1", + "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0", ], setup_requires=[ "setuptools_trial", From 22dd1cde2d83a2448074816108b85d1957315236 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:32:35 +0000 Subject: [PATCH 177/630] Filter the recent events before applying the limit when doing an incremental sync with a gap --- synapse/api/filtering.py | 2 -- synapse/handlers/sync.py | 53 ++++++++++++++++++++-------- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/storage/stream.py | 21 +++++++---- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index b7e5d3222..fa4de2614 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 576870219..0df1851b0 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -278,6 +278,40 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) + @defer.inlineCallbacks + def load_filtered_recents(self, room_id, sync_config, since_token, + now_token): + limited = True + recents = [] + filtering_factor = 2 + load_limit = max(sync_config.limit * filtering_factor, 100) + max_repeat = 3 # Only try a few times per room, otherwise + room_key = now_token.room_key + + while limited and len(recents) < sync_config.limit and max_repeat: + events, room_key = yield self.store.get_recent_events_for_room( + room_id, + limit=load_limit + 1, + from_token=since_token.room_key, + end_token=room_key, + ) + loaded_recents = sync_config.filter.filter_room_events(events) + loaded_recents.extend(recents) + recents = loaded_recents + if len(events) <= load_limit: + limited = False + max_repeat -= 1 + + if len(recents) > sync_config.limit: + recents = recents[-sync_config.limit:] + room_key = recents[0].internal_metadata.before + + prev_batch_token = now_token.copy_and_replace( + "room_key", room_key + ) + + defer.returnValue((recents, prev_batch_token, limited)) + @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, @@ -288,28 +322,17 @@ class SyncHandler(BaseHandler): Returns: A Deferred RoomSyncResult """ + # TODO(mjark): Check if they have joined the room between # the previous sync and this one. - # TODO(mjark): Apply the event filter in sync_config taking care to get - # enough events to reach the limit # TODO(mjark): Check for redactions we might have missed. - recents, token = yield self.store.get_recent_events_for_room( - room_id, - limit=sync_config.limit + 1, - from_token=since_token.room_key, - end_token=now_token.room_key, + + recents, prev_batch_token, limited = self.load_filtered_recents( + room_id, sync_config, since_token, ) logging.debug("Recents %r", recents) - if len(recents) > sync_config.limit: - limited = True - recents = recents[1:] - else: - limited = False - - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) - # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is current_state_events = yield self.state_handler.get_current_state( diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index c1277d267..46ea50d11 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -116,7 +116,7 @@ class SyncRestServlet(RestServlet): user.localpart, filter_id ) except: - filter = Filter({}) + filter = Filter({}) # filter = filter.apply_overrides(http_request) #if filter.matches(event): # # stuff diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 2ea5e1a02..73504c8b5 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -181,15 +181,11 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) - for event, row in zip(ret, rows): - stream = row["stream_ordering"] - topo = event.depth - internal = event.internal_metadata - internal.before = str(_StreamToken(topo, stream - 1)) - internal.after = str(_StreamToken(topo, stream)) + self._set_before_and_after(ret, rows) if rows: key = "s%d" % max([r["stream_ordering"] for r in rows]) + else: # Assume we didn't get anything because there was nothing to # get. @@ -267,6 +263,8 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(events, rows) + return events, next_token, return self.runInteraction("paginate_room_events", f) @@ -328,6 +326,8 @@ class StreamStore(SQLBaseStore): get_prev_content=True ) + self._set_before_and_after(events, rows) + return events, token return self.runInteraction( @@ -354,3 +354,12 @@ class StreamStore(SQLBaseStore): key = res[0]["m"] return "s%d" % (key,) + + @staticmethod + def _set_before_and_after(events, rows): + for event, row in zip(events, rows): + stream = row["stream_ordering"] + topo = event.depth + internal = event.internal_metadata + internal.before = str(_StreamToken(topo, stream - 1)) + internal.after = str(_StreamToken(topo, stream)) From e97de6d96a25912b34fa38cd3a587d798f6b676c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:35:20 +0000 Subject: [PATCH 178/630] Filter the recent events before applying the limit when doing an initial sync --- synapse/handlers/sync.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 0df1851b0..b83fcad65 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -163,12 +163,11 @@ class SyncHandler(BaseHandler): Returns: A Deferred RoomSyncResult. """ - recent_events, token = yield self.store.get_recent_events_for_room( - room_id, - limit=sync_config.limit, - end_token=now_token.room_key, + + recents, prev_batch_token, limited = self.load_filtered_recents( + room_id, sync_config, now_token, ) - prev_batch_token = now_token.copy_and_replace("room_key", token[0]) + current_state_events = yield self.state_handler.get_current_state( room_id ) @@ -176,10 +175,10 @@ class SyncHandler(BaseHandler): defer.returnValue(RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, - events=recent_events, + events=recents, prev_batch=prev_batch_token, state=current_state_events, - limited=True, + limited=limited, ephemeral=[], )) @@ -279,8 +278,8 @@ class SyncHandler(BaseHandler): )) @defer.inlineCallbacks - def load_filtered_recents(self, room_id, sync_config, since_token, - now_token): + def load_filtered_recents(self, room_id, sync_config, now_token, + since_token=None): limited = True recents = [] filtering_factor = 2 @@ -292,7 +291,7 @@ class SyncHandler(BaseHandler): events, room_key = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, - from_token=since_token.room_key, + from_token=since_token.room_key if since_token else None, end_token=room_key, ) loaded_recents = sync_config.filter.filter_room_events(events) @@ -328,7 +327,7 @@ class SyncHandler(BaseHandler): # TODO(mjark): Check for redactions we might have missed. recents, prev_batch_token, limited = self.load_filtered_recents( - room_id, sync_config, since_token, + room_id, sync_config, now_token, since_token, ) logging.debug("Recents %r", recents) From 8498d348d818aa2d2cb9bb9bb2775103840f355d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:42:09 +0000 Subject: [PATCH 179/630] Fix token formatting --- synapse/handlers/sync.py | 6 +++--- synapse/storage/stream.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b83fcad65..3c68e2a9e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -164,7 +164,7 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult. """ - recents, prev_batch_token, limited = self.load_filtered_recents( + recents, prev_batch_token, limited = yield self.load_filtered_recents( room_id, sync_config, now_token, ) @@ -288,7 +288,7 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key while limited and len(recents) < sync_config.limit and max_repeat: - events, room_key = yield self.store.get_recent_events_for_room( + events, (room_key,_) = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, @@ -326,7 +326,7 @@ class SyncHandler(BaseHandler): # the previous sync and this one. # TODO(mjark): Check for redactions we might have missed. - recents, prev_batch_token, limited = self.load_filtered_recents( + recents, prev_batch_token, limited = yield self.load_filtered_recents( room_id, sync_config, now_token, since_token, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 73504c8b5..3ccb6f8a6 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -316,9 +316,9 @@ class StreamStore(SQLBaseStore): toke = rows[0]["stream_ordering"] - 1 start_token = str(_StreamToken(topo, toke)) - token = (start_token, end_token) + token = (start_token, str(end_token)) else: - token = (end_token, end_token) + token = (str(end_token), str(end_token)) events = self._get_events_txn( txn, From c562f237f6236c981f2e7858ff2748f62bd63ad1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:43:00 +0000 Subject: [PATCH 180/630] Unused import --- synapse/api/filtering.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index b7e5d3222..fa4de2614 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID From 4a67834bc84f604605c618049599f4638434c7cf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 11:50:15 +0000 Subject: [PATCH 181/630] Pass client info to the sync_config --- synapse/handlers/sync.py | 5 +++-- synapse/rest/client/v2_alpha/sync.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 3c68e2a9e..1a74a4c97 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "device", + "client_info", "limit", "gap", "sort", @@ -288,12 +288,13 @@ class SyncHandler(BaseHandler): room_key = now_token.room_key while limited and len(recents) < sync_config.limit and max_repeat: - events, (room_key,_) = yield self.store.get_recent_events_for_room( + events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, end_token=room_key, ) + (room_key, _) = keys loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 46ea50d11..81d5cf8ea 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -123,7 +123,7 @@ class SyncRestServlet(RestServlet): sync_config = SyncConfig( user=user, - device="TODO", # TODO(mjark) Get the device_id from access_token + client_info=client, gap=gap, limit=limit, sort=sort, From a70a801184814d116ed5b10a952e17c45df7bfc8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:34:01 +0000 Subject: [PATCH 182/630] Fix bug where we superfluously asked for current state. Change API of /query_auth/ so that we don't duplicate events in the response. --- synapse/api/auth.py | 2 + synapse/federation/federation_client.py | 7 +--- synapse/federation/federation_server.py | 12 +++--- synapse/handlers/federation.py | 51 +++++++++---------------- synapse/state.py | 20 ++++++++-- 5 files changed, 43 insertions(+), 49 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3471afd7e..37e31d2b6 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -102,6 +102,8 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) + logger.debug("Got curr_state %s", curr_state) + for event in curr_state: if event.type == EventTypes.Member: try: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 1173ca817..e1539bd0e 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -357,15 +357,10 @@ class FederationClient(object): for e in content["auth_chain"] ] - missing = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) - for e in content.get("missing", []) - ] - ret = { "auth_chain": auth_chain, "rejects": content.get("rejects", []), - "missing": missing, + "missing": content.get("missing", []), } defer.returnValue(ret) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 845a07a3a..84ed0a0ba 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -252,11 +252,8 @@ class FederationServer(object): e.get_pdu_json(time_now) for e in ret["auth_chain"] ], - "rejects": content.get("rejects", []), - "missing": [ - e.get_pdu_json(time_now) - for e in ret.get("missing", []) - ], + "rejects": ret.get("rejects", []), + "missing": ret.get("missing", []), } defer.returnValue( @@ -372,7 +369,10 @@ class FederationServer(object): logger.exception("Failed to get PDU") fetch_state = True else: - fetch_state = True + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if prevs - seen: + fetch_state = True else: fetch_state = True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index cc22f21cd..35cad4182 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -121,38 +121,18 @@ class FederationHandler(BaseHandler): ) if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") - - replication = self.replication_layer - - if not state: - state, auth_chain = yield replication.get_state_for_room( - origin, room_id=event.room_id, event_id=event.event_id, - ) - - if not auth_chain: - auth_chain = yield replication.get_event_auth( - origin, - context=event.room_id, - event_id=event.event_id, - ) - - for e in auth_chain: - e.internal_metadata.outlier = True - try: - yield self._handle_new_event(origin, e) - except: - logger.exception( - "Failed to handle auth event %s", - e.event_id, - ) - current_state = state - if state: + if state and auth_chain is not None: for e in state: e.internal_metadata.outlier = True try: - yield self._handle_new_event(origin, e) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event(origin, e, auth_events=auth) except: logger.exception( "Failed to handle state event %s", @@ -809,18 +789,23 @@ class FederationHandler(BaseHandler): ) # 3. Process any remote auth chain events we haven't seen. - for e in result.get("missing", []): + for missing_id in result.get("missing", []): try: - auth_ids = [e_id for e_id, _ in e.auth_events] + for e in result["auth_chain"]: + if e.event_id == missing_id: + ev = e + break + + auth_ids = [e_id for e_id, _ in ev.auth_events] auth = { (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } - e.internal_metadata.outlier = True + ev.internal_metadata.outlier = True yield self._handle_new_event( - origin, e, auth_events=auth + origin, ev, auth_events=auth ) - auth_events[(e.type, e.state_key)] = e + auth_events[(ev.type, ev.state_key)] = ev except AuthError: pass @@ -970,5 +955,5 @@ class FederationHandler(BaseHandler): } for e in base_remote_rejected }, - "missing": missing_locals, + "missing": [e.event_id for e in missing_locals], }) diff --git a/synapse/state.py b/synapse/state.py index d9fdfb34b..e6632978b 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -166,10 +166,17 @@ class StateHandler(object): first is the name of a state group if one and only one is involved, otherwise `None`. """ + logger.debug("resolve_state_groups event_ids %s", event_ids) + state_groups = yield self.store.get_state_groups( event_ids ) + logger.debug( + "resolve_state_groups state_groups %s", + state_groups.keys() + ) + group_names = set(state_groups.keys()) if len(group_names) == 1: name, state_list = state_groups.items().pop() @@ -205,6 +212,15 @@ class StateHandler(object): if len(v.values()) > 1 } + logger.debug( + "resolve_state_groups Unconflicted state: %s", + unconflicted_state.values(), + ) + logger.debug( + "resolve_state_groups Conflicted state: %s", + conflicted_state.values(), + ) + if event_type: prev_states_events = conflicted_state.get( (event_type, state_key), [] @@ -240,10 +256,6 @@ class StateHandler(object): 1. power levels 2. memberships 3. other events. - - :param conflicted_state: - :param auth_events: - :return: """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") From 8fe39a03119b3d092e84319d6d99b6ffc49a83f5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 13:33:41 +0000 Subject: [PATCH 183/630] Check if the user has joined the room between incremental syncs --- synapse/handlers/sync.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1a74a4c97..9e07f30b2 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -16,7 +16,7 @@ from ._base import BaseHandler from synapse.streams.config import PaginationConfig -from synapse.api.constants import Membership +from synapse.api.constants import Membership, EventTypes from twisted.internet import defer @@ -250,6 +250,11 @@ class SyncHandler(BaseHandler): ) else: prev_batch = now_token + + state = yield self.check_joined_room( + sync_config, room_id, state + ) + room_sync = RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, @@ -323,8 +328,6 @@ class SyncHandler(BaseHandler): A Deferred RoomSyncResult """ - # TODO(mjark): Check if they have joined the room between - # the previous sync and this one. # TODO(mjark): Check for redactions we might have missed. recents, prev_batch_token, limited = yield self.load_filtered_recents( @@ -349,6 +352,10 @@ class SyncHandler(BaseHandler): current_state=current_state_events, ) + state_events_delta = yield self.check_joined_room( + sync_config, room_id, state_events_delta + ) + room_sync = RoomSyncResult( room_id=room_id, published=room_id in published_room_ids, @@ -356,7 +363,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, - typing=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, None) ) logging.debug("Room sync: %r", room_sync) @@ -402,3 +409,19 @@ class SyncHandler(BaseHandler): if event.event_id not in previous_dict: state_delta.append(event) return state_delta + + @defer.inlineCallbacks + def check_joined_room(self, sync_config, room_id, state_delta): + joined = False + for event in state_delta: + if ( + event.type == EventTypes.Member + and event.state_key == sync_config.user.to_string() + ): + if event.content["membership"] == Membership.JOIN: + joined = True + + if joined: + state_delta = yield self.state_handler.get_current_state(room_id) + + defer.returnValue(state_delta) From 76d7fd39cd44393a5c712930e77c64e202df17cc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:52:02 +0000 Subject: [PATCH 184/630] Style changes. --- synapse/state.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index d9fdfb34b..43bda3525 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -37,6 +37,9 @@ def _get_state_key_from_event(event): KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) +AuthEventTypes = (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + + class StateHandler(object): """ Responsible for doing state conflict resolution. """ @@ -215,7 +218,7 @@ class StateHandler(object): auth_events = { k: e for k, e in unconflicted_state.items() - if k[0] in (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) + if k[0] in AuthEventTypes } try: @@ -240,10 +243,6 @@ class StateHandler(object): 1. power levels 2. memberships 3. other events. - - :param conflicted_state: - :param auth_events: - :return: """ resolved_state = {} power_key = (EventTypes.PowerLevels, "") @@ -305,4 +304,4 @@ class StateHandler(object): def key_func(e): return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() - return sorted(events, key=key_func) \ No newline at end of file + return sorted(events, key=key_func) From 7a9f6f083e0998893eb9d837512009509d81c998 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 13:55:46 +0000 Subject: [PATCH 185/630] Remove commented line --- synapse/state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index 43bda3525..dff11711a 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -46,7 +46,6 @@ class StateHandler(object): def __init__(self, hs): self.store = hs.get_datastore() - # self.auth = hs.get_auth() self.hs = hs @defer.inlineCallbacks From 3d7026e709b145378a6b8a37404910951ef682b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:37:19 +0000 Subject: [PATCH 186/630] Add a slightly more helpful comment --- synapse/state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index dff11711a..081bc31bb 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -296,7 +296,8 @@ class StateHandler(object): except AuthError: pass - # Oh dear. + # Use the last event (the one with the least depth) if they all fail + # the auth check. return event def _ordered_events(self, events): From 322a047502c938bfe9a6acab47e370e69fefc522 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 14:46:03 +0000 Subject: [PATCH 187/630] Add room member count condition and default rule to make a noise on rooms of only 2 people. --- synapse/push/__init__.py | 50 +++++++++++++++++++++++++++++++++---- synapse/push/baserules.py | 14 +++++++++++ synapse/storage/__init__.py | 5 +++- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 19478c72a..cc05278c8 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -24,6 +24,7 @@ import baserules import logging import fnmatch import json +import re logger = logging.getLogger(__name__) @@ -34,6 +35,8 @@ class Pusher(object): GIVE_UP_AFTER = 24 * 60 * 60 * 1000 DEFAULT_ACTIONS = ['notify'] + INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") + def __init__(self, _hs, instance_handle, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): @@ -88,11 +91,21 @@ class Pusher(object): member_events_for_room = yield self.store.get_current_state( room_id=ev['room_id'], event_type='m.room.member', - state_key=self.user_name + state_key=None ) my_display_name = None - if len(member_events_for_room) > 0: - my_display_name = member_events_for_room[0].content['displayname'] + room_member_count = 0 + for mev in member_events_for_room: + if mev.content['membership'] != 'join': + continue + + # This loop does two things: + # 1) Find our current display name + if mev.state_key == self.user_name: + my_display_name = mev.content['displayname'] + + # and 2) Get the number of people in that room + room_member_count += 1 for r in rules: matches = True @@ -102,7 +115,8 @@ class Pusher(object): for c in conditions: matches &= self._event_fulfills_condition( - ev, c, display_name=my_display_name + ev, c, display_name=my_display_name, + room_member_count=room_member_count ) # ignore rules with no actions (we have an explict 'dont_notify' if len(actions) == 0: @@ -116,7 +130,7 @@ class Pusher(object): defer.returnValue(Pusher.DEFAULT_ACTIONS) - def _event_fulfills_condition(self, ev, condition, display_name): + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") @@ -138,9 +152,35 @@ class Pusher(object): # the event stream. if 'content' not in ev or 'body' not in ev['content']: return False + if not display_name: + return False return fnmatch.fnmatch( ev['content']['body'].upper(), "*%s*" % (display_name.upper(),) ) + elif condition['kind'] == 'room_member_count': + if 'is' not in condition: + return False + m = Pusher.INEQUALITY_EXPR.match(condition['is']) + if not m: + return False + ineq = m.group(1) + rhs = m.group(2) + if not rhs.isdigit(): + return False + rhs = int(rhs) + + if ineq == '' or ineq == '==': + return room_member_count == rhs + elif ineq == '<': + return room_member_count < rhs + elif ineq == '>': + return room_member_count > rhs + elif ineq == '>=': + return room_member_count >= rhs + elif ineq == '<=': + return room_member_count <= rhs + else: + return False else: return True diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 4caf7beed..bd162baad 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -32,4 +32,18 @@ def make_base_rules(user_name): } ] }, + { + 'conditions': [ + { + 'kind': 'room_member_count', + 'is': '2' + } + ], + 'actions': [ + 'notify', + { + 'set_sound': 'default' + } + ] + } ] \ No newline at end of file diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 277581b4e..7b18acf42 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -375,9 +375,12 @@ class DataStore(RoomMemberStore, RoomStore, "redacted": del_sql, } - if event_type: + if event_type and state_key is not None: sql += " AND s.type = ? AND s.state_key = ? " args = (room_id, event_type, state_key) + elif event_type: + sql += " AND s.type = ?" + args = (room_id, event_type) else: args = (room_id, ) From 472cf532b7e64a30c5d7ecdeec9b8f89abf8276c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:48:03 +0000 Subject: [PATCH 188/630] Put CREATE rejections into seperate .sql --- synapse/storage/schema/im.sql | 7 ------- synapse/storage/schema/rejections.sql | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 synapse/storage/schema/rejections.sql diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index bc7c6b6ed..dd00c1cd2 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -123,10 +123,3 @@ CREATE TABLE IF NOT EXISTS room_hosts( ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); - -CREATE TABLE IF NOT EXISTS rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE -); diff --git a/synapse/storage/schema/rejections.sql b/synapse/storage/schema/rejections.sql new file mode 100644 index 000000000..bd2a8b1bb --- /dev/null +++ b/synapse/storage/schema/rejections.sql @@ -0,0 +1,21 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE +); From 2f4cb04f455d24d0086b37bc363137e995d908d5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:48:11 +0000 Subject: [PATCH 189/630] Be more specific in naming columns in selects. --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1f5e74a16..b350fd61f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -504,7 +504,7 @@ class SQLBaseStore(object): def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): sql = ( - "SELECT internal_metadata, json, r.event_id, reason " + "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " "FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " "LEFT JOIN rejections as rej on rej.event_id = e.event_id " From e97f756a05519f9d5a8a6ff78182b691dd1355df Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 14:54:06 +0000 Subject: [PATCH 190/630] Use 'in' to test if the key exists, remove unused _filters_for_user --- synapse/api/filtering.py | 8 ++------ synapse/storage/filtering.py | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index fa4de2614..4d570b74f 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -114,21 +114,17 @@ class Filtering(object): if not isinstance(event_type, basestring): raise SynapseError(400, "Event type should be a string") - try: + if "format" in definition: event_format = definition["format"] if event_format not in ["federation", "events"]: raise SynapseError(400, "Invalid format: %s" % (event_format,)) - except KeyError: - pass # format is optional - try: + if "select" in definition: event_select_list = definition["select"] for select_key in event_select_list: if select_key not in ["event_id", "origin_server_ts", "thread_id", "content", "content.body"]: raise SynapseError(400, "Bad select: %s" % (select_key,)) - except KeyError: - pass # select is optional if ("bundle_updates" in definition and type(definition["bundle_updates"]) != bool): diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index cb01c2040..e86eeced4 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -20,10 +20,6 @@ from ._base import SQLBaseStore import json -# TODO(paul) -_filters_for_user = {} - - class FilteringStore(SQLBaseStore): @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): From 4f7fe63b6df891c196698b9d896ca7893c7a8a8e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:57:53 +0000 Subject: [PATCH 191/630] Remember to add schema file to list --- synapse/storage/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f233ff2a2..d03ee8030 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -70,6 +70,7 @@ SCHEMAS = [ "pusher", "media_repository", "filtering", + "rejections", ] From 91015ad008b0d4538022fbddae7da397f7bd7000 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 14:58:54 +0000 Subject: [PATCH 192/630] Remove merge conflict --- synapse/storage/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d03ee8030..f35ece644 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -32,11 +32,8 @@ from .event_federation import EventFederationStore from .pusher import PusherStore from .push_rule import PushRuleStore from .media_repository import MediaRepositoryStore -<<<<<<< HEAD from .rejections import RejectionsStore -======= ->>>>>>> 471c47441d0c188e845b75c8f446c44899fdcfe7 from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore From 4ffac34a646fa2e763ba9214199d7803d1174959 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 15:02:21 +0000 Subject: [PATCH 193/630] Add glob asterisks when running rules. Means that now you can't do exact matches even in override rules, but I think we can live with that. Advantage is that you'll now always get back what was put in to the API. --- synapse/push/__init__.py | 5 +++++ synapse/rest/client/v1/push_rule.py | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index cc05278c8..6a302305f 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -137,6 +137,11 @@ class Pusher(object): return False pat = condition['pattern'] + if pat.strip("*?[]") == pat: + # no special glob characters so we assume the user means + # 'contains this string' rather than 'is this string' + pat = "*%s*" % (pat,) + val = _value_for_dotted_key(condition['key'], ev) if val is None: return False diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 0f78fa667..61e3bc823 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -98,10 +98,7 @@ class PushRuleRestServlet(ClientV1RestServlet): if 'pattern' not in req_obj: raise InvalidRuleException("Content rule missing 'pattern'") pat = req_obj['pattern'] - if pat.strip("*?[]") == pat: - # no special glob characters so we assume the user means - # 'contains this string' rather than 'is this string' - pat = "*%s*" % (pat,) + conditions = [{ 'kind': 'event_match', 'key': 'content.body', From 017dfaef4c0d4550acd45d27cdcea1c20766684d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 15:52:05 +0000 Subject: [PATCH 194/630] Add doc string for __nonzero__ overrides for sync results, raise not implemented if the client attempts to do a gapless sync --- synapse/handlers/sync.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9e07f30b2..dc69b3cfe 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -49,6 +49,9 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ __slots__ = [] def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ return bool(self.events or self.state or self.ephemeral) @@ -61,6 +64,10 @@ class SyncResult(collections.namedtuple("SyncResult", [ __slots__ = [] def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if the notifier needs to wait for more events when polling for + events. + """ return bool( self.private_user_data or self.public_user_data or self.rooms ) @@ -108,7 +115,7 @@ class SyncHandler(BaseHandler): return self.incremental_sync_with_gap(sync_config, since_token) else: #TODO(mjark): Handle gapless sync - pass + raise NotImplementedError() @defer.inlineCallbacks def initial_sync(self, sync_config): From b4b892f4a3f9194289f57ed2d166eca26da41594 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 15:54:29 +0000 Subject: [PATCH 195/630] Spit out server default rules too. --- synapse/push/baserules.py | 11 +++++------ synapse/rest/client/v1/push_rule.py | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index bd162baad..382de118e 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,9 +1,5 @@ def make_base_rules(user_name): - """ - Nominally we reserve priority class 0 for these rules, although - in practice we just append them to the end so we don't actually need it. - """ - return [ + rules = [ { 'conditions': [ { @@ -46,4 +42,7 @@ def make_base_rules(user_name): } ] } - ] \ No newline at end of file + ] + for r in rules: + r['priority_class'] = 0 + return rules \ No newline at end of file diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 61e3bc823..faa7919fb 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -19,6 +19,7 @@ from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, No StoreError from .base import ClientV1RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException +import synapse.push.baserules as baserules import json @@ -26,6 +27,7 @@ import json class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") PRIORITY_CLASS_MAP = { + 'default': 0, 'underride': 1, 'sender': 2, 'room': 3, @@ -137,6 +139,9 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) + if spec['template'] == 'default': + raise SynapseError(403, "The default rules are immutable.") + content = _parse_json(request) try: @@ -218,6 +223,10 @@ class PushRuleRestServlet(ClientV1RestServlet): # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + for r in rawrules: + r["conditions"] = json.loads(r["conditions"]) + r["actions"] = json.loads(r["actions"]) + rawrules.extend(baserules.make_base_rules(user.to_string())) rules = {'global': {}, 'device': {}} @@ -226,9 +235,6 @@ class PushRuleRestServlet(ClientV1RestServlet): for r in rawrules: rulearray = None - r["conditions"] = json.loads(r["conditions"]) - r["actions"] = json.loads(r["actions"]) - template_name = _priority_class_to_template_name(r['priority_class']) if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: @@ -356,7 +362,9 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): template_name = _priority_class_to_template_name(rule['priority_class']) - if template_name in ['override', 'underride']: + if template_name in ['default']: + return {k: rule[k] for k in ["conditions", "actions"]} + elif template_name in ['override', 'underride']: return {k: rule[k] for k in ["rule_id", "conditions", "actions"]} elif template_name in ["sender", "room"]: return {k: rule[k] for k in ["rule_id", "actions"]} From 7a1e881665ab72e29ff931f88ee2b65e8e690f74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:56:32 +0000 Subject: [PATCH 196/630] Remove debug logging --- synapse/state.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 038e5eba1..8a056ee95 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -214,15 +214,6 @@ class StateHandler(object): if len(v.values()) > 1 } - logger.debug( - "resolve_state_groups Unconflicted state: %s", - unconflicted_state.values(), - ) - logger.debug( - "resolve_state_groups Conflicted state: %s", - conflicted_state.values(), - ) - if event_type: prev_states_events = conflicted_state.get( (event_type, state_key), [] From b724a809c46f782231e18b70a632ce1ea9c540da Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:57:53 +0000 Subject: [PATCH 197/630] Only auth_events with event if event in event.auth_events --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 35cad4182..40836e0c5 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -805,7 +805,9 @@ class FederationHandler(BaseHandler): yield self._handle_new_event( origin, ev, auth_events=auth ) - auth_events[(ev.type, ev.state_key)] = ev + + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev except AuthError: pass From 776ac820f9bb7f0e9c2fae9facbee05b0132079e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 15:58:28 +0000 Subject: [PATCH 198/630] Briefly doc structure of query_auth API. --- synapse/federation/federation_server.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 84ed0a0ba..5fbd8b19d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -232,6 +232,24 @@ class FederationServer(object): @defer.inlineCallbacks def on_query_auth_request(self, origin, content, event_id): + """ + Content is a dict with keys:: + auth_chain (list): A list of events that give the auth chain. + missing (list): A list of event_ids indicating what the other + side (`origin`) think we're missing. + rejects (dict): A mapping from event_id to a 2-tuple of reason + string and a proof (or None) of why the event was rejected. + The keys of this dict give the list of events the `origin` has + rejected. + + Args: + origin (str) + content (dict) + event_id (str) + + Returns: + Deferred: Results in `dict` with the same format as `content` + """ auth_chain = [ (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) for e in content["auth_chain"] From 88391bcdc328e9bfdae05b99f46ccdfcbe225382 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 16:09:30 +0000 Subject: [PATCH 199/630] Allow any greater version for webclient --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3249e87a9..abccca0f2 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.6.0", + "matrix_angular_sdk>=0.6.0", "Twisted==14.0.2", "service_identity>=1.0.0", "pyopenssl>=0.14", From 2cd29dbdd91a8f58f988eb3c417ad6a187fc7e87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 16:51:58 +0000 Subject: [PATCH 200/630] Fix bug where accepting invite over federation didn't work. Add logging. --- synapse/handlers/federation.py | 59 ++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 40836e0c5..623c54d58 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -343,6 +343,10 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True + + if e.event_id == event.event_id: + continue + try: auth_ids = [e_id for e_id, _ in e.auth_events] auth = { @@ -359,7 +363,9 @@ class FederationHandler(BaseHandler): ) for e in state: - # FIXME: Auth these. + if e.event_id == event.event_id: + continue + e.internal_metadata.outlier = True try: auth_ids = [e_id for e_id, _ in e.auth_events] @@ -376,11 +382,18 @@ class FederationHandler(BaseHandler): e.event_id, ) + auth_ids = [e_id for e_id, _ in event.auth_events] + auth_events = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( target_host, new_event, state=state, current_state=state, + auth_events=auth_events, ) yield self.notifier.on_new_room_event( @@ -752,7 +765,17 @@ class FederationHandler(BaseHandler): origin, event.room_id, event.event_id ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in remote_auth_chain] + ) + for e in remote_auth_chain: + if e.event_id in seen_remotes.keys(): + continue + + if e.event_id == event.event_id: + continue + try: auth_ids = [e_id for e_id, _ in e.auth_events] auth = { @@ -760,10 +783,17 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } e.internal_metadata.outlier = True + + logger.debug( + "do_auth %s missing_auth: %s", + event.event_id, e.event_id + ) yield self._handle_new_event( origin, e, auth_events=auth ) - auth_events[(e.type, e.state_key)] = e + + if e.event_id in event_auth_events: + auth_events[(e.type, e.state_key)] = e except AuthError: pass @@ -788,20 +818,31 @@ class FederationHandler(BaseHandler): local_auth_chain, ) - # 3. Process any remote auth chain events we haven't seen. - for missing_id in result.get("missing", []): - try: - for e in result["auth_chain"]: - if e.event_id == missing_id: - ev = e - break + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue + + if ev.event_id == event.event_id: + continue + + try: auth_ids = [e_id for e_id, _ in ev.auth_events] auth = { (e.type, e.state_key): e for e in result["auth_chain"] if e.event_id in auth_ids } ev.internal_metadata.outlier = True + + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) + yield self._handle_new_event( origin, ev, auth_events=auth ) From 0b1cc7cc0bd81f93ed353a6ac67e06f91b62ab90 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 16:24:40 +0000 Subject: [PATCH 201/630] Return empty list rather than None when there are no emphemeral events for a room --- synapse/handlers/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index dc69b3cfe..962686f4b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -370,7 +370,7 @@ class SyncHandler(BaseHandler): prev_batch=prev_batch_token, state=state_events_delta, limited=limited, - ephemeral=typing_by_room.get(room_id, None) + ephemeral=typing_by_room.get(room_id, []) ) logging.debug("Room sync: %r", room_sync) From e709d61964834244c9dae673d4e6fefa6075cc5e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 16:56:53 +0000 Subject: [PATCH 202/630] Update documentation to recommend virtual env --- README.rst | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 768da3df6..ddc5d4f78 100644 --- a/README.rst +++ b/README.rst @@ -95,7 +95,7 @@ Installing prerequisites on Ubuntu or Debian:: $ sudo apt-get install build-essential python2.7-dev libffi-dev \ python-pip python-setuptools sqlite3 \ - libssl-dev + libssl-dev python-virtualenv libjpeg-dev Installing prerequisites on Mac OS X:: @@ -103,19 +103,12 @@ Installing prerequisites on Mac OS X:: To install the synapse homeserver run:: - $ pip install --user --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + $ virutalenv ~/.synapse + $ source ~/.synapse/bin/activate + $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master -This installs synapse, along with the libraries it uses, into -``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX. - -Your python may not give priority to locally installed libraries over system -libraries, in which case you must add your local packages to your python path:: - - $ # on Linux: - $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages:$PYTHONPATH - - $ # on OSX: - $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages:$PYTHONPATH +This installs synapse, along with the libraries it uses, into a virtual +environment under ``synapse``. For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. @@ -182,23 +175,13 @@ Running Your Homeserver To actually run your new homeserver, pick a working directory for Synapse to run (e.g. ``~/.synapse``), and:: - $ mkdir ~/.synapse $ cd ~/.synapse - - $ # on Linux - $ ~/.local/bin/synctl start - - $ # on OSX - $ ~/Library/Python/2.7/bin/synctl start + $ ./bin/activate + $ synctl start Troubleshooting Running ----------------------- -If ``synctl`` fails with ``pkg_resources.DistributionNotFound`` errors you may -need a newer version of setuptools than that provided by your OS.:: - - $ sudo pip install setuptools --upgrade - If synapse fails with ``missing "sodium.h"`` crypto errors, you may need to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for encryption and digital signatures. @@ -225,13 +208,15 @@ directory of your choice:: $ cd synapse The homeserver has a number of external dependencies, that are easiest -to install by making setup.py do so, in --user mode:: +to install using pip and a virtualenv:: - $ python setup.py develop --user + $ virtualenv env + $ env/bin/activate + $ python synapse/dependencies | xargs -i pip install + $ pip install setuptools_trial mock -This will run a process of downloading and installing into your -user's .local/lib directory all of the required dependencies that are -missing. +This will run a process of downloading and installing all the needed +dependencies into a virtual env. Once this is done, you may wish to run the homeserver's unit tests, to check that everything is installed as it should be:: From 33cf48118f3a4fcbd0ad31e52c2ad5b09abb2c60 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 17:00:32 +0000 Subject: [PATCH 203/630] Tell people to "source" the activate script for virtualenv, Remove --user from pip install --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index ddc5d4f78..285865f5f 100644 --- a/README.rst +++ b/README.rst @@ -121,19 +121,19 @@ you get errors about ``error: no such option: --process-dependency-links`` you may need to manually upgrade it:: $ sudo pip install --upgrade pip - + If pip crashes mid-installation for reason (e.g. lost terminal), pip may refuse to run until you remove the temporary installation directory it created. To reset the installation:: $ rm -rf /tmp/pip_install_matrix - + pip seems to leak *lots* of memory during installation. For instance, a Linux host with 512MB of RAM may run out of memory whilst installing Twisted. If this happens, you will have to individually install the dependencies which are failing, e.g.:: - $ pip install --user twisted + $ pip install twisted On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you will need to export CFLAGS=-Qunused-arguments. @@ -148,7 +148,7 @@ Synapse can be installed on Cygwin. It requires the following Cygwin packages: - openssl (and openssl-devel, python-openssl) - python - python-setuptools - + The content repository requires additional packages and will be unable to process uploads without them: - libjpeg8 @@ -176,7 +176,7 @@ To actually run your new homeserver, pick a working directory for Synapse to run (e.g. ``~/.synapse``), and:: $ cd ~/.synapse - $ ./bin/activate + $ source ./bin/activate $ synctl start Troubleshooting Running @@ -211,7 +211,7 @@ The homeserver has a number of external dependencies, that are easiest to install using pip and a virtualenv:: $ virtualenv env - $ env/bin/activate + $ source env/bin/activate $ python synapse/dependencies | xargs -i pip install $ pip install setuptools_trial mock @@ -237,7 +237,7 @@ IMPORTANT: Before upgrading an existing homeserver to a new version, please refer to UPGRADE.rst for any additional instructions. Otherwise, simply re-install the new codebase over the current one - e.g. -by ``pip install --user --process-dependency-links +by ``pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master`` if using pip, or by ``git pull`` if running off a git working copy. From 166c2cd4f3e3828668f73bb1f8248d850234fc5c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 17:11:29 +0000 Subject: [PATCH 204/630] add generate config instruction to the HS setup part --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index 285865f5f..820b16d06 100644 --- a/README.rst +++ b/README.rst @@ -110,6 +110,15 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into a virtual environment under ``synapse``. +To set up your homeserver, run (in your virtualenv, as before):: + + $ python -m synapse.app.homeserver \ + --server-name machine.my.domain.name \ + --config-path homeserver.config \ + --generate-config + +Substituting your host and domain name as appropriate. + For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. From 9ccfdfcd7cc15f5c944670e6b6bcfe74090763f1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 17:15:39 +0000 Subject: [PATCH 205/630] Add twisted to setup requires so it gets processed before setuptools_trial --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index abccca0f2..eb1d17c25 100755 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ setup( "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0", ], setup_requires=[ + "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 "setuptools_trial", "setuptools>=1.0.0", # Needs setuptools that supports git+ssh. # TODO: Do we need this now? we don't use git+ssh. From 68bd7dfbb72ee37495bf180e277ace85aec6d34b Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 17:37:37 +0000 Subject: [PATCH 206/630] s/homeserver.config/homeserver.yaml/ because that's what synctl looks for. --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 820b16d06..cac387292 100644 --- a/README.rst +++ b/README.rst @@ -114,7 +114,7 @@ To set up your homeserver, run (in your virtualenv, as before):: $ python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ - --config-path homeserver.config \ + --config-path homeserver.yaml \ --generate-config Substituting your host and domain name as appropriate. @@ -273,9 +273,9 @@ For the first form, simply pass the required hostname (of the machine) as the $ python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ - --config-path homeserver.config \ + --config-path homeserver.yaml \ --generate-config - $ python -m synapse.app.homeserver --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.yaml Alternatively, you can run ``synctl start`` to guide you through the process. @@ -295,9 +295,9 @@ SRV record, as that is the name other machines will expect it to have:: $ python -m synapse.app.homeserver \ --server-name YOURDOMAIN \ --bind-port 8448 \ - --config-path homeserver.config \ + --config-path homeserver.yaml \ --generate-config - $ python -m synapse.app.homeserver --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.yaml You may additionally want to pass one or more "-v" options, in order to From 4c0da49d7ca58774adad870da98a7e168a5be18c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 30 Jan 2015 22:53:13 +0000 Subject: [PATCH 207/630] Resign events when we return them via /query_auth/ --- synapse/handlers/federation.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 623c54d58..8bf5a4cc1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -741,6 +741,15 @@ class FederationHandler(BaseHandler): local_auth_chain, remote_auth_chain ) + for event in ret["auth_chain"]: + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + logger.debug("on_query_auth reutrning: %s", ret) defer.returnValue(ret) From fe10b882b74479b6d139039956932b98629c8165 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 23:05:49 +0000 Subject: [PATCH 208/630] Don't assume all member events have a display nme. --- synapse/push/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 6a302305f..75867b3c3 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -204,10 +204,11 @@ class Pusher(object): event_type='m.room.member', state_key=ev['user_id'] ) - if len(their_member_events_for_room) > 0: - dn = their_member_events_for_room[0].content['displayname'] - if dn is not None: - ctx['sender_display_name'] = dn + for mev in their_member_events_for_room: + if mev.content['membership'] == 'join' and 'displayname' in mev.content: + dn = mev.content['displayname'] + if dn is not None: + ctx['sender_display_name'] = dn defer.returnValue(ctx) From 0b354fcb8452540baf59b714f4d2186615dc5383 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 30 Jan 2015 23:10:35 +0000 Subject: [PATCH 209/630] Again, don't assume all member events have displayname. --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 75867b3c3..28e5dae81 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -101,7 +101,7 @@ class Pusher(object): # This loop does two things: # 1) Find our current display name - if mev.state_key == self.user_name: + if mev.state_key == self.user_name and 'displayname' in mev.content: my_display_name = mev.content['displayname'] # and 2) Get the number of people in that room From e9dfc4cfae3a9abc85ae2882ebfbd0df689a9d2f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 31 Jan 2015 06:09:59 +0100 Subject: [PATCH 210/630] fix OSX stuff and typos --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index cac387292..282a53873 100644 --- a/README.rst +++ b/README.rst @@ -100,15 +100,16 @@ Installing prerequisites on Ubuntu or Debian:: Installing prerequisites on Mac OS X:: $ xcode-select --install + $ sudo pip install virtualenv To install the synapse homeserver run:: - $ virutalenv ~/.synapse + $ virtualenv ~/.synapse $ source ~/.synapse/bin/activate $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master This installs synapse, along with the libraries it uses, into a virtual -environment under ``synapse``. +environment under ``~/.synapse``. To set up your homeserver, run (in your virtualenv, as before):: From 365e007bee27e230f1280a2174cfc9abfb2deac2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Sat, 31 Jan 2015 12:48:06 +0000 Subject: [PATCH 211/630] Ignore empty strings for display names & room names in notifications --- synapse/push/httppusher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index d4c5f03b0..7c6953c98 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -92,9 +92,9 @@ class HttpPusher(Pusher): if len(ctx['aliases']): d['notification']['room_alias'] = ctx['aliases'][0] - if 'sender_display_name' in ctx: + if 'sender_display_name' in ctx and len(ctx['sender_display_name']) > 0: d['notification']['sender_display_name'] = ctx['sender_display_name'] - if 'name' in ctx: + if 'name' in ctx and len(ctx['name']) > 0: d['notification']['room_name'] = ctx['name'] defer.returnValue(d) From c934760014dd71db9c1457be44d734806762a1eb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Feb 2015 13:03:03 +0000 Subject: [PATCH 212/630] Bump version of webclient pulled in. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 043cd044a..b8115a701 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.6.0", + "matrix_angular_sdk==0.6.1", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", @@ -47,7 +47,7 @@ setup( dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0", + "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.1/#egg=matrix_angular_sdk-0.6.1", ], setup_requires=[ "setuptools_trial", From 3b33529dfdf1a94aebd357eebdfc75e903bf1591 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Feb 2015 13:03:25 +0000 Subject: [PATCH 213/630] Bump version --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 3b3e72317..492b488cd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1b +0.6.1c diff --git a/synapse/__init__.py b/synapse/__init__.py index e7e27b06e..fdfe5eda3 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1b" +__version__ = "0.6.1c" From 83c31735d065934a4b647fb2200646093f5745b5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Feb 2015 13:07:43 +0000 Subject: [PATCH 214/630] Use >= for version of webclient --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b8115a701..f0e9e7d26 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.6.1", + "matrix_angular_sdk>=0.6.1", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", From b993555bf4ed829af71129bff80b76d2891aa514 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 16:56:53 +0000 Subject: [PATCH 215/630] Update documentation to recommend virtual env Conflicts: README.rst --- README.rst | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 92b94bcd7..ddc5d4f78 100644 --- a/README.rst +++ b/README.rst @@ -95,7 +95,7 @@ Installing prerequisites on Ubuntu or Debian:: $ sudo apt-get install build-essential python2.7-dev libffi-dev \ python-pip python-setuptools sqlite3 \ - libssl-dev + libssl-dev python-virtualenv libjpeg-dev Installing prerequisites on Mac OS X:: @@ -103,10 +103,12 @@ Installing prerequisites on Mac OS X:: To install the synapse homeserver run:: - $ pip install --user --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + $ virutalenv ~/.synapse + $ source ~/.synapse/bin/activate + $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master -This installs synapse, along with the libraries it uses, into -``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX. +This installs synapse, along with the libraries it uses, into a virtual +environment under ``synapse``. For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. @@ -173,23 +175,13 @@ Running Your Homeserver To actually run your new homeserver, pick a working directory for Synapse to run (e.g. ``~/.synapse``), and:: - $ mkdir ~/.synapse $ cd ~/.synapse - - $ # on Linux - $ ~/.local/bin/synctl start - - $ # on OSX - $ ~/Library/Python/2.7/bin/synctl start + $ ./bin/activate + $ synctl start Troubleshooting Running ----------------------- -If ``synctl`` fails with ``pkg_resources.DistributionNotFound`` errors you may -need a newer version of setuptools than that provided by your OS.:: - - $ sudo pip install setuptools --upgrade - If synapse fails with ``missing "sodium.h"`` crypto errors, you may need to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for encryption and digital signatures. @@ -216,13 +208,15 @@ directory of your choice:: $ cd synapse The homeserver has a number of external dependencies, that are easiest -to install by making setup.py do so, in --user mode:: +to install using pip and a virtualenv:: - $ python setup.py develop --user + $ virtualenv env + $ env/bin/activate + $ python synapse/dependencies | xargs -i pip install + $ pip install setuptools_trial mock -This will run a process of downloading and installing into your -user's .local/lib directory all of the required dependencies that are -missing. +This will run a process of downloading and installing all the needed +dependencies into a virtual env. Once this is done, you may wish to run the homeserver's unit tests, to check that everything is installed as it should be:: From c3979b236e35523bc68a5473b922d41a63dceae6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 30 Jan 2015 17:00:32 +0000 Subject: [PATCH 216/630] Tell people to "source" the activate script for virtualenv, Remove --user from pip install --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index ddc5d4f78..285865f5f 100644 --- a/README.rst +++ b/README.rst @@ -121,19 +121,19 @@ you get errors about ``error: no such option: --process-dependency-links`` you may need to manually upgrade it:: $ sudo pip install --upgrade pip - + If pip crashes mid-installation for reason (e.g. lost terminal), pip may refuse to run until you remove the temporary installation directory it created. To reset the installation:: $ rm -rf /tmp/pip_install_matrix - + pip seems to leak *lots* of memory during installation. For instance, a Linux host with 512MB of RAM may run out of memory whilst installing Twisted. If this happens, you will have to individually install the dependencies which are failing, e.g.:: - $ pip install --user twisted + $ pip install twisted On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you will need to export CFLAGS=-Qunused-arguments. @@ -148,7 +148,7 @@ Synapse can be installed on Cygwin. It requires the following Cygwin packages: - openssl (and openssl-devel, python-openssl) - python - python-setuptools - + The content repository requires additional packages and will be unable to process uploads without them: - libjpeg8 @@ -176,7 +176,7 @@ To actually run your new homeserver, pick a working directory for Synapse to run (e.g. ``~/.synapse``), and:: $ cd ~/.synapse - $ ./bin/activate + $ source ./bin/activate $ synctl start Troubleshooting Running @@ -211,7 +211,7 @@ The homeserver has a number of external dependencies, that are easiest to install using pip and a virtualenv:: $ virtualenv env - $ env/bin/activate + $ source env/bin/activate $ python synapse/dependencies | xargs -i pip install $ pip install setuptools_trial mock @@ -237,7 +237,7 @@ IMPORTANT: Before upgrading an existing homeserver to a new version, please refer to UPGRADE.rst for any additional instructions. Otherwise, simply re-install the new codebase over the current one - e.g. -by ``pip install --user --process-dependency-links +by ``pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master`` if using pip, or by ``git pull`` if running off a git working copy. From d3dd7490447bc3bd067ea34af8a69f157374c980 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 13:17:45 +0000 Subject: [PATCH 217/630] Pin the version of Twisted to 14.0.2 since we are using some of its internals --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f0e9e7d26..bde26c0b9 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( install_requires=[ "syutil==0.0.2", "matrix_angular_sdk>=0.6.1", - "Twisted>=14.0.0", + "Twisted==14.0.2", "service_identity>=1.0.0", "pyopenssl>=0.14", "pyyaml", @@ -50,6 +50,7 @@ setup( "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.1/#egg=matrix_angular_sdk-0.6.1", ], setup_requires=[ + "Twisted==14.0.2", "setuptools_trial", "setuptools>=1.0.0", # Needs setuptools that supports git+ssh. # TODO: Do we need this now? we don't use git+ssh. From d8cf06e525241252bcfac358d44f98d15dba9d0f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 13:18:36 +0000 Subject: [PATCH 218/630] Bump version to 0.6.1d --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 492b488cd..1c29ff4d3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1c +0.6.1d diff --git a/synapse/__init__.py b/synapse/__init__.py index fdfe5eda3..dacda8986 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1c" +__version__ = "0.6.1d" From 30ed0884fcd282710dc5f4d401feb39e81b9b8f2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 31 Jan 2015 06:09:59 +0100 Subject: [PATCH 219/630] fix OSX stuff and typos --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 285865f5f..ccbee989c 100644 --- a/README.rst +++ b/README.rst @@ -100,15 +100,16 @@ Installing prerequisites on Ubuntu or Debian:: Installing prerequisites on Mac OS X:: $ xcode-select --install + $ sudo pip install virtualenv To install the synapse homeserver run:: - $ virutalenv ~/.synapse + $ virtualenv ~/.synapse $ source ~/.synapse/bin/activate $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master This installs synapse, along with the libraries it uses, into a virtual -environment under ``synapse``. +environment under ``~/.synapse``. For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. From 22c1ffb0a0b1a817acade3e8a19948b208a2c9f3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 16:02:31 +0000 Subject: [PATCH 220/630] Add a media/v1/identicon resource for generating identicons using pydenticon --- setup.py | 1 + synapse/rest/media/v1/identicon_resource.py | 48 +++++++++++++++++++++ synapse/rest/media/v1/media_repository.py | 2 + 3 files changed, 51 insertions(+) create mode 100644 synapse/rest/media/v1/identicon_resource.py diff --git a/setup.py b/setup.py index 28e8188f9..bd2766b24 100755 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ setup( "py-bcrypt", "frozendict>=0.4", "pillow", + "pydenticon", ], dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py new file mode 100644 index 000000000..8878d4c3d --- /dev/null +++ b/synapse/rest/media/v1/identicon_resource.py @@ -0,0 +1,48 @@ +from pydenticon import Generator +from twisted.web.resource import Resource + +FOREGROUND = [ + "rgb(45,79,255)", + "rgb(254,180,44)", + "rgb(226,121,234)", + "rgb(30,179,253)", + "rgb(232,77,65)", + "rgb(49,203,115)", + "rgb(141,69,170)" +] + +BACKGROUND = "rgb(224,224,224)" +SIZE = 5 + + +class IdenticonResource(Resource): + isLeaf = True + + def __init__(self): + Resource.__init__(self) + self.generator = Generator( + SIZE, SIZE, foreground=FOREGROUND, background=BACKGROUND, + ) + + def generate_identicon(self, name, width, height): + v_padding = width % SIZE + h_padding = height % SIZE + top_padding = v_padding // 2 + left_padding = h_padding // 2 + bottom_padding = v_padding - top_padding + right_padding = h_padding - left_padding + width -= v_padding + height -= h_padding + padding = (top_padding, bottom_padding, left_padding, right_padding) + identicon = self.generator.generate( + name, width, height, padding=padding + ) + return identicon + + def render_GET(self, request): + name = "/".join(request.postpath) + width = int(request.args.get("width", 96)) + height = int(request.args.get("width", 96)) + identicon_bytes = self.generate_identicon(name, width, height) + request.setHeader(b"Content-Type", b"image/png") + return identicon_bytes diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 461cc001f..61ed90f39 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -16,6 +16,7 @@ from .upload_resource import UploadResource from .download_resource import DownloadResource from .thumbnail_resource import ThumbnailResource +from .identicon_resource import IdenticonResource from .filepath import MediaFilePaths from twisted.web.resource import Resource @@ -75,3 +76,4 @@ class MediaRepositoryResource(Resource): self.putChild("upload", UploadResource(hs, filepaths)) self.putChild("download", DownloadResource(hs, filepaths)) self.putChild("thumbnail", ThumbnailResource(hs, filepaths)) + self.putChild("identicon", IdenticonResource()) From a006d168c556ca71ad0bbb680c60f1ba170338fb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Feb 2015 16:05:34 +0000 Subject: [PATCH 221/630] Actually merge into develop. --- synapse/storage/__init__.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9bbd553df..1f207495f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,12 +67,9 @@ SCHEMAS = [ "event_signatures", "pusher", "media_repository", -<<<<<<< HEAD "application_services" -======= "filtering", "rejections", ->>>>>>> develop ] @@ -92,17 +89,13 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, -<<<<<<< HEAD - EventFederationStore, MediaRepositoryStore, - ApplicationServiceStore -======= + ApplicationServiceStore, EventFederationStore, MediaRepositoryStore, RejectionsStore, FilteringStore, PusherStore, PushRuleStore ->>>>>>> develop ): def __init__(self, hs): From 038f5afb07aa1df27528116eeed0f6e44d67c579 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 16:29:18 +0000 Subject: [PATCH 222/630] Spell height more correctly --- synapse/rest/media/v1/identicon_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py index 8878d4c3d..636e3a33c 100644 --- a/synapse/rest/media/v1/identicon_resource.py +++ b/synapse/rest/media/v1/identicon_resource.py @@ -42,7 +42,7 @@ class IdenticonResource(Resource): def render_GET(self, request): name = "/".join(request.postpath) width = int(request.args.get("width", 96)) - height = int(request.args.get("width", 96)) + height = int(request.args.get("height", 96)) identicon_bytes = self.generate_identicon(name, width, height) request.setHeader(b"Content-Type", b"image/png") return identicon_bytes From f2eda123b78a7b61587db60d6c8995571a7eb7fb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 16:32:33 +0000 Subject: [PATCH 223/630] Fix setting identicon width and height --- synapse/rest/media/v1/identicon_resource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py index 636e3a33c..12d98e177 100644 --- a/synapse/rest/media/v1/identicon_resource.py +++ b/synapse/rest/media/v1/identicon_resource.py @@ -41,8 +41,8 @@ class IdenticonResource(Resource): def render_GET(self, request): name = "/".join(request.postpath) - width = int(request.args.get("width", 96)) - height = int(request.args.get("height", 96)) + width = int(request.args.get("width", [96])[0]) + height = int(request.args.get("height", [96])[0]) identicon_bytes = self.generate_identicon(name, width, height) request.setHeader(b"Content-Type", b"image/png") return identicon_bytes From 941f59101b51e9225dbdc38b22110a01de194242 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Feb 2015 16:56:01 +0000 Subject: [PATCH 224/630] Don't fail an entire request if one of the returned events fails a signature check. If an event does fail a signature check, look in the local database and request it from the originator. --- synapse/federation/federation_client.py | 107 ++++++++++++++++++------ synapse/storage/__init__.py | 21 +++-- 2 files changed, 94 insertions(+), 34 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index e1539bd0e..b809e935a 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -224,17 +224,17 @@ class FederationClient(object): for p in result.get("auth_chain", []) ] - for i, pdu in enumerate(pdus): - pdus[i] = yield self._check_sigs_and_hash(pdu) + signed_pdus = yield self._check_sigs_and_hash_and_fetch( + pdus, outlier=True + ) - # FIXME: We should handle signature failures more gracefully. + signed_auth = yield self._check_sigs_and_hash_and_fetch( + auth_chain, outlier=True + ) - for i, pdu in enumerate(auth_chain): - auth_chain[i] = yield self._check_sigs_and_hash(pdu) + signed_auth.sort(key=lambda e: e.depth) - # FIXME: We should handle signature failures more gracefully. - - defer.returnValue((pdus, auth_chain)) + defer.returnValue((signed_pdus, signed_auth)) @defer.inlineCallbacks @log_function @@ -248,14 +248,13 @@ class FederationClient(object): for p in res["auth_chain"] ] - for i, pdu in enumerate(auth_chain): - auth_chain[i] = yield self._check_sigs_and_hash(pdu) + signed_auth = yield self._check_sigs_and_hash_and_fetch( + auth_chain, outlier=True + ) - # FIXME: We should handle signature failures more gracefully. + signed_auth.sort(key=lambda e: e.depth) - auth_chain.sort(key=lambda e: e.depth) - - defer.returnValue(auth_chain) + defer.returnValue(signed_auth) @defer.inlineCallbacks def make_join(self, destination, room_id, user_id): @@ -291,21 +290,19 @@ class FederationClient(object): for p in content.get("auth_chain", []) ] - for i, pdu in enumerate(state): - state[i] = yield self._check_sigs_and_hash(pdu) + signed_state = yield self._check_sigs_and_hash_and_fetch( + state, outlier=True + ) - # FIXME: We should handle signature failures more gracefully. - - for i, pdu in enumerate(auth_chain): - auth_chain[i] = yield self._check_sigs_and_hash(pdu) - - # FIXME: We should handle signature failures more gracefully. + signed_auth = yield self._check_sigs_and_hash_and_fetch( + auth_chain, outlier=True + ) auth_chain.sort(key=lambda e: e.depth) defer.returnValue({ - "state": state, - "auth_chain": auth_chain, + "state": signed_state, + "auth_chain": signed_auth, }) @defer.inlineCallbacks @@ -353,12 +350,18 @@ class FederationClient(object): ) auth_chain = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + self.event_from_pdu_json(e) for e in content["auth_chain"] ] + signed_auth = yield self._check_sigs_and_hash_and_fetch( + auth_chain, outlier=True + ) + + signed_auth.sort(key=lambda e: e.depth) + ret = { - "auth_chain": auth_chain, + "auth_chain": signed_auth, "rejects": content.get("rejects", []), "missing": content.get("missing", []), } @@ -374,6 +377,58 @@ class FederationClient(object): return event + @defer.inlineCallbacks + def _check_sigs_and_hash_and_fetch(self, pdus, outlier=False): + """Takes a list of PDUs and checks the signatures and hashs of each + one. If a PDU fails its signature check then we check if we have it in + the database and if not then request if from the originating server of + that PDU. + + If a PDU fails its content hash check then it is redacted. + + The given list of PDUs are not modified, instead the function returns + a new list. + + Args: + pdu (list) + outlier (bool) + + Returns: + Deferred : A list of PDUs that have valid signatures and hashes. + """ + signed_pdus = [] + for pdu in pdus: + try: + new_pdu = yield self._check_sigs_and_hash(pdu) + signed_pdus.append(new_pdu) + except SynapseError: + # FIXME: We should handle signature failures more gracefully. + + # Check local db. + new_pdu = yield self.store.get_event( + pdu.event_id, + allow_rejected=True + ) + if new_pdu: + signed_pdus.append(new_pdu) + continue + + # Check pdu.origin + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) + + if new_pdu: + signed_pdus.append(new_pdu) + continue + + logger.warn("Failed to find copy of %s with valid signature") + + defer.returnValue(signed_pdus) + + @defer.inlineCallbacks def _check_sigs_and_hash(self, pdu): """Throws a SynapseError if the PDU does not have the correct diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7c54b1b9d..b4a7a3f06 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -128,16 +128,21 @@ class DataStore(RoomMemberStore, RoomStore, pass @defer.inlineCallbacks - def get_event(self, event_id, allow_none=False): - events = yield self._get_events([event_id]) + def get_event(self, event_id, check_redacted=True, + get_prev_content=False, allow_rejected=False, + allow_none=False): + event = yield self.runInteraction( + "get_event", self._get_event_txn, + event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) - if not events: - if allow_none: - defer.returnValue(None) - else: - raise RuntimeError("Could not find event %s" % (event_id,)) + if not event and not allow_none: + raise RuntimeError("Could not find event %s" % (event_id,)) - defer.returnValue(events[0]) + defer.returnValue(event) @log_function def _persist_event_txn(self, txn, event, context, backfilled, From 1bb0528316729eb19536d8f3a45564d8dc02cb4a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 16:34:07 +0000 Subject: [PATCH 225/630] Add Cache-Control header to identicon --- synapse/rest/media/v1/identicon_resource.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py index 12d98e177..912856386 100644 --- a/synapse/rest/media/v1/identicon_resource.py +++ b/synapse/rest/media/v1/identicon_resource.py @@ -45,4 +45,7 @@ class IdenticonResource(Resource): height = int(request.args.get("height", [96])[0]) identicon_bytes = self.generate_identicon(name, width, height) request.setHeader(b"Content-Type", b"image/png") + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" + ) return identicon_bytes From 40c6fe1b81e4d92cba797b0c966fd774e2a60a28 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Feb 2015 17:06:37 +0000 Subject: [PATCH 226/630] Don't bother requesting PDUs with bad signatures from the same server --- synapse/federation/federation_client.py | 31 +++++++++++++------------ 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index b809e935a..f87e84db7 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -225,11 +225,11 @@ class FederationClient(object): ] signed_pdus = yield self._check_sigs_and_hash_and_fetch( - pdus, outlier=True + destination, pdus, outlier=True ) signed_auth = yield self._check_sigs_and_hash_and_fetch( - auth_chain, outlier=True + destination, auth_chain, outlier=True ) signed_auth.sort(key=lambda e: e.depth) @@ -249,7 +249,7 @@ class FederationClient(object): ] signed_auth = yield self._check_sigs_and_hash_and_fetch( - auth_chain, outlier=True + destination, auth_chain, outlier=True ) signed_auth.sort(key=lambda e: e.depth) @@ -291,11 +291,11 @@ class FederationClient(object): ] signed_state = yield self._check_sigs_and_hash_and_fetch( - state, outlier=True + destination, state, outlier=True ) signed_auth = yield self._check_sigs_and_hash_and_fetch( - auth_chain, outlier=True + destination, auth_chain, outlier=True ) auth_chain.sort(key=lambda e: e.depth) @@ -355,7 +355,7 @@ class FederationClient(object): ] signed_auth = yield self._check_sigs_and_hash_and_fetch( - auth_chain, outlier=True + destination, auth_chain, outlier=True ) signed_auth.sort(key=lambda e: e.depth) @@ -378,7 +378,7 @@ class FederationClient(object): return event @defer.inlineCallbacks - def _check_sigs_and_hash_and_fetch(self, pdus, outlier=False): + def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False): """Takes a list of PDUs and checks the signatures and hashs of each one. If a PDU fails its signature check then we check if we have it in the database and if not then request if from the originating server of @@ -414,15 +414,16 @@ class FederationClient(object): continue # Check pdu.origin - new_pdu = yield self.get_pdu( - destinations=[pdu.origin], - event_id=pdu.event_id, - outlier=outlier, - ) + if pdu.origin != origin: + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) - if new_pdu: - signed_pdus.append(new_pdu) - continue + if new_pdu: + signed_pdus.append(new_pdu) + continue logger.warn("Failed to find copy of %s with valid signature") From 8c52e6e8a1a9afd3e5d3164b7ac97c4986d6d353 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 2 Feb 2015 17:12:23 +0000 Subject: [PATCH 227/630] fix typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 282a53873..01e0f36c8 100644 --- a/README.rst +++ b/README.rst @@ -222,7 +222,7 @@ to install using pip and a virtualenv:: $ virtualenv env $ source env/bin/activate - $ python synapse/dependencies | xargs -i pip install + $ python synapse/python_dependencies.py | xargs -i pip install $ pip install setuptools_trial mock This will run a process of downloading and installing all the needed From 4574b5a9e6a52086bd08d23409cc6e15070fe904 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 17:22:40 +0000 Subject: [PATCH 228/630] Generate a list of dependencies from synapse/python_dependencies.py --- synapse/python_dependencies.py | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 826a36f20..da15c6da4 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,6 +18,27 @@ REQUIREMENTS = { "pillow": ["PIL"], } +def github_link(project, version, egg): + return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) + +DEPENDENCY_LINKS=[ + github_link( + project="matrix-org/syutil", + version="v0.0.2", + egg="syutil-0.0.2", + ), + github_link( + project="matrix-org/matrix-angular-sdk", + version="v0.6.0", + egg="matrix_angular_sdk-0.6.0", + ), + github_link( + project="pyca/pynacl", + version="d4d3175589b892f6ea7c22f466e0e223853516fa", + egg="pynacl-0.3.0", + ) +] + class MissingRequirementError(Exception): pass @@ -78,3 +99,23 @@ def check_requirements(): "Unexpected version of %r in %r. %r != %r" % (dependency, file_path, version, required_version) ) + +def list_requirements(): + result = [] + linked = [] + for link in DEPENDENCY_LINKS: + egg = link.split("#egg=")[1] + linked.append(egg.split('-')[0]) + result.append(link) + for requirement in REQUIREMENTS: + is_linked = False + for link in linked: + if requirement.replace('-','_').startswith(link): + is_linked = True + if not is_linked: + result.append(requirement) + return result + +if __name__ == "__main__": + import sys + sys.stdout.writelines(req + "\n" for req in list_requirements()) From f3a4267757cbbfbfc996d2ececf34f98928fc90f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 2 Feb 2015 17:31:58 +0000 Subject: [PATCH 229/630] less obscure xargs --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 01e0f36c8..e1fa4d75f 100644 --- a/README.rst +++ b/README.rst @@ -222,7 +222,7 @@ to install using pip and a virtualenv:: $ virtualenv env $ source env/bin/activate - $ python synapse/python_dependencies.py | xargs -i pip install + $ python synapse/python_dependencies.py | xargs -n1 pip install $ pip install setuptools_trial mock This will run a process of downloading and installing all the needed From a2da04b8ab269a8353ecfca20132c273e972a4db Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 2 Feb 2015 17:37:26 +0000 Subject: [PATCH 230/630] Add pydenticon to python_dependencies --- synapse/python_dependencies.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 826a36f20..168fab065 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -16,6 +16,7 @@ REQUIREMENTS = { "py-bcrypt": ["bcrypt"], "frozendict>=0.4": ["frozendict"], "pillow": ["PIL"], + "pydenticon": ["pydenticon"], } From 1a2de0c5feb1183b35045bb7fb9e379a9598d1cb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Feb 2015 17:39:41 +0000 Subject: [PATCH 231/630] Implement txns for AS (un)registration. --- synapse/handlers/appservice.py | 3 +- synapse/storage/__init__.py | 2 +- synapse/storage/appservice.py | 120 +++++++++++++++++++++++++++------ 3 files changed, 102 insertions(+), 23 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8bd475cbf..da994ba8e 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -43,7 +43,8 @@ class ApplicationServicesHandler(BaseHandler): "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - # TODO store this AS + logger.info("Updating application service info...") + yield self.store.update_app_service(app_service) def unregister(self, token): logger.info("Unregister as_token=%s", token) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1f207495f..6ff009313 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,7 @@ SCHEMAS = [ "event_signatures", "pusher", "media_repository", - "application_services" + "application_services", "filtering", "rejections", ] diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 5a0e47e0d..db0c54621 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -15,11 +15,17 @@ import logging from twisted.internet import defer +from synapse.api.errors import StoreError from ._base import SQLBaseStore logger = logging.getLogger(__name__) +namespace_enum = [ + "users", # 0 + "aliases", # 1 + "rooms" # 2 +] # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): @@ -30,25 +36,26 @@ class ApplicationService(object): def __init__(self, token, url=None, namespaces=None): self.token = token - if url: - self.url = url - if namespaces: - self._set_namespaces(namespaces) + self.url = url + self.namespaces = self._get_namespaces(namespaces) - def _set_namespaces(self, namespaces): + def _get_namespaces(self, namespaces): # Sanity check that it is of the form: # { # users: ["regex",...], # aliases: ["regex",...], # rooms: ["regex",...], # } + if not namespaces: + return None + for ns in ["users", "rooms", "aliases"]: if type(namespaces[ns]) != list: raise ValueError("Bad namespace value for '%s'", ns) for regex in namespaces[ns]: if not isinstance(regex, basestring): raise ValueError("Expected string regex for ns '%s'", ns) - self.namespaces = namespaces + return namespaces def is_interested(self, event): """Check if this service is interested in this event. @@ -110,10 +117,38 @@ class ApplicationServiceStore(SQLBaseStore): This removes all AS specific regex and the base URL. The token is the only thing preserved for future registration attempts. """ - # TODO: DELETE FROM application_services_regex WHERE id=this service - # TODO: SET url=NULL WHERE token=token - # TODO: Update cache - pass + yield self.runInteraction( + "unregister_app_service", + self._unregister_app_service_txn, + token, + ) + # update cache TODO: Should this be in the txn? + for service in self.cache.services: + if service.token == token: + service.url = None + service.namespaces = None + + def _unregister_app_service_txn(self, txn, token): + # kill the url to prevent pushes + txn.execute( + "UPDATE application_services SET url=NULL WHERE token=?", + (token,) + ) + + # cleanup regex + as_id = self._get_as_id_txn(txn, token) + if not as_id: + logger.warning( + "unregister_app_service_txn: Failed to find as_id for token=", + token + ) + return False + + txn.execute( + "DELETE FROM application_services_regex WHERE as_id=?", + (as_id,) + ) + return True def update_app_service(self, service): """Update an application service, clobbering what was previously there. @@ -124,12 +159,61 @@ class ApplicationServiceStore(SQLBaseStore): # NB: There is no "insert" since we provide no public-facing API to # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. + if not service.token or not service.url: + raise StoreError(400, "Token and url must be specified.") - # TODO: UPDATE application_services, SET url WHERE token=service.token - # TODO: DELETE FROM application_services_regex WHERE id=this service - # TODO: INSERT INTO application_services_regex - # TODO: Update cache - pass + yield self.runInteraction( + "update_app_service", + self._update_app_service_txn, + service + ) + + # update cache TODO: Should this be in the txn? + for (index, cache_service) in enumerate(self.cache.services): + if service.token == cache_service.token: + self.cache.services[index] = service + logger.info("Updated: %s", service) + return + # new entry + self.cache.services.append(service) + logger.info("Updated(new): %s", service) + + def _update_app_service_txn(self, txn, service): + as_id = self._get_as_id_txn(txn, service.token) + if not as_id: + logger.warning( + "update_app_service_txn: Failed to find as_id for token=", + service.token + ) + return False + + txn.execute( + "UPDATE application_services SET url=? WHERE id=?", + (service.url, as_id,) + ) + # cleanup regex + txn.execute( + "DELETE FROM application_services_regex WHERE id=?", + (as_id,) + ) + for (ns_int, ns_str) in enumerate(namespace_enum): + if ns_str in service.namespaces: + for regex in service.namespaces[ns_str]: + txn.execute( + "INSERT INTO application_services_regex(" + "as_id, namespace, regex) values(?,?,?)", + (as_id, ns_int, regex) + ) + return True + + def _get_as_id_txn(self, txn, token): + cursor = txn.execute( + "SELECT id FROM application_services WHERE token=?", + (token,) + ) + res = cursor.fetchone() + if res: + return res[0] def get_services_for_event(self, event): return self.cache.get_services_for_event(event) @@ -161,12 +245,6 @@ class ApplicationServiceStore(SQLBaseStore): sql = ("SELECT * FROM application_services LEFT JOIN " "application_services_regex ON application_services.id = " "application_services_regex.as_id") - - namespace_enum = [ - "users", # 0 - "aliases", # 1 - "rooms" # 2 - ] # SQL results in the form: # [ # { From 9ff349a3cb1868bb2827047ed0f0d01a9a4c38c7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 2 Feb 2015 17:42:49 +0000 Subject: [PATCH 232/630] Add defers in the right places. --- synapse/storage/appservice.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index db0c54621..dd9b34937 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -111,6 +111,7 @@ class ApplicationServiceStore(SQLBaseStore): self.cache = ApplicationServiceCache() self._populate_cache() + @defer.inlineCallbacks def unregister_app_service(self, token): """Unregisters this service. @@ -150,6 +151,7 @@ class ApplicationServiceStore(SQLBaseStore): ) return True + @defer.inlineCallbacks def update_app_service(self, service): """Update an application service, clobbering what was previously there. From e7ca813dd476c83497d4130ad8efa9424d86e921 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 10:38:14 +0000 Subject: [PATCH 233/630] Try to ensure we don't persist an event we have already persisted. In persist_event check if we already have the event, if so then update instead of replacing so that we don't cause a bump of the stream_ordering. --- synapse/handlers/federation.py | 42 ++++++++++++++++++++----------- synapse/storage/__init__.py | 40 ++++++++++++++++++++++++++--- tests/handlers/test_federation.py | 5 +++- 3 files changed, 68 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8bf5a4cc1..c384789c2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -112,6 +112,14 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) + event_ids = set() + if state: + event_ids += {e.event_id for e in state} + if auth_chain: + event_ids += {e.event_id for e in auth_chain} + + seen_ids = (yield self.store.have_events(event_ids)).keys() + # FIXME (erikj): Awful hack to make the case where we are not currently # in the room work current_state = None @@ -124,20 +132,26 @@ class FederationHandler(BaseHandler): current_state = state if state and auth_chain is not None: - for e in state: - e.internal_metadata.outlier = True - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - yield self._handle_new_event(origin, e, auth_events=auth) - except: - logger.exception( - "Failed to handle state event %s", - e.event_id, - ) + for list_of_pdus in [auth_chain, state]: + for e in list_of_pdus: + if e.event_id in seen_ids: + continue + + e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + except: + logger.exception( + "Failed to handle state event %s", + e.event_id, + ) try: yield self._handle_new_event( diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b4a7a3f06..93aefe0c4 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -161,6 +161,39 @@ class DataStore(RoomMemberStore, RoomStore, outlier = event.internal_metadata.is_outlier() + have_persisted = self._simple_select_one_onecol_txn( + txn, + table="event_json", + keyvalues={"event_id": event.event_id}, + retcol="event_id", + allow_none=True, + ) + + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ) + + if have_persisted: + if not outlier: + sql = ( + "UPDATE event_json SET internal_metadata = ?" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (metadata_json.decode("UTF-8"), event.event_id,) + ) + + sql = ( + "UPDATE events SET outlier = 0" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (event.event_id,) + ) + return + event_dict = { k: v for k, v in event.get_dict().items() @@ -170,10 +203,6 @@ class DataStore(RoomMemberStore, RoomStore, ] } - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() - ) - self._simple_insert_txn( txn, table="event_json", @@ -482,6 +511,9 @@ class DataStore(RoomMemberStore, RoomStore, the rejected reason string if we rejected the event, else maps to None. """ + if not event_ids: + return defer.succeed({}) + def f(txn): sql = ( "SELECT e.event_id, reason FROM events as e " diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 44dbce6be..427048113 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -91,7 +91,10 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) - self.datastore.have_events.return_value = defer.succeed({}) + + def have_events(event_ids): + return defer.succeed({}) + self.datastore.have_events.side_effect = have_events def annotate(ev, old_state=None): context = Mock() From 51969f9e5f4774e4d0ddcc2e8ebcf96803cbf295 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 10:40:14 +0000 Subject: [PATCH 234/630] Return rejected events if asked for it over federation. --- synapse/handlers/federation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c384789c2..0161fbe49 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -632,6 +632,7 @@ class FederationHandler(BaseHandler): event = yield self.store.get_event( event_id, allow_none=True, + allow_rejected=True, ) if event: From 0f48e22ef66ff8a34d4af13c25e20a461c8a8390 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 10:43:29 +0000 Subject: [PATCH 235/630] PEP8 --- synapse/federation/federation_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index f87e84db7..9ceb66e6f 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -429,7 +429,6 @@ class FederationClient(object): defer.returnValue(signed_pdus) - @defer.inlineCallbacks def _check_sigs_and_hash(self, pdu): """Throws a SynapseError if the PDU does not have the correct From 4ff2273b300c0a40fcf054a613aac24c51af22f1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 11:23:26 +0000 Subject: [PATCH 236/630] Add FIXME note. --- synapse/handlers/federation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0161fbe49..322c1b407 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -716,6 +716,8 @@ class FederationHandler(BaseHandler): context.rejected = RejectedReason.AUTH_ERROR + # FIXME: Don't store as rejected with AUTH_ERROR if we haven't + # seen all the auth events. yield self.store.persist_event( event, context=context, From 06c34bfbaee553b851e0f02d8e17d5bff7360376 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 11:23:44 +0000 Subject: [PATCH 237/630] Give exception better message --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 322c1b407..9d7557f57 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1000,7 +1000,7 @@ class FederationHandler(BaseHandler): if reason is None: # FIXME: ERRR?! logger.warn("Could not find reason for %s", e.event_id) - raise RuntimeError("") + raise RuntimeError("Could not find reason for %s" % e.event_id) reason_map[e.event_id] = reason From 197f3ea4bad066da251c7925336baab8bee296c9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 11:26:33 +0000 Subject: [PATCH 238/630] Implement regex checks for app services. Expose handler.get_services_for_event which manages the checks for all services. --- synapse/handlers/appservice.py | 25 +++++++++++- synapse/storage/appservice.py | 71 +++++++++++++++++++++------------- 2 files changed, 67 insertions(+), 29 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index da994ba8e..9b8dd1bb4 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -34,9 +34,11 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Register -> %s", app_service) # check the token is recognised try: - stored_service = yield self.store.get_app_service(app_service.token) + stored_service = yield self.store.get_app_service_by_token( + app_service.token + ) if not stored_service: - raise StoreError(404, "Application Service Not found") + raise StoreError(404, "Application service not found") except StoreError: raise SynapseError( 403, "Unrecognised application services token. " @@ -50,6 +52,25 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) + def get_services_for_event(self, event): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. + Returns: + list: A list of services interested in this + event based on the service regex. + """ + # We need to know the aliases associated with this event.room_id, if any + alias_list = [] # TODO + + interested_list = [ + s for s in self.store.get_app_services() if ( + s.is_interested(event, alias_list) + ) + ] + return interested_list + def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index dd9b34937..c4e50be4c 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import re from twisted.internet import defer +from synapse.api.constants import EventTypes from synapse.api.errors import StoreError from ._base import SQLBaseStore @@ -27,6 +29,7 @@ namespace_enum = [ "rooms" # 2 ] + # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): """Defines an application service. @@ -37,9 +40,9 @@ class ApplicationService(object): def __init__(self, token, url=None, namespaces=None): self.token = token self.url = url - self.namespaces = self._get_namespaces(namespaces) + self.namespaces = self._check_namespaces(namespaces) - def _get_namespaces(self, namespaces): + def _check_namespaces(self, namespaces): # Sanity check that it is of the form: # { # users: ["regex",...], @@ -57,22 +60,50 @@ class ApplicationService(object): raise ValueError("Expected string regex for ns '%s'", ns) return namespaces - def is_interested(self, event): + def _matches_regex(self, test_string, namespace_key): + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "user_id") and + self._matches_regex(event.user_id, "users")): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex(event.state_key, "users")): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex(event.room_id, "rooms") + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, "aliases"): + return True + return False + + def is_interested(self, event, aliases_for_event=None): """Check if this service is interested in this event. Args: event(Event): The event to check. + aliases_for_event(list): A list of all the known room aliases for + this event. Returns: bool: True if this service would like to know about this event. """ - # NB: This does not check room alias regex matches because that requires - # more context that an Event can provide. Room alias matches are checked - # in the ApplicationServiceHandler. + if aliases_for_event is None: + aliases_for_event = [] - # TODO check if event.room_id regex matches - # TODO check if event.user_id regex matches (or m.room.member state_key) - - return True + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) def __str__(self): return "ApplicationService: %s" % (self.__dict__,) @@ -89,20 +120,6 @@ class ApplicationServiceCache(object): def __init__(self): self.services = [] - def get_services_for_event(self, event): - """Retrieve a list of application services interested in this event. - - Args: - event(Event): The event to check. - Returns: - list: A list of services interested in this - event based on the service regex. - """ - interested_list = [ - s for s in self.services if s.is_event_claimed(event) - ] - return interested_list - class ApplicationServiceStore(SQLBaseStore): @@ -217,10 +234,10 @@ class ApplicationServiceStore(SQLBaseStore): if res: return res[0] - def get_services_for_event(self, event): - return self.cache.get_services_for_event(event) + def get_app_services(self): + return self.cache.services - def get_app_service(self, token, from_cache=True): + def get_app_service_by_token(self, token, from_cache=True): """Get the application service with the given token. Args: From 3bd2841fdbbbc2e290d13cbd1aa9becc315d2f1c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 11:37:52 +0000 Subject: [PATCH 239/630] Everyone loves SQL typos --- synapse/storage/appservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index c4e50be4c..07ed0adcf 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -212,7 +212,7 @@ class ApplicationServiceStore(SQLBaseStore): ) # cleanup regex txn.execute( - "DELETE FROM application_services_regex WHERE id=?", + "DELETE FROM application_services_regex WHERE as_id=?", (as_id,) ) for (ns_int, ns_str) in enumerate(namespace_enum): From a060b47b13037da56ed8db2978a297133c23fc7f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 13:17:28 +0000 Subject: [PATCH 240/630] Add namespace constants. Add restrict_to option to limit namespace checks. --- synapse/handlers/appservice.py | 23 +++++++++++-------- synapse/storage/appservice.py | 41 ++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9b8dd1bb4..bf68b3339 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -52,11 +52,12 @@ class ApplicationServicesHandler(BaseHandler): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) - def get_services_for_event(self, event): + def get_services_for_event(self, event, restrict_to=""): """Retrieve a list of application services interested in this event. Args: event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. Returns: list: A list of services interested in this event based on the service regex. @@ -66,7 +67,7 @@ class ApplicationServicesHandler(BaseHandler): interested_list = [ s for s in self.store.get_app_services() if ( - s.is_interested(event, alias_list) + s.is_interested(event, restrict_to, alias_list) ) ] return interested_list @@ -80,11 +81,15 @@ class ApplicationServicesHandler(BaseHandler): Args: event(Event): The event to push out to interested services. """ - # TODO: Gather interested services - # get_services_for_event(event) <-- room IDs and user IDs - # Get a list of room aliases. Check regex. - # TODO: If unknown user: poke User Query API. - # TODO: If unknown room alias: poke Room Alias Query API. + # Gather interested services + services = self.get_services_for_event(event) + if len(services) == 0: + return # no services need notifying - # TODO: Fork off pushes to these services - XXX First cut, best effort - pass + # Do we know this user exists? If not, poke the user query API for + # all services which match that user regex. + + # Do we know this room alias exists? If not, poke the room alias query + # API for all services which match that room alias regex. + + # Fork off pushes to these services - XXX First cut, best effort diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 07ed0adcf..277741fce 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -23,12 +23,6 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) -namespace_enum = [ - "users", # 0 - "aliases", # 1 - "rooms" # 2 -] - # XXX: This feels like it should belong in a "models" module, not storage. class ApplicationService(object): @@ -36,6 +30,10 @@ class ApplicationService(object): Provides methods to check if this service is "interested" in events. """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None): self.token = token @@ -52,7 +50,7 @@ class ApplicationService(object): if not namespaces: return None - for ns in ["users", "rooms", "aliases"]: + for ns in ApplicationService.NS_LIST: if type(namespaces[ns]) != list: raise ValueError("Bad namespace value for '%s'", ns) for regex in namespaces[ns]: @@ -68,31 +66,36 @@ class ApplicationService(object): def _matches_user(self, event): if (hasattr(event, "user_id") and - self._matches_regex(event.user_id, "users")): + self._matches_regex( + event.user_id, ApplicationService.NS_USERS)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member and hasattr(event, "state_key") - and self._matches_regex(event.state_key, "users")): + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): return True return False def _matches_room_id(self, event): if hasattr(event, "room_id"): - return self._matches_regex(event.room_id, "rooms") + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) return False def _matches_aliases(self, event, alias_list): for alias in alias_list: - if self._matches_regex(alias, "aliases"): + if self._matches_regex(alias, ApplicationService.NS_ALIASES): return True return False - def is_interested(self, event, aliases_for_event=None): + def is_interested(self, event, restrict_to=None, aliases_for_event=None): """Check if this service is interested in this event. Args: event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. Returns: @@ -100,6 +103,9 @@ class ApplicationService(object): """ if aliases_for_event is None: aliases_for_event = [] + if restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so raise a general exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) return (self._matches_user(event) or self._matches_aliases(event, aliases_for_event) @@ -215,7 +221,7 @@ class ApplicationServiceStore(SQLBaseStore): "DELETE FROM application_services_regex WHERE as_id=?", (as_id,) ) - for (ns_int, ns_str) in enumerate(namespace_enum): + for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST): if ns_str in service.namespaces: for regex in service.namespaces[ns_str]: txn.execute( @@ -285,9 +291,9 @@ class ApplicationServiceStore(SQLBaseStore): "url": res["url"], "token": as_token, "namespaces": { - "users": [], - "aliases": [], - "rooms": [] + ApplicationService.NS_USERS: [], + ApplicationService.NS_ALIASES: [], + ApplicationService.NS_ROOMS: [] } } # add the namespace regex if one exists @@ -295,7 +301,8 @@ class ApplicationServiceStore(SQLBaseStore): if ns_int is None: continue try: - services[as_token]["namespaces"][namespace_enum[ns_int]].append( + services[as_token]["namespaces"][ + ApplicationService.NS_LIST[ns_int]].append( res["regex"] ) except IndexError: From fed29251d70b050ea5799708b6b13be9617d1f6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:23:58 +0000 Subject: [PATCH 241/630] Spelling --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9d7557f57..b9b2e25d1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -767,7 +767,7 @@ class FederationHandler(BaseHandler): ) ) - logger.debug("on_query_auth reutrning: %s", ret) + logger.debug("on_query_auth returning: %s", ret) defer.returnValue(ret) From f2c039bfb958ed349bce42098e296995786374cc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 13:29:27 +0000 Subject: [PATCH 242/630] Implement restricted namespace checks. Begin fleshing out the main hook for notifying application services. --- synapse/handlers/appservice.py | 19 +++++++++++++++++++ synapse/storage/appservice.py | 21 ++++++++++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index bf68b3339..dac63e224 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -17,6 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError +from synapse.storage.appservice import ApplicationService import logging @@ -88,8 +89,26 @@ class ApplicationServicesHandler(BaseHandler): # Do we know this user exists? If not, poke the user query API for # all services which match that user regex. + unknown_user = False # TODO check + if unknown_user: + user_query_services = self.get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + pass # TODO poke User Query API # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. + unknown_room_alias = False # TODO check + if unknown_room_alias: + alias_query_services = self.get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_ALIASES + ) + for alias_service in alias_query_services: + pass # TODO poke Room Alias Query API # Fork off pushes to these services - XXX First cut, best effort + for service in services: + pass # TODO push event to service diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 277741fce..cdf26ee43 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -33,6 +33,9 @@ class ApplicationService(object): NS_USERS = "users" NS_ALIASES = "aliases" NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None): @@ -103,13 +106,21 @@ class ApplicationService(object): """ if aliases_for_event is None: aliases_for_event = [] - if restrict_to not in ApplicationService.NS_LIST: - # this is a programming error, so raise a general exception + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception raise Exception("Unexpected restrict_to value: %s". restrict_to) - return (self._matches_user(event) - or self._matches_aliases(event, aliases_for_event) - or self._matches_room_id(event)) + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) def __str__(self): return "ApplicationService: %s" % (self.__dict__,) From 77a076bd25fc355c49251bcf489c0511c0f0f0af Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:35:17 +0000 Subject: [PATCH 243/630] Set combinations is | and not + --- synapse/handlers/federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b9b2e25d1..653ab0dbf 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -114,9 +114,9 @@ class FederationHandler(BaseHandler): event_ids = set() if state: - event_ids += {e.event_id for e in state} + event_ids |= {e.event_id for e in state} if auth_chain: - event_ids += {e.event_id for e in auth_chain} + event_ids |= {e.event_id for e in auth_chain} seen_ids = (yield self.store.have_events(event_ids)).keys() From 6efd4d1649a539ba4bf1c884ebb90a48c2d1c8df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 13:57:54 +0000 Subject: [PATCH 244/630] Don't completely die if get auth_chain or querying auth_chain requests fail --- synapse/handlers/federation.py | 135 ++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 63 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 653ab0dbf..6727155c3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -787,41 +787,45 @@ class FederationHandler(BaseHandler): if missing_auth: logger.debug("Missing auth: %s", missing_auth) # If we don't have all the auth events, we need to get them. - remote_auth_chain = yield self.replication_layer.get_event_auth( - origin, event.room_id, event.event_id - ) + try: + remote_auth_chain = yield self.replication_layer.get_event_auth( + origin, event.room_id, event.event_id + ) - seen_remotes = yield self.store.have_events( - [e.event_id for e in remote_auth_chain] - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in remote_auth_chain] + ) - for e in remote_auth_chain: - if e.event_id in seen_remotes.keys(): - continue + for e in remote_auth_chain: + if e.event_id in seen_remotes.keys(): + continue - if e.event_id == event.event_id: - continue + if e.event_id == event.event_id: + continue - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in remote_auth_chain - if e.event_id in auth_ids - } - e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in remote_auth_chain + if e.event_id in auth_ids + } + e.internal_metadata.outlier = True - logger.debug( - "do_auth %s missing_auth: %s", - event.event_id, e.event_id - ) - yield self._handle_new_event( - origin, e, auth_events=auth - ) + logger.debug( + "do_auth %s missing_auth: %s", + event.event_id, e.event_id + ) + yield self._handle_new_event( + origin, e, auth_events=auth + ) - if e.event_id in event_auth_events: - auth_events[(e.type, e.state_key)] = e - except AuthError: - pass + if e.event_id in event_auth_events: + auth_events[(e.type, e.state_key)] = e + except AuthError: + pass + except: + # FIXME: + logger.exception("Failed to get auth chain") # FIXME: Assumes we have and stored all the state for all the # prev_events @@ -836,47 +840,52 @@ class FederationHandler(BaseHandler): auth_ids = self.auth.compute_auth_events(event, context) local_auth_chain = yield self.store.get_auth_chain(auth_ids) - # 2. Get remote difference. - result = yield self.replication_layer.query_auth( - origin, - event.room_id, - event.event_id, - local_auth_chain, - ) + try: + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) - seen_remotes = yield self.store.have_events( - [e.event_id for e in result["auth_chain"]] - ) + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) - # 3. Process any remote auth chain events we haven't seen. - for ev in result["auth_chain"]: - if ev.event_id in seen_remotes.keys(): - continue + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue - if ev.event_id == event.event_id: - continue + if ev.event_id == event.event_id: + continue - try: - auth_ids = [e_id for e_id, _ in ev.auth_events] - auth = { - (e.type, e.state_key): e for e in result["auth_chain"] - if e.event_id in auth_ids - } - ev.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in ev.auth_events] + auth = { + (e.type, e.state_key): e for e in result["auth_chain"] + if e.event_id in auth_ids + } + ev.internal_metadata.outlier = True - logger.debug( - "do_auth %s different_auth: %s", - event.event_id, e.event_id - ) + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) - yield self._handle_new_event( - origin, ev, auth_events=auth - ) + yield self._handle_new_event( + origin, ev, auth_events=auth + ) - if ev.event_id in event_auth_events: - auth_events[(ev.type, ev.state_key)] = ev - except AuthError: - pass + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev + except AuthError: + pass + + except: + # FIXME: + logger.exception("Failed to query auth chain") # 4. Look at rejects and their proofs. # TODO. From 94a5db9f4d400a345c5d8b9f7bacb0c9ccf99959 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 3 Feb 2015 14:44:16 +0000 Subject: [PATCH 245/630] Add appservice package and move ApplicationService into it. --- synapse/appservice/__init__.py | 119 +++++++++++++++++++++++++++++++++ synapse/appservice/api.py | 15 +++++ synapse/handlers/appservice.py | 4 +- synapse/storage/appservice.py | 105 +---------------------------- 4 files changed, 138 insertions(+), 105 deletions(-) create mode 100644 synapse/appservice/__init__.py create mode 100644 synapse/appservice/api.py diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py new file mode 100644 index 000000000..f801fb532 --- /dev/null +++ b/synapse/appservice/__init__.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.api.constants import EventTypes + +import re + + +class ApplicationService(object): + """Defines an application service. This definition is mostly what is + provided to the /register AS API. + + Provides methods to check if this service is "interested" in events. + """ + NS_USERS = "users" + NS_ALIASES = "aliases" + NS_ROOMS = "rooms" + # The ordering here is important as it is used to map database values (which + # are stored as ints representing the position in this list) to namespace + # values. + NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + + def __init__(self, token, url=None, namespaces=None): + self.token = token + self.url = url + self.namespaces = self._check_namespaces(namespaces) + + def _check_namespaces(self, namespaces): + # Sanity check that it is of the form: + # { + # users: ["regex",...], + # aliases: ["regex",...], + # rooms: ["regex",...], + # } + if not namespaces: + return None + + for ns in ApplicationService.NS_LIST: + if type(namespaces[ns]) != list: + raise ValueError("Bad namespace value for '%s'", ns) + for regex in namespaces[ns]: + if not isinstance(regex, basestring): + raise ValueError("Expected string regex for ns '%s'", ns) + return namespaces + + def _matches_regex(self, test_string, namespace_key): + for regex in self.namespaces[namespace_key]: + if re.match(regex, test_string): + return True + return False + + def _matches_user(self, event): + if (hasattr(event, "user_id") and + self._matches_regex( + event.user_id, ApplicationService.NS_USERS)): + return True + # also check m.room.member state key + if (hasattr(event, "type") and event.type == EventTypes.Member + and hasattr(event, "state_key") + and self._matches_regex( + event.state_key, ApplicationService.NS_USERS)): + return True + return False + + def _matches_room_id(self, event): + if hasattr(event, "room_id"): + return self._matches_regex( + event.room_id, ApplicationService.NS_ROOMS + ) + return False + + def _matches_aliases(self, event, alias_list): + for alias in alias_list: + if self._matches_regex(alias, ApplicationService.NS_ALIASES): + return True + return False + + def is_interested(self, event, restrict_to=None, aliases_for_event=None): + """Check if this service is interested in this event. + + Args: + event(Event): The event to check. + restrict_to(str): The namespace to restrict regex tests to. + aliases_for_event(list): A list of all the known room aliases for + this event. + Returns: + bool: True if this service would like to know about this event. + """ + if aliases_for_event is None: + aliases_for_event = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: + # this is a programming error, so fail early and raise a general + # exception + raise Exception("Unexpected restrict_to value: %s". restrict_to) + + if not restrict_to: + return (self._matches_user(event) + or self._matches_aliases(event, aliases_for_event) + or self._matches_room_id(event)) + elif restrict_to == ApplicationService.NS_ALIASES: + return self._matches_aliases(event, aliases_for_event) + elif restrict_to == ApplicationService.NS_ROOMS: + return self._matches_room_id(event) + elif restrict_to == ApplicationService.NS_USERS: + return self._matches_user(event) + + def __str__(self): + return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py new file mode 100644 index 000000000..803f97ea4 --- /dev/null +++ b/synapse/appservice/api.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index dac63e224..f05b57bcb 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError -from synapse.storage.appservice import ApplicationService +from synapse.appservice import ApplicationService import logging @@ -96,6 +96,7 @@ class ApplicationServicesHandler(BaseHandler): restrict_to=ApplicationService.NS_USERS ) for user_service in user_query_services: + # this needs to block XXX: Need to feed response back to caller pass # TODO poke User Query API # Do we know this room alias exists? If not, poke the room alias query @@ -107,6 +108,7 @@ class ApplicationServicesHandler(BaseHandler): restrict_to=ApplicationService.NS_ALIASES ) for alias_service in alias_query_services: + # this needs to block XXX: Need to feed response back to caller pass # TODO poke Room Alias Query API # Fork off pushes to these services - XXX First cut, best effort diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index cdf26ee43..48bc7e0fe 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,119 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import re from twisted.internet import defer -from synapse.api.constants import EventTypes from synapse.api.errors import StoreError +from synapse.appservice import ApplicationService from ._base import SQLBaseStore logger = logging.getLogger(__name__) -# XXX: This feels like it should belong in a "models" module, not storage. -class ApplicationService(object): - """Defines an application service. - - Provides methods to check if this service is "interested" in events. - """ - NS_USERS = "users" - NS_ALIASES = "aliases" - NS_ROOMS = "rooms" - # The ordering here is important as it is used to map database values (which - # are stored as ints representing the position in this list) to namespace - # values. - NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - - def __init__(self, token, url=None, namespaces=None): - self.token = token - self.url = url - self.namespaces = self._check_namespaces(namespaces) - - def _check_namespaces(self, namespaces): - # Sanity check that it is of the form: - # { - # users: ["regex",...], - # aliases: ["regex",...], - # rooms: ["regex",...], - # } - if not namespaces: - return None - - for ns in ApplicationService.NS_LIST: - if type(namespaces[ns]) != list: - raise ValueError("Bad namespace value for '%s'", ns) - for regex in namespaces[ns]: - if not isinstance(regex, basestring): - raise ValueError("Expected string regex for ns '%s'", ns) - return namespaces - - def _matches_regex(self, test_string, namespace_key): - for regex in self.namespaces[namespace_key]: - if re.match(regex, test_string): - return True - return False - - def _matches_user(self, event): - if (hasattr(event, "user_id") and - self._matches_regex( - event.user_id, ApplicationService.NS_USERS)): - return True - # also check m.room.member state key - if (hasattr(event, "type") and event.type == EventTypes.Member - and hasattr(event, "state_key") - and self._matches_regex( - event.state_key, ApplicationService.NS_USERS)): - return True - return False - - def _matches_room_id(self, event): - if hasattr(event, "room_id"): - return self._matches_regex( - event.room_id, ApplicationService.NS_ROOMS - ) - return False - - def _matches_aliases(self, event, alias_list): - for alias in alias_list: - if self._matches_regex(alias, ApplicationService.NS_ALIASES): - return True - return False - - def is_interested(self, event, restrict_to=None, aliases_for_event=None): - """Check if this service is interested in this event. - - Args: - event(Event): The event to check. - restrict_to(str): The namespace to restrict regex tests to. - aliases_for_event(list): A list of all the known room aliases for - this event. - Returns: - bool: True if this service would like to know about this event. - """ - if aliases_for_event is None: - aliases_for_event = [] - if restrict_to and restrict_to not in ApplicationService.NS_LIST: - # this is a programming error, so fail early and raise a general - # exception - raise Exception("Unexpected restrict_to value: %s". restrict_to) - - if not restrict_to: - return (self._matches_user(event) - or self._matches_aliases(event, aliases_for_event) - or self._matches_room_id(event)) - elif restrict_to == ApplicationService.NS_ALIASES: - return self._matches_aliases(event, aliases_for_event) - elif restrict_to == ApplicationService.NS_ROOMS: - return self._matches_room_id(event) - elif restrict_to == ApplicationService.NS_USERS: - return self._matches_user(event) - - def __str__(self): - return "ApplicationService: %s" % (self.__dict__,) - - class ApplicationServiceCache(object): """Caches ApplicationServices and provides utility functions on top. From 0dd3aea319c13e66eb1d75b5b8a196032ee332b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 14:58:30 +0000 Subject: [PATCH 246/630] Keep around the old (buggy) version of the prune_event function so that we can use it to check signatures for events on old servers --- synapse/api/auth.py | 2 - synapse/events/utils.py | 79 ++++++++++++++++++++ synapse/federation/federation_client.py | 96 +------------------------ synapse/federation/federation_server.py | 52 +++----------- 4 files changed, 92 insertions(+), 137 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 37e31d2b6..3471afd7e 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -102,8 +102,6 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) - logger.debug("Got curr_state %s", curr_state) - for event in curr_state: if event.type == EventTypes.Member: try: diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 1aa952150..65a9f7098 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -94,6 +94,85 @@ def prune_event(event): ) +def old_prune_event(event): + """This is an old and buggy version of the prune event function. The + difference between this and the new version is that when including dicts + in the content they were included as frozen_dicts rather than dicts. This + caused the JSON encoder to encode as a list of the keys rather than the + dict. + """ + event_type = event.type + + allowed_keys = [ + "event_id", + "sender", + "room_id", + "hashes", + "signatures", + "content", + "type", + "state_key", + "depth", + "prev_events", + "prev_state", + "auth_events", + "origin", + "origin_server_ts", + "membership", + ] + + event_dict = event.get_dict() + + new_content = {} + + def add_fields(*fields): + for field in fields: + if field in event.content: + # This is the line that is buggy: event.content may return + # a frozen_dict which the json encoders encode as lists rather + # than dicts. + new_content[field] = event.content[field] + + if event_type == EventTypes.Member: + add_fields("membership") + elif event_type == EventTypes.Create: + add_fields("creator") + elif event_type == EventTypes.JoinRules: + add_fields("join_rule") + elif event_type == EventTypes.PowerLevels: + add_fields( + "users", + "users_default", + "events", + "events_default", + "events_default", + "state_default", + "ban", + "kick", + "redact", + ) + elif event_type == EventTypes.Aliases: + add_fields("aliases") + + allowed_fields = { + k: v + for k, v in event_dict.items() + if k in allowed_keys + } + + allowed_fields["content"] = new_content + + allowed_fields["unsigned"] = {} + + if "age_ts" in event.unsigned: + allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] + + return type(event)( + allowed_fields, + internal_metadata_dict=event.internal_metadata.get_dict() + ) + + def format_event_raw(d): return d diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 9ceb66e6f..5fac62970 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -16,17 +16,11 @@ from twisted.internet import defer +from .federation_base import FederationBase from .units import Edu from synapse.util.logutils import log_function from synapse.events import FrozenEvent -from synapse.events.utils import prune_event - -from syutil.jsonutil import encode_canonical_json - -from synapse.crypto.event_signing import check_event_content_hash - -from synapse.api.errors import SynapseError import logging @@ -34,7 +28,7 @@ import logging logger = logging.getLogger(__name__) -class FederationClient(object): +class FederationClient(FederationBase): @log_function def send_pdu(self, pdu, destinations): """Informs the replication layer about a new PDU generated within the @@ -376,89 +370,3 @@ class FederationClient(object): event.internal_metadata.outlier = outlier return event - - @defer.inlineCallbacks - def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False): - """Takes a list of PDUs and checks the signatures and hashs of each - one. If a PDU fails its signature check then we check if we have it in - the database and if not then request if from the originating server of - that PDU. - - If a PDU fails its content hash check then it is redacted. - - The given list of PDUs are not modified, instead the function returns - a new list. - - Args: - pdu (list) - outlier (bool) - - Returns: - Deferred : A list of PDUs that have valid signatures and hashes. - """ - signed_pdus = [] - for pdu in pdus: - try: - new_pdu = yield self._check_sigs_and_hash(pdu) - signed_pdus.append(new_pdu) - except SynapseError: - # FIXME: We should handle signature failures more gracefully. - - # Check local db. - new_pdu = yield self.store.get_event( - pdu.event_id, - allow_rejected=True - ) - if new_pdu: - signed_pdus.append(new_pdu) - continue - - # Check pdu.origin - if pdu.origin != origin: - new_pdu = yield self.get_pdu( - destinations=[pdu.origin], - event_id=pdu.event_id, - outlier=outlier, - ) - - if new_pdu: - signed_pdus.append(new_pdu) - continue - - logger.warn("Failed to find copy of %s with valid signature") - - defer.returnValue(signed_pdus) - - @defer.inlineCallbacks - def _check_sigs_and_hash(self, pdu): - """Throws a SynapseError if the PDU does not have the correct - signatures. - - Returns: - FrozenEvent: Either the given event or it redacted if it failed the - content hash check. - """ - # Check signatures are correct. - redacted_event = prune_event(pdu) - redacted_pdu_json = redacted_event.get_pdu_json() - - try: - yield self.keyring.verify_json_for_server( - pdu.origin, redacted_pdu_json - ) - except SynapseError: - logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), - ) - raise - - if not check_event_content_hash(pdu): - logger.warn( - "Event content has been tampered, redacting %s, %s", - pdu.event_id, encode_canonical_json(pdu.get_dict()) - ) - defer.returnValue(redacted_event) - - defer.returnValue(pdu) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 5fbd8b19d..97dca3f84 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -16,6 +16,7 @@ from twisted.internet import defer +from .federation_base import FederationBase from .units import Transaction, Edu from synapse.util.logutils import log_function @@ -35,7 +36,7 @@ import logging logger = logging.getLogger(__name__) -class FederationServer(object): +class FederationServer(FederationBase): def set_handler(self, handler): """Sets the handler that the replication layer will use to communicate receipt of new PDUs from other home servers. The required methods are @@ -251,17 +252,20 @@ class FederationServer(object): Deferred: Results in `dict` with the same format as `content` """ auth_chain = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) + self.event_from_pdu_json(e) for e in content["auth_chain"] ] - missing = [ - (yield self._check_sigs_and_hash(self.event_from_pdu_json(e))) - for e in content.get("missing", []) - ] + signed_auth = yield self._check_sigs_and_hash_and_fetch( + origin, auth_chain, outlier=True + ) ret = yield self.handler.on_query_auth( - origin, event_id, auth_chain, content.get("rejects", []), missing + origin, + event_id, + signed_auth, + content.get("rejects", []), + content.get("missing", []), ) time_now = self._clock.time_msec() @@ -426,37 +430,3 @@ class FederationServer(object): event.internal_metadata.outlier = outlier return event - - @defer.inlineCallbacks - def _check_sigs_and_hash(self, pdu): - """Throws a SynapseError if the PDU does not have the correct - signatures. - - Returns: - FrozenEvent: Either the given event or it redacted if it failed the - content hash check. - """ - # Check signatures are correct. - redacted_event = prune_event(pdu) - redacted_pdu_json = redacted_event.get_pdu_json() - - try: - yield self.keyring.verify_json_for_server( - pdu.origin, redacted_pdu_json - ) - except SynapseError: - logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), - ) - raise - - if not check_event_content_hash(pdu): - logger.warn( - "Event content has been tampered, redacting %s, %s", - pdu.event_id, encode_canonical_json(pdu.get_dict()) - ) - defer.returnValue(redacted_event) - - defer.returnValue(pdu) From 7b810e136ef2040fe55a778d4a4a790cdbd0f84c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 15:00:42 +0000 Subject: [PATCH 247/630] Add new FederationBase --- synapse/federation/federation_base.py | 126 ++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 synapse/federation/federation_base.py diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py new file mode 100644 index 000000000..d26a2396a --- /dev/null +++ b/synapse/federation/federation_base.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from synapse.events.utils import prune_event, old_prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import SynapseError + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationBase(object): + @defer.inlineCallbacks + def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False): + """Takes a list of PDUs and checks the signatures and hashs of each + one. If a PDU fails its signature check then we check if we have it in + the database and if not then request if from the originating server of + that PDU. + + If a PDU fails its content hash check then it is redacted. + + The given list of PDUs are not modified, instead the function returns + a new list. + + Args: + pdu (list) + outlier (bool) + + Returns: + Deferred : A list of PDUs that have valid signatures and hashes. + """ + signed_pdus = [] + for pdu in pdus: + try: + new_pdu = yield self._check_sigs_and_hash(pdu) + signed_pdus.append(new_pdu) + except SynapseError: + # FIXME: We should handle signature failures more gracefully. + + # Check local db. + new_pdu = yield self.store.get_event( + pdu.event_id, + allow_rejected=True + ) + if new_pdu: + signed_pdus.append(new_pdu) + continue + + # Check pdu.origin + if pdu.origin != origin: + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) + + if new_pdu: + signed_pdus.append(new_pdu) + continue + + logger.warn("Failed to find copy of %s with valid signature") + + defer.returnValue(signed_pdus) + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + old_redacted = old_prune_event(pdu) + old_redacted_pdu_json = old_redacted.get_pdu_json() + + try: + try: + yield self.keyring.verify_json_for_server( + pdu.origin, old_redacted_pdu_json + ) + except SynapseError: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) \ No newline at end of file From 8dae5c81085458a3a0b3dc92278e8c317d5de204 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 15:01:12 +0000 Subject: [PATCH 248/630] Remove unused imports --- synapse/federation/federation_server.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 97dca3f84..4742ca939 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -22,11 +22,6 @@ from .units import Transaction, Edu from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.events import FrozenEvent -from synapse.events.utils import prune_event - -from syutil.jsonutil import encode_canonical_json - -from synapse.crypto.event_signing import check_event_content_hash from synapse.api.errors import FederationError, SynapseError From 9bace3a36751deed141225ccabd5bebecebc25f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 15:32:17 +0000 Subject: [PATCH 249/630] Actually, the old prune_event function was non-deterministic, so no point keeping it around :( --- synapse/events/utils.py | 79 --------------------------- synapse/federation/federation_base.py | 16 ++---- 2 files changed, 4 insertions(+), 91 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 65a9f7098..1aa952150 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -94,85 +94,6 @@ def prune_event(event): ) -def old_prune_event(event): - """This is an old and buggy version of the prune event function. The - difference between this and the new version is that when including dicts - in the content they were included as frozen_dicts rather than dicts. This - caused the JSON encoder to encode as a list of the keys rather than the - dict. - """ - event_type = event.type - - allowed_keys = [ - "event_id", - "sender", - "room_id", - "hashes", - "signatures", - "content", - "type", - "state_key", - "depth", - "prev_events", - "prev_state", - "auth_events", - "origin", - "origin_server_ts", - "membership", - ] - - event_dict = event.get_dict() - - new_content = {} - - def add_fields(*fields): - for field in fields: - if field in event.content: - # This is the line that is buggy: event.content may return - # a frozen_dict which the json encoders encode as lists rather - # than dicts. - new_content[field] = event.content[field] - - if event_type == EventTypes.Member: - add_fields("membership") - elif event_type == EventTypes.Create: - add_fields("creator") - elif event_type == EventTypes.JoinRules: - add_fields("join_rule") - elif event_type == EventTypes.PowerLevels: - add_fields( - "users", - "users_default", - "events", - "events_default", - "events_default", - "state_default", - "ban", - "kick", - "redact", - ) - elif event_type == EventTypes.Aliases: - add_fields("aliases") - - allowed_fields = { - k: v - for k, v in event_dict.items() - if k in allowed_keys - } - - allowed_fields["content"] = new_content - - allowed_fields["unsigned"] = {} - - if "age_ts" in event.unsigned: - allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] - - return type(event)( - allowed_fields, - internal_metadata_dict=event.internal_metadata.get_dict() - ) - - def format_event_raw(d): return d diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index d26a2396a..27c5918e0 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -16,7 +16,7 @@ from twisted.internet import defer -from synapse.events.utils import prune_event, old_prune_event +from synapse.events.utils import prune_event from syutil.jsonutil import encode_canonical_json @@ -96,18 +96,10 @@ class FederationBase(object): redacted_event = prune_event(pdu) redacted_pdu_json = redacted_event.get_pdu_json() - old_redacted = old_prune_event(pdu) - old_redacted_pdu_json = old_redacted.get_pdu_json() - try: - try: - yield self.keyring.verify_json_for_server( - pdu.origin, old_redacted_pdu_json - ) - except SynapseError: - yield self.keyring.verify_json_for_server( - pdu.origin, redacted_pdu_json - ) + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) except SynapseError: logger.warn( "Signature check failed for %s redacted to %s", From 9a71add1c077130bf2cf3998ab0dd2226ba0c75b Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 3 Feb 2015 16:06:31 +0000 Subject: [PATCH 250/630] Use set_tweak instead of set_sound --- synapse/push/__init__.py | 4 ++-- synapse/push/baserules.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 28e5dae81..00f3513c2 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -400,8 +400,8 @@ def _tweaks_for_actions(actions): for a in actions: if not isinstance(a, dict): continue - if 'set_sound' in a: - tweaks['sound'] = a['set_sound'] + if 'set_tweak' in a and 'value' in a: + tweaks[a['set_tweak']] = a['value'] return tweaks diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 382de118e..376d1d4d3 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -38,7 +38,8 @@ def make_base_rules(user_name): 'actions': [ 'notify', { - 'set_sound': 'default' + 'set_tweak': 'sound', + 'value': 'default' } ] } From 7dd1c5c542d58464085b11ababe746c6a60515ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 16:12:04 +0000 Subject: [PATCH 251/630] Neaten the handling of state and auth_chain up a bit --- synapse/handlers/federation.py | 59 ++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6727155c3..86953bf8c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -30,6 +30,7 @@ from synapse.types import UserID from twisted.internet import defer +import itertools import logging @@ -112,14 +113,6 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) - event_ids = set() - if state: - event_ids |= {e.event_id for e in state} - if auth_chain: - event_ids |= {e.event_id for e in auth_chain} - - seen_ids = (yield self.store.have_events(event_ids)).keys() - # FIXME (erikj): Awful hack to make the case where we are not currently # in the room work current_state = None @@ -131,27 +124,37 @@ class FederationHandler(BaseHandler): logger.debug("Got event for room we're not in.") current_state = state - if state and auth_chain is not None: - for list_of_pdus in [auth_chain, state]: - for e in list_of_pdus: - if e.event_id in seen_ids: - continue + event_ids = set() + if state: + event_ids |= {e.event_id for e in state} + if auth_chain: + event_ids |= {e.event_id for e in auth_chain} - e.internal_metadata.outlier = True - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - yield self._handle_new_event( - origin, e, auth_events=auth - ) - except: - logger.exception( - "Failed to handle state event %s", - e.event_id, - ) + seen_ids = (yield self.store.have_events(event_ids)).keys() + + if state and auth_chain is not None: + # If we have any state or auth_chain given to us by the replication + # layer, then we should handle them (if we haven't before.) + for e in itertools.chain(auth_chain, state): + if e.event_id in seen_ids: + continue + + e.internal_metadata.outlier = True + try: + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + yield self._handle_new_event( + origin, e, auth_events=auth + ) + seen_ids.add(e.event_id) + except: + logger.exception( + "Failed to handle state event %s", + e.event_id, + ) try: yield self._handle_new_event( From 3c39f42a0526186f85e69988504fff6adbf41d91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 16:14:19 +0000 Subject: [PATCH 252/630] New line --- synapse/federation/federation_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 27c5918e0..a990aec4f 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -115,4 +115,4 @@ class FederationBase(object): ) defer.returnValue(redacted_event) - defer.returnValue(pdu) \ No newline at end of file + defer.returnValue(pdu) From dc7bb70f22edf8ef0631c961f2c77a82de7c76d5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 3 Feb 2015 16:51:07 +0000 Subject: [PATCH 253/630] s/instance_handle/profile_tag/ --- synapse/push/__init__.py | 8 ++++---- synapse/push/httppusher.py | 4 ++-- synapse/push/pusherpool.py | 12 ++++++------ synapse/rest/client/v1/push_rule.py | 28 ++++++++++++++-------------- synapse/rest/client/v1/pusher.py | 4 ++-- synapse/storage/pusher.py | 14 +++++++------- synapse/storage/schema/delta/v12.sql | 2 +- synapse/storage/schema/pusher.sql | 2 +- 8 files changed, 37 insertions(+), 37 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 00f3513c2..8c6f0a657 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -37,14 +37,14 @@ class Pusher(object): INEQUALITY_EXPR = re.compile("^([=<>]*)([0-9]*)$") - def __init__(self, _hs, instance_handle, user_name, app_id, + def __init__(self, _hs, profile_tag, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): self.hs = _hs self.evStreamHandler = self.hs.get_handlers().event_stream_handler self.store = self.hs.get_datastore() self.clock = self.hs.get_clock() - self.instance_handle = instance_handle + self.profile_tag = profile_tag self.user_name = user_name self.app_id = app_id self.app_display_name = app_display_name @@ -147,9 +147,9 @@ class Pusher(object): return False return fnmatch.fnmatch(val.upper(), pat.upper()) elif condition['kind'] == 'device': - if 'instance_handle' not in condition: + if 'profile_tag' not in condition: return True - return condition['instance_handle'] == self.instance_handle + return condition['profile_tag'] == self.profile_tag elif condition['kind'] == 'contains_display_name': # This is special because display names can be different # between rooms and so you can't really hard code it in a rule. diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 7c6953c98..5788db4eb 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -24,12 +24,12 @@ logger = logging.getLogger(__name__) class HttpPusher(Pusher): - def __init__(self, _hs, instance_handle, user_name, app_id, + def __init__(self, _hs, profile_tag, user_name, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, data, last_token, last_success, failing_since): super(HttpPusher, self).__init__( _hs, - instance_handle, + profile_tag, user_name, app_id, app_display_name, diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 4892c21e7..5a525befd 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -55,7 +55,7 @@ class PusherPool: self._start_pushers(pushers) @defer.inlineCallbacks - def add_pusher(self, user_name, instance_handle, kind, app_id, + def add_pusher(self, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -64,7 +64,7 @@ class PusherPool: self._create_pusher({ "user_name": user_name, "kind": kind, - "instance_handle": instance_handle, + "profile_tag": profile_tag, "app_id": app_id, "app_display_name": app_display_name, "device_display_name": device_display_name, @@ -77,18 +77,18 @@ class PusherPool: "failing_since": None }) yield self._add_pusher_to_store( - user_name, instance_handle, kind, app_id, + user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, instance_handle, kind, app_id, + def _add_pusher_to_store(self, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, - instance_handle=instance_handle, + profile_tag=profile_tag, kind=kind, app_id=app_id, app_display_name=app_display_name, @@ -104,7 +104,7 @@ class PusherPool: if pusherdict['kind'] == 'http': return HttpPusher( self.hs, - instance_handle=pusherdict['instance_handle'], + profile_tag=pusherdict['profile_tag'], user_name=pusherdict['user_name'], app_id=pusherdict['app_id'], app_display_name=pusherdict['app_display_name'], diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index faa7919fb..348adb9c0 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -112,7 +112,7 @@ class PushRuleRestServlet(ClientV1RestServlet): if device: conditions.append({ 'kind': 'device', - 'instance_handle': device + 'profile_tag': device }) if 'actions' not in req_obj: @@ -195,7 +195,7 @@ class PushRuleRestServlet(ClientV1RestServlet): for r in rules: conditions = json.loads(r['conditions']) - ih = _instance_handle_from_conditions(conditions) + ih = _profile_tag_from_conditions(conditions) if ih == spec['device'] and r['priority_class'] == priority_class: yield self.hs.get_datastore().delete_push_rule( user.to_string(), spec['rule_id'] @@ -239,19 +239,19 @@ class PushRuleRestServlet(ClientV1RestServlet): if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: # per-device rule - instance_handle = _instance_handle_from_conditions(r["conditions"]) + profile_tag = _profile_tag_from_conditions(r["conditions"]) r = _strip_device_condition(r) - if not instance_handle: + if not profile_tag: continue - if instance_handle not in rules['device']: - rules['device'][instance_handle] = {} - rules['device'][instance_handle] = ( + if profile_tag not in rules['device']: + rules['device'][profile_tag] = {} + rules['device'][profile_tag] = ( _add_empty_priority_class_arrays( - rules['device'][instance_handle] + rules['device'][profile_tag] ) ) - rulearray = rules['device'][instance_handle][template_name] + rulearray = rules['device'][profile_tag][template_name] else: rulearray = rules['global'][template_name] @@ -282,13 +282,13 @@ class PushRuleRestServlet(ClientV1RestServlet): if path[0] == '': defer.returnValue((200, rules['device'])) - instance_handle = path[0] + profile_tag = path[0] path = path[1:] - if instance_handle not in rules['device']: + if profile_tag not in rules['device']: ret = {} ret = _add_empty_priority_class_arrays(ret) defer.returnValue((200, ret)) - ruleset = rules['device'][instance_handle] + ruleset = rules['device'][profile_tag] result = _filter_ruleset_with_path(ruleset, path) defer.returnValue((200, result)) else: @@ -304,14 +304,14 @@ def _add_empty_priority_class_arrays(d): return d -def _instance_handle_from_conditions(conditions): +def _profile_tag_from_conditions(conditions): """ Given a list of conditions, return the instance handle of the device rule if there is one """ for c in conditions: if c['kind'] == 'device': - return c['instance_handle'] + return c['profile_tag'] return None diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 353a4a658..e10d2576d 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -41,7 +41,7 @@ class PusherRestServlet(ClientV1RestServlet): ) defer.returnValue((200, {})) - reqd = ['instance_handle', 'kind', 'app_id', 'app_display_name', + reqd = ['profile_tag', 'kind', 'app_id', 'app_display_name', 'device_display_name', 'pushkey', 'lang', 'data'] missing = [] for i in reqd: @@ -54,7 +54,7 @@ class PusherRestServlet(ClientV1RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), - instance_handle=content['instance_handle'], + profile_tag=content['profile_tag'], kind=content['kind'], app_id=content['app_id'], app_display_name=content['app_display_name'], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index f253c9e2c..e2a662a6c 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,7 +29,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): sql = ( - "SELECT id, user_name, kind, instance_handle, app_id," + "SELECT id, user_name, kind, profile_tag, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers " @@ -45,7 +45,7 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "instance_handle": r[3], + "profile_tag": r[3], "app_id": r[4], "app_display_name": r[5], "device_display_name": r[6], @@ -64,7 +64,7 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers(self): sql = ( - "SELECT id, user_name, kind, instance_handle, app_id," + "SELECT id, user_name, kind, profile_tag, app_id," "app_display_name, device_display_name, pushkey, ts, data, " "last_token, last_success, failing_since " "FROM pushers" @@ -77,7 +77,7 @@ class PusherStore(SQLBaseStore): "id": r[0], "user_name": r[1], "kind": r[2], - "instance_handle": r[3], + "profile_tag": r[3], "app_id": r[4], "app_display_name": r[5], "device_display_name": r[6], @@ -94,7 +94,7 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, instance_handle, kind, app_id, + def add_pusher(self, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: @@ -107,7 +107,7 @@ class PusherStore(SQLBaseStore): dict( user_name=user_name, kind=kind, - instance_handle=instance_handle, + profile_tag=profile_tag, app_display_name=app_display_name, device_display_name=device_display_name, ts=pushkey_ts, @@ -158,7 +158,7 @@ class PushersTable(Table): "id", "user_name", "kind", - "instance_handle", + "profile_tag", "app_id", "app_display_name", "device_display_name", diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index a6867cba6..16c2258ca 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS rejections( CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, - instance_handle varchar(32) NOT NULL, + profile_tag varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql index 8c4dfd5c1..3735b1154 100644 --- a/synapse/storage/schema/pusher.sql +++ b/synapse/storage/schema/pusher.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, - instance_handle varchar(32) NOT NULL, + profile_tag varchar(32) NOT NULL, kind varchar(8) NOT NULL, app_id varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL, From 02be8da5e11d9abcfc962f962bbc4e9940b69199 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 17:34:07 +0000 Subject: [PATCH 254/630] Add doc to get_event --- synapse/storage/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 93aefe0c4..93ab26fcd 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -131,6 +131,21 @@ class DataStore(RoomMemberStore, RoomStore, def get_event(self, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False, allow_none=False): + """Get an event from the database by event_id. + + Args: + event_id (str): The event_id of the event to fetch + check_redacted (bool): If True, check if event has been redacted + and redact it. + get_prev_content (bool): If True and event is a state event, + include the previous states content in the unsigned field. + allow_rejected (bool): If True return rejected events. + allow_none (bool): If True, return None if no event found, if + False throw an exception. + + Returns: + Deferred : A FrozenEvent. + """ event = yield self.runInteraction( "get_event", self._get_event_txn, event_id, From c0462dbf1533f285f632dcb0a74c0ef0c3e2475b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 10:16:51 +0000 Subject: [PATCH 255/630] Rearrange persist_event so that do all the queries that need to be done before returning early if we have already persisted that event. --- synapse/events/__init__.py | 2 +- synapse/handlers/federation.py | 2 + synapse/storage/__init__.py | 145 +++++++++++++++++---------------- 3 files changed, 77 insertions(+), 72 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index bf0795102..8f0c6e959 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -77,7 +77,7 @@ class EventBase(object): return self.content["membership"] def is_state(self): - return hasattr(self, "state_key") + return hasattr(self, "state_key") and self.state_key is not None def get_dict(self): d = dict(self._event_dict) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 86953bf8c..0876589e3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -515,6 +515,8 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) + destinations.remove(origin) + logger.debug( "on_send_join_request: Sending event: %s, signatures: %s", event.event_id, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 93ab26fcd..30ce37890 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -163,19 +163,70 @@ class DataStore(RoomMemberStore, RoomStore, def _persist_event_txn(self, txn, event, context, backfilled, stream_ordering=None, is_new_state=True, current_state=None): - if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) - elif event.type == EventTypes.Feedback: - self._store_feedback_txn(txn, event) - elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) - elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) - elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) + + # We purposefully do this first since if we include a `current_state` + # key, we *want* to update the `current_state_events` table + if current_state: + txn.execute( + "DELETE FROM current_state_events WHERE room_id = ?", + (event.room_id,) + ) + + for s in current_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": s.event_id, + "room_id": s.room_id, + "type": s.type, + "state_key": s.state_key, + }, + or_replace=True, + ) + + if event.is_state() and is_new_state: + if not backfilled and not context.rejected: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + or_replace=True, + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) outlier = event.internal_metadata.is_outlier() + if not outlier: + self._store_state_groups_txn(txn, event, context) + + self._update_min_depth_for_room_txn( + txn, + event.room_id, + event.depth + ) + + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + have_persisted = self._simple_select_one_onecol_txn( txn, table="event_json", @@ -209,6 +260,17 @@ class DataStore(RoomMemberStore, RoomStore, ) return + if event.type == EventTypes.Member: + self._store_room_member_txn(txn, event) + elif event.type == EventTypes.Feedback: + self._store_feedback_txn(txn, event) + elif event.type == EventTypes.Name: + self._store_room_name_txn(txn, event) + elif event.type == EventTypes.Topic: + self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Redaction: + self._store_redaction(txn, event) + event_dict = { k: v for k, v in event.get_dict().items() @@ -273,41 +335,10 @@ class DataStore(RoomMemberStore, RoomStore, ) raise _RollbackButIsFineException("_persist_event") - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - - if not outlier: - self._store_state_groups_txn(txn, event, context) - if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) - if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) - ) - - for s in current_state: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": s.event_id, - "room_id": s.room_id, - "type": s.type, - "state_key": s.state_key, - }, - or_replace=True, - ) - - is_state = hasattr(event, "state_key") and event.state_key is not None - if is_state: + if event.is_state(): vals = { "event_id": event.event_id, "room_id": event.room_id, @@ -315,6 +346,7 @@ class DataStore(RoomMemberStore, RoomStore, "state_key": event.state_key, } + # TODO: How does this work with backfilling? if hasattr(event, "replaces_state"): vals["prev_state"] = event.replaces_state @@ -351,28 +383,6 @@ class DataStore(RoomMemberStore, RoomStore, or_ignore=True, ) - if not backfilled and not context.rejected: - self._simple_insert_txn( - txn, - table="state_forward_extremities", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - or_replace=True, - ) - - for prev_state_id, _ in event.prev_state: - self._simple_delete_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "event_id": prev_state_id, - } - ) - for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( @@ -403,13 +413,6 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, ref_alg, ref_hash_bytes ) - if not outlier: - self._update_min_depth_for_room_txn( - txn, - event.room_id, - event.depth - ) - def _store_redaction(self, txn, event): txn.execute( "INSERT OR IGNORE INTO redactions " From f275ba49bbcb86e111ed0d66dd99da617220ae79 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 10:36:28 +0000 Subject: [PATCH 256/630] Fix state resolution to remember join_rules is a type of auth event. --- synapse/state.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index 8a056ee95..6a6fb8aea 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -37,7 +37,10 @@ def _get_state_key_from_event(event): KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) -AuthEventTypes = (EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,) +AuthEventTypes = ( + EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels, + EventTypes.JoinRules, +) class StateHandler(object): @@ -258,6 +261,15 @@ class StateHandler(object): auth_events.update(resolved_state) + for key, events in conflicted_state.items(): + if key[0] == EventTypes.JoinRules: + resolved_state[key] = self._resolve_auth_events( + events, + auth_events + ) + + auth_events.update(resolved_state) + for key, events in conflicted_state.items(): if key[0] == EventTypes.Member: resolved_state[key] = self._resolve_auth_events( From 03d415a6a23300e36b5e6c35080ac4dd8ab06815 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 10:40:59 +0000 Subject: [PATCH 257/630] Brief comment on why we do some things on every call to persist_event and not others --- synapse/storage/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 30ce37890..a63c59a8a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -239,6 +239,12 @@ class DataStore(RoomMemberStore, RoomStore, event.internal_metadata.get_dict() ) + # If we have already persisted this event, we don't need to do any + # more processing. + # The processing above must be done on every call to persist event, + # since they might not have happened on previous calls. For example, + # if we are persisting an event that we had persisted as an outlier, + # but is no longer one. if have_persisted: if not outlier: sql = ( From 17753f0c20d0d8190095c5a3183630b78bf9650c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 11:19:18 +0000 Subject: [PATCH 258/630] Add stub ApplicationServiceApi and glue it with the handler. --- synapse/appservice/__init__.py | 3 ++- synapse/appservice/api.py | 21 +++++++++++++++++++++ synapse/handlers/appservice.py | 18 +++++++++++++++--- synapse/storage/appservice.py | 1 + 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f801fb532..92f64619c 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -31,10 +31,11 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None): + def __init__(self, token, url=None, namespaces=None, txn_id=None): self.token = token self.url = url self.namespaces = self._check_namespaces(namespaces) + self.txn_id = None def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 803f97ea4..158aded66 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,3 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. + +class ApplicationServiceApi(object): + """This class manages HS -> AS communications, including querying and + pushing. + """ + + def __init__(self, hs): + self.hs_token = "_hs_token_" # TODO extract hs token + + def query_user(self, service, user_id): + pass + + def query_alias(self, service, alias): + pass + + def push_bulk(self, service, events): + pass + + def push(self, service, event): + pass + diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f05b57bcb..9cdeaa2d9 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -18,6 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService +from synapse.appservice.api import ApplicationServiceApi import logging @@ -29,6 +30,7 @@ class ApplicationServicesHandler(BaseHandler): def __init__(self, hs): super(ApplicationServicesHandler, self).__init__(hs) + self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks def register(self, app_service): @@ -97,7 +99,12 @@ class ApplicationServicesHandler(BaseHandler): ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke User Query API + is_known_user = self.appservice_api.query_user( + user_service, event + ) + if is_known_user: + # the user exists now,so don't query more ASes. + break # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. @@ -109,8 +116,13 @@ class ApplicationServicesHandler(BaseHandler): ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - pass # TODO poke Room Alias Query API + is_known_alias = self.appservice_api.query_alias( + alias_service, event + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + break # Fork off pushes to these services - XXX First cut, best effort for service in services: - pass # TODO push event to service + self.appservice_api.push(service, event) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 48bc7e0fe..abb617f04 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -216,6 +216,7 @@ class ApplicationServiceStore(SQLBaseStore): except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) + # TODO get last successful txn id f.e. service for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( From 525a218b2b072b24721c9c9efae42aae21388fc8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 12:24:20 +0000 Subject: [PATCH 259/630] Begin to add unit tests for appservice glue and regex testing. --- synapse/appservice/__init__.py | 14 +++++- synapse/handlers/appservice.py | 8 +++- tests/appservice/__init__.py | 14 ++++++ tests/appservice/test_appservice.py | 58 ++++++++++++++++++++++++ tests/handlers/test_appservice.py | 68 +++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 tests/appservice/__init__.py create mode 100644 tests/appservice/test_appservice.py create mode 100644 tests/handlers/test_appservice.py diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 92f64619c..0c7f58574 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -14,8 +14,11 @@ # limitations under the License. from synapse.api.constants import EventTypes +import logging import re +logger = logging.getLogger(__name__) + class ApplicationService(object): """Defines an application service. This definition is mostly what is @@ -56,15 +59,22 @@ class ApplicationService(object): return namespaces def _matches_regex(self, test_string, namespace_key): + if not isinstance(test_string, basestring): + logger.warning( + "Expected a string to test regex against, but got %s", + test_string + ) + return False + for regex in self.namespaces[namespace_key]: if re.match(regex, test_string): return True return False def _matches_user(self, event): - if (hasattr(event, "user_id") and + if (hasattr(event, "sender") and self._matches_regex( - event.user_id, ApplicationService.NS_USERS)): + event.sender, ApplicationService.NS_USERS)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9cdeaa2d9..3188c60f3 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -26,10 +26,14 @@ import logging logger = logging.getLogger(__name__) -class ApplicationServicesHandler(BaseHandler): +# NB: Purposefully not inheriting BaseHandler since that contains way too much +# setup code which this handler does not need or use. This makes testing a lot +# easier. +class ApplicationServicesHandler(object): def __init__(self, hs): - super(ApplicationServicesHandler, self).__init__(hs) + self.store = hs.get_datastore() + self.hs = hs self.appservice_api = ApplicationServiceApi(hs) @defer.inlineCallbacks diff --git a/tests/appservice/__init__.py b/tests/appservice/__init__.py new file mode 100644 index 000000000..1a84d94cd --- /dev/null +++ b/tests/appservice/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py new file mode 100644 index 000000000..5cfd26daa --- /dev/null +++ b/tests/appservice/test_appservice.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.appservice import ApplicationService + +from mock import Mock, PropertyMock +from tests import unittest + + +class ApplicationServiceTestCase(unittest.TestCase): + + def setUp(self): + self.service = ApplicationService( + url="some_url", + token="some_token", + namespaces={ + ApplicationService.NS_USERS: [], + ApplicationService.NS_ROOMS: [], + ApplicationService.NS_ALIASES: [] + } + ) + self.event = Mock( + type="m.something", room_id="!foo:bar", sender="@someone:somewhere" + ) + + def test_regex_user_id_prefix_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_user_id_prefix_no_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_room_member_is_checked(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@someone_else:matrix.org" + self.event.type = "m.room.member" + self.event.state_key = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py new file mode 100644 index 000000000..9c464e7fb --- /dev/null +++ b/tests/handlers/test_appservice.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from .. import unittest + +from synapse.handlers.appservice import ApplicationServicesHandler + +from collections import namedtuple +from mock import Mock + +# TODO: Should this be a more general thing? tests/api/test_filtering.py uses it +MockEvent = namedtuple("MockEvent", "sender type room_id") + + +class AppServiceHandlerTestCase(unittest.TestCase): + """ Tests the ApplicationServicesHandler. """ + + def setUp(self): + self.mock_store = Mock() + self.mock_as_api = Mock() + hs = Mock() + hs.get_datastore = Mock(return_value=self.mock_store) + self.handler = ApplicationServicesHandler(hs) # thing being tested + + # FIXME Would be nice to DI this rather than monkey patch:( + if not hasattr(self.handler, "appservice_api"): + # someone probably updated the handler but not the tests. Fail fast. + raise Exception("Test expected handler.appservice_api to exist.") + self.handler.appservice_api = self.mock_as_api + + def test_notify_interested_services(self): + interested_service = self._mkservice(is_interested=True) + services = [ + self._mkservice(is_interested=False), + interested_service, + self._mkservice(is_interested=False) + ] + + self.mock_store.get_app_services = Mock(return_value=services) + + event = MockEvent( + sender="@someone:anywhere", + type="m.room.message", + room_id="!foo:bar" + ) + self.mock_as_api.push = Mock() + self.handler.notify_interested_services(event) + self.mock_as_api.push.assert_called_once_with(interested_service, event) + + def _mkservice(self, is_interested): + service = Mock() + service.is_interested = Mock(return_value=is_interested) + service.token = "mock_service_token" + service.url = "mock_service_url" + return service From ff78eded015b7596e883623bf826aa579662e766 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 13:55:10 +0000 Subject: [PATCH 260/630] Retry make_join --- synapse/federation/federation_client.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 5fac62970..d6b8c4391 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -251,16 +251,21 @@ class FederationClient(FederationBase): defer.returnValue(signed_auth) @defer.inlineCallbacks - def make_join(self, destination, room_id, user_id): - ret = yield self.transport_layer.make_join( - destination, room_id, user_id - ) + def make_join(self, destinations, room_id, user_id): + for destination in destinations: + try: + ret = yield self.transport_layer.make_join( + destination, room_id, user_id + ) - pdu_dict = ret["event"] + pdu_dict = ret["event"] - logger.debug("Got response to make_join: %s", pdu_dict) + logger.debug("Got response to make_join: %s", pdu_dict) - defer.returnValue(self.event_from_pdu_json(pdu_dict)) + defer.returnValue(self.event_from_pdu_json(pdu_dict)) + break + except Exception as e: + logger.warn("Failed to make_join via %s", destination) @defer.inlineCallbacks def send_join(self, destination, pdu): From 650e32d45580ddd13826364291bda6760c014df9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 14:06:42 +0000 Subject: [PATCH 261/630] Change context.auth_events to what the auth_events would be bases on context.current_state, rather than based on the auth_events from the event. --- synapse/api/auth.py | 12 ++++++------ synapse/handlers/federation.py | 4 +++- synapse/state.py | 8 ++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3471afd7e..7105ee21d 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -358,7 +358,7 @@ class Auth(object): def add_auth_events(self, builder, context): yield run_on_reactor() - auth_ids = self.compute_auth_events(builder, context) + auth_ids = self.compute_auth_events(builder, context.current_state) auth_events_entries = yield self.store.add_event_hashes( auth_ids @@ -372,26 +372,26 @@ class Auth(object): if v.event_id in auth_ids } - def compute_auth_events(self, event, context): + def compute_auth_events(self, event, current_state): if event.type == EventTypes.Create: return [] auth_ids = [] key = (EventTypes.PowerLevels, "", ) - power_level_event = context.current_state.get(key) + power_level_event = current_state.get(key) if power_level_event: auth_ids.append(power_level_event.event_id) key = (EventTypes.JoinRules, "", ) - join_rule_event = context.current_state.get(key) + join_rule_event = current_state.get(key) key = (EventTypes.Member, event.user_id, ) - member_event = context.current_state.get(key) + member_event = current_state.get(key) key = (EventTypes.Create, "", ) - create_event = context.current_state.get(key) + create_event = current_state.get(key) if create_event: auth_ids.append(create_event.event_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0876589e3..2e2c23ef6 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -842,7 +842,9 @@ class FederationHandler(BaseHandler): logger.debug("Different auth: %s", different_auth) # 1. Get what we think is the auth chain. - auth_ids = self.auth.compute_auth_events(event, context) + auth_ids = self.auth.compute_auth_events( + event, context.current_state + ) local_auth_chain = yield self.store.get_auth_chain(auth_ids) try: diff --git a/synapse/state.py b/synapse/state.py index 6a6fb8aea..695a5e7ac 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -103,7 +103,9 @@ class StateHandler(object): context.state_group = None if hasattr(event, "auth_events") and event.auth_events: - auth_ids = zip(*event.auth_events)[0] + auth_ids = self.hs.get_auth().compute_auth_events( + event, context.current_state + ) context.auth_events = { k: v for k, v in context.current_state.items() @@ -149,7 +151,9 @@ class StateHandler(object): event.unsigned["replaces_state"] = replaces.event_id if hasattr(event, "auth_events") and event.auth_events: - auth_ids = zip(*event.auth_events)[0] + auth_ids = self.hs.get_auth().compute_auth_events( + event, context.current_state + ) context.auth_events = { k: v for k, v in context.current_state.items() From 95e2d2d36d6dfb7205c40fa8e59ef350f8096395 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 15:02:23 +0000 Subject: [PATCH 262/630] When returning lists of servers from alias lookups, put the current server first in the list --- synapse/handlers/directory.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 58e9a9156..7b6092104 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -113,7 +113,16 @@ class DirectoryHandler(BaseHandler): ) extra_servers = yield self.store.get_joined_hosts_for_room(room_id) - servers = list(set(extra_servers) | set(servers)) + servers = set(extra_servers) | set(servers) + + # If this server is in the list of servers, return it first. + if self.server_name in servers: + servers = ( + [self.server_name] + + [s for s in servers if s != self.server_name] + ) + else: + servers = list(servers) defer.returnValue({ "room_id": room_id, From 89f2e8fbdf7965d02426ef17ca6a9490219a2ec4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 15:21:03 +0000 Subject: [PATCH 263/630] Fix bug in store defer. Add more unit tests. --- synapse/storage/appservice.py | 18 +++-- tests/appservice/test_appservice.py | 87 +++++++++++++++++++++++ tests/handlers/test_appservice.py | 6 +- tests/storage/test_appservice.py | 105 ++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 9 deletions(-) create mode 100644 tests/storage/test_appservice.py diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index abb617f04..b64416de2 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -40,7 +40,7 @@ class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.cache = ApplicationServiceCache() - self._populate_cache() + self.cache_defer = self._populate_cache() @defer.inlineCallbacks def unregister_app_service(self, token): @@ -49,6 +49,7 @@ class ApplicationServiceStore(SQLBaseStore): This removes all AS specific regex and the base URL. The token is the only thing preserved for future registration attempts. """ + yield self.cache_defer # make sure the cache is ready yield self.runInteraction( "unregister_app_service", self._unregister_app_service_txn, @@ -89,9 +90,13 @@ class ApplicationServiceStore(SQLBaseStore): Args: service(ApplicationService): The updated service. """ + yield self.cache_defer # make sure the cache is ready + # NB: There is no "insert" since we provide no public-facing API to # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. + + if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") @@ -148,9 +153,12 @@ class ApplicationServiceStore(SQLBaseStore): if res: return res[0] + @defer.inlineCallbacks def get_app_services(self): - return self.cache.services + yield self.cache_defer # make sure the cache is ready + defer.returnValue(self.cache.services) + @defer.inlineCallbacks def get_app_service_by_token(self, token, from_cache=True): """Get the application service with the given token. @@ -161,12 +169,14 @@ class ApplicationServiceStore(SQLBaseStore): Raises: StoreError if there was a problem retrieving this service. """ + yield self.cache_defer # make sure the cache is ready if from_cache: for service in self.cache.services: if service.token == token: - return service - return None + defer.returnValue(service) + return + defer.returnValue(None) # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 5cfd26daa..c0aaf1278 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -56,3 +56,90 @@ class ApplicationServiceTestCase(unittest.TestCase): self.event.type = "m.room.member" self.event.state_key = "@irc_foobar:matrix.org" self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_room_id_match(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!some_prefix.*some_suffix:matrix.org" + ) + self.event.room_id = "!some_prefixs0m3th1nGsome_suffix:matrix.org" + self.assertTrue(self.service.is_interested(self.event)) + + def test_regex_room_id_no_match(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!some_prefix.*some_suffix:matrix.org" + ) + self.event.room_id = "!XqBunHwQIXUiqCaoxq:matrix.org" + self.assertFalse(self.service.is_interested(self.event)) + + def test_regex_alias_match(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.assertTrue(self.service.is_interested( + self.event, + aliases_for_event=["#irc_foobar:matrix.org", "#athing:matrix.org"] + )) + + def test_regex_alias_no_match(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.assertFalse(self.service.is_interested( + self.event, + aliases_for_event=["#xmpp_foobar:matrix.org", "#athing:matrix.org"] + )) + + def test_regex_multiple_matches(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#irc_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertTrue(self.service.is_interested( + self.event, + aliases_for_event=["#irc_barfoo:matrix.org"] + )) + + def test_restrict_to_rooms(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + "!flibble_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.event.room_id = "!wibblewoo:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_ROOMS + )) + + def test_restrict_to_aliases(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#xmpp_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@irc_foobar:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_ALIASES, + aliases_for_event=["#irc_barfoo:matrix.org"] + )) + + def test_restrict_to_senders(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + "#xmpp_.*:matrix.org" + ) + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + self.event.sender = "@xmpp_foobar:matrix.org" + self.assertFalse(self.service.is_interested( + self.event, + restrict_to=ApplicationService.NS_USERS, + aliases_for_event=["#xmpp_barfoo:matrix.org"] + )) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 9c464e7fb..1daa314f2 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -18,12 +18,8 @@ from .. import unittest from synapse.handlers.appservice import ApplicationServicesHandler -from collections import namedtuple from mock import Mock -# TODO: Should this be a more general thing? tests/api/test_filtering.py uses it -MockEvent = namedtuple("MockEvent", "sender type room_id") - class AppServiceHandlerTestCase(unittest.TestCase): """ Tests the ApplicationServicesHandler. """ @@ -51,7 +47,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): self.mock_store.get_app_services = Mock(return_value=services) - event = MockEvent( + event = Mock( sender="@someone:anywhere", type="m.room.message", room_id="!foo:bar" diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py new file mode 100644 index 000000000..56fdda377 --- /dev/null +++ b/tests/storage/test_appservice.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests import unittest +from twisted.internet import defer + +from synapse.appservice import ApplicationService +from synapse.server import HomeServer +from synapse.storage.appservice import ApplicationServiceStore + +from tests.utils import SQLiteMemoryDbPool, MockClock + + +class ApplicationServiceStoreTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer("test", db_pool=db_pool, clock=MockClock()) + self.as_token = "token1" + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", + (self.as_token,) + ) + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", ("token2",) + ) + db_pool.runQuery( + "INSERT INTO application_services(token) VALUES(?)", ("token3",) + ) + # must be done after inserts + self.store = ApplicationServiceStore(hs) + + @defer.inlineCallbacks + def test_update_and_retrieval_of_service(self): + url = "https://matrix.org/appservices/foobar" + user_regex = ["@foobar_.*:matrix.org"] + alias_regex = ["#foobar_.*:matrix.org"] + room_regex = [] + service = ApplicationService(url=url, token=self.as_token, namespaces={ + ApplicationService.NS_USERS: user_regex, + ApplicationService.NS_ALIASES: alias_regex, + ApplicationService.NS_ROOMS: room_regex + }) + yield self.store.update_app_service(service) + + stored_service = yield self.store.get_app_service_by_token( + self.as_token + ) + self.assertEquals(stored_service.token, self.as_token) + self.assertEquals(stored_service.url, url) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ALIASES], + alias_regex + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ROOMS], + room_regex + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_USERS], + user_regex + ) + + @defer.inlineCallbacks + def test_retrieve_unknown_service_token(self): + service = yield self.store.get_app_service_by_token("invalid_token") + self.assertEquals(service, None) + + @defer.inlineCallbacks + def test_retrieval_of_service(self): + stored_service = yield self.store.get_app_service_by_token( + self.as_token + ) + self.assertEquals(stored_service.token, self.as_token) + self.assertEquals(stored_service.url, None) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ALIASES], + [] + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_ROOMS], + [] + ) + self.assertEquals( + stored_service.namespaces[ApplicationService.NS_USERS], + [] + ) + + @defer.inlineCallbacks + def test_retrieval_of_all_services(self): + services = yield self.store.get_app_services() + self.assertEquals(len(services), 3) From 2e77ba637a75adcf0681e04c281f4fea74ebec6b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 Feb 2015 16:24:15 +0000 Subject: [PATCH 264/630] More s/instance_handle/profile_tag/ --- synapse/rest/client/v1/push_rule.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 348adb9c0..5582f33c8 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -73,7 +73,7 @@ class PushRuleRestServlet(ClientV1RestServlet): 'rule_id': rule_id } if device: - spec['device'] = device + spec['profile_tag'] = device return spec def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj, device=None): @@ -188,15 +188,15 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) - if 'device' in spec: + if 'profile_tag' in spec: rules = yield self.hs.get_datastore().get_push_rules_for_user_name( user.to_string() ) for r in rules: conditions = json.loads(r['conditions']) - ih = _profile_tag_from_conditions(conditions) - if ih == spec['device'] and r['priority_class'] == priority_class: + pt = _profile_tag_from_conditions(conditions) + if pt == spec['profile_tag'] and r['priority_class'] == priority_class: yield self.hs.get_datastore().delete_push_rule( user.to_string(), spec['rule_id'] ) @@ -306,7 +306,7 @@ def _add_empty_priority_class_arrays(d): def _profile_tag_from_conditions(conditions): """ - Given a list of conditions, return the instance handle of the + Given a list of conditions, return the profile tag of the device rule if there is one """ for c in conditions: From ae46f10fc5dba0f81518e3144ab8d9ed7a7d03bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 16:28:12 +0000 Subject: [PATCH 265/630] Apply sanity to the transport client interface. Convert 'make_join' and 'send_join' to accept iterables of destinations --- synapse/api/errors.py | 8 ++- synapse/federation/federation_client.py | 82 +++++++++++++++--------- synapse/federation/transaction_queue.py | 23 +++++-- synapse/federation/transport/client.py | 84 +++++++++++-------------- synapse/handlers/federation.py | 4 +- synapse/http/matrixfederationclient.py | 42 +++++++++++-- 6 files changed, 151 insertions(+), 92 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index ad478aa6b..5041828f1 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -39,7 +39,7 @@ class Codes(object): TOO_LARGE = "M_TOO_LARGE" -class CodeMessageException(Exception): +class CodeMessageException(RuntimeError): """An exception with integer code and message string attributes.""" def __init__(self, code, msg): @@ -227,3 +227,9 @@ class FederationError(RuntimeError): "affected": self.affected, "source": self.source if self.source else self.affected, } + + +class HttpResponseException(CodeMessageException): + def __init__(self, code, msg, response): + self.response = response + super(HttpResponseException, self).__init__(code, msg) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index d6b8c4391..eb36ec040 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -19,6 +19,7 @@ from twisted.internet import defer from .federation_base import FederationBase from .units import Edu +from synapse.api.errors import CodeMessageException from synapse.util.logutils import log_function from synapse.events import FrozenEvent @@ -180,7 +181,8 @@ class FederationClient(FederationBase): pdu = yield self._check_sigs_and_hash(pdu) break - + except CodeMessageException: + raise except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -264,45 +266,63 @@ class FederationClient(FederationBase): defer.returnValue(self.event_from_pdu_json(pdu_dict)) break - except Exception as e: - logger.warn("Failed to make_join via %s", destination) + except CodeMessageException: + raise + except RuntimeError as e: + logger.warn( + "Failed to make_join via %s: %s", + destination, e.message + ) + + raise RuntimeError("Failed to send to any server.") @defer.inlineCallbacks - def send_join(self, destination, pdu): - time_now = self._clock.time_msec() - _, content = yield self.transport_layer.send_join( - destination=destination, - room_id=pdu.room_id, - event_id=pdu.event_id, - content=pdu.get_pdu_json(time_now), - ) + def send_join(self, destinations, pdu): + for destination in destinations: + try: + time_now = self._clock.time_msec() + _, content = yield self.transport_layer.send_join( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) - logger.debug("Got content: %s", content) + logger.debug("Got content: %s", content) - state = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("state", []) - ] + state = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("state", []) + ] - auth_chain = [ - self.event_from_pdu_json(p, outlier=True) - for p in content.get("auth_chain", []) - ] + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("auth_chain", []) + ] - signed_state = yield self._check_sigs_and_hash_and_fetch( - destination, state, outlier=True - ) + signed_state = yield self._check_sigs_and_hash_and_fetch( + destination, state, outlier=True + ) - signed_auth = yield self._check_sigs_and_hash_and_fetch( - destination, auth_chain, outlier=True - ) + signed_auth = yield self._check_sigs_and_hash_and_fetch( + destination, auth_chain, outlier=True + ) - auth_chain.sort(key=lambda e: e.depth) + auth_chain.sort(key=lambda e: e.depth) - defer.returnValue({ - "state": signed_state, - "auth_chain": signed_auth, - }) + defer.returnValue({ + "state": signed_state, + "auth_chain": signed_auth, + }) + except CodeMessageException: + raise + except RuntimeError as e: + logger.warn( + "Failed to send_join via %s: %s", + destination, e.message + ) + + raise RuntimeError("Failed to send to any server.") @defer.inlineCallbacks def send_invite(self, destination, room_id, event_id, pdu): diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 9d4f2c09a..f38aeba7c 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -19,6 +19,7 @@ from twisted.internet import defer from .persistence import TransactionActions from .units import Transaction +from synapse.api.errors import HttpResponseException from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext @@ -238,9 +239,14 @@ class TransactionQueue(object): del p["age_ts"] return data - code, response = yield self.transport_layer.send_transaction( - transaction, json_data_cb - ) + try: + response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + code = 200 + except HttpResponseException as e: + code = e.code + response = e.response logger.info("TX [%s] got %d response", destination, code) @@ -274,8 +280,7 @@ class TransactionQueue(object): pass logger.debug("TX [%s] Yielded to callbacks", destination) - - except Exception as e: + except RuntimeError as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. logger.warn( @@ -283,6 +288,14 @@ class TransactionQueue(object): destination, e, ) + except Exception as e: + # We capture this here as there as nothing actually listens + # for this finishing functions deferred. + logger.exception( + "TX [%s] Problem in _attempt_transaction: %s", + destination, + e, + ) self.set_retrying(destination, retry_interval) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 4cb1dea2d..8b137e712 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -19,7 +19,6 @@ from synapse.api.urls import FEDERATION_PREFIX as PREFIX from synapse.util.logutils import log_function import logging -import json logger = logging.getLogger(__name__) @@ -129,7 +128,7 @@ class TransportLayerClient(object): # generated by the json_data_callback. json_data = transaction.get_dict() - code, response = yield self.client.put_json( + response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, @@ -137,95 +136,86 @@ class TransportLayerClient(object): ) logger.debug( - "send_data dest=%s, txid=%s, got response: %d", - transaction.destination, transaction.transaction_id, code + "send_data dest=%s, txid=%s, got response: 200", + transaction.destination, transaction.transaction_id, ) - defer.returnValue((code, response)) + defer.returnValue(response) @defer.inlineCallbacks @log_function def make_query(self, destination, query_type, args, retry_on_dns_fail): path = PREFIX + "/query/%s" % query_type - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, args=args, retry_on_dns_fail=retry_on_dns_fail, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def make_join(self, destination, room_id, user_id, retry_on_dns_fail=True): path = PREFIX + "/make_join/%s/%s" % (room_id, user_id) - response = yield self.client.get_json( + content = yield self.client.get_json( destination=destination, path=path, retry_on_dns_fail=retry_on_dns_fail, ) - defer.returnValue(response) + defer.returnValue(content) @defer.inlineCallbacks @log_function def send_join(self, destination, room_id, event_id, content): path = PREFIX + "/send_join/%s/%s" % (room_id, event_id) - code, content = yield self.client.put_json( + response = yield self.client.put_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_join", code) - - defer.returnValue(json.loads(content)) - - @defer.inlineCallbacks - @log_function - def send_invite(self, destination, room_id, event_id, content): - path = PREFIX + "/invite/%s/%s" % (room_id, event_id) - - code, content = yield self.client.put_json( - destination=destination, - path=path, - data=content, - ) - - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_invite", code) - - defer.returnValue(json.loads(content)) - - @defer.inlineCallbacks - @log_function - def get_event_auth(self, destination, room_id, event_id): - path = PREFIX + "/event_auth/%s/%s" % (room_id, event_id) - - response = yield self.client.get_json( - destination=destination, - path=path, - ) - defer.returnValue(response) @defer.inlineCallbacks @log_function - def send_query_auth(self, destination, room_id, event_id, content): - path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) + def send_invite(self, destination, room_id, event_id, content): + path = PREFIX + "/invite/%s/%s" % (room_id, event_id) - code, content = yield self.client.post_json( + response = yield self.client.put_json( destination=destination, path=path, data=content, ) - if not 200 <= code < 300: - raise RuntimeError("Got %d from send_invite", code) + defer.returnValue(response) - defer.returnValue(json.loads(content)) + @defer.inlineCallbacks + @log_function + def get_event_auth(self, destination, room_id, event_id): + path = PREFIX + "/event_auth/%s/%s" % (room_id, event_id) + + content = yield self.client.get_json( + destination=destination, + path=path, + ) + + defer.returnValue(content) + + @defer.inlineCallbacks + @log_function + def send_query_auth(self, destination, room_id, event_id, content): + path = PREFIX + "/query_auth/%s/%s" % (room_id, event_id) + + content = yield self.client.post_json( + destination=destination, + path=path, + data=content, + ) + + defer.returnValue(content) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0876589e3..a968a8736 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -288,7 +288,7 @@ class FederationHandler(BaseHandler): logger.debug("Joining %s to %s", joinee, room_id) pdu = yield self.replication_layer.make_join( - target_host, + [target_host], room_id, joinee ) @@ -331,7 +331,7 @@ class FederationHandler(BaseHandler): new_event = builder.build() ret = yield self.replication_layer.send_join( - target_host, + [target_host], new_event ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c7bf1b47b..8559d06b7 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -27,7 +27,9 @@ from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json -from synapse.api.errors import CodeMessageException, SynapseError, Codes +from synapse.api.errors import ( + SynapseError, Codes, HttpResponseException, +) from syutil.crypto.jsonsign import sign_json @@ -163,13 +165,12 @@ class MatrixFederationHttpClient(object): ) if 200 <= response.code < 300: - # We need to update the transactions table to say it was sent? pass else: # :'( # Update transactions table? - raise CodeMessageException( - response.code, response.phrase + raise HttpResponseException( + response.code, response.phrase, response ) defer.returnValue(response) @@ -238,11 +239,20 @@ class MatrixFederationHttpClient(object): headers_dict={"Content-Type": ["application/json"]}, ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + logger.debug("Getting resp body") body = yield readBody(response) logger.debug("Got resp body") - defer.returnValue((response.code, body)) + defer.returnValue(json.loads(body)) @defer.inlineCallbacks def post_json(self, destination, path, data={}): @@ -275,11 +285,20 @@ class MatrixFederationHttpClient(object): headers_dict={"Content-Type": ["application/json"]}, ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + logger.debug("Getting resp body") body = yield readBody(response) logger.debug("Got resp body") - defer.returnValue((response.code, body)) + defer.returnValue(json.loads(body)) @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): @@ -321,7 +340,18 @@ class MatrixFederationHttpClient(object): retry_on_dns_fail=retry_on_dns_fail ) + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + c_type = response.headers.getRawHeaders("Content-Type") + + if "application/json" not in c_type: + raise RuntimeError( + "Content-Type not application/json" + ) + + logger.debug("Getting resp body") body = yield readBody(response) + logger.debug("Got resp body") defer.returnValue(json.loads(body)) From aa8cce58bf3d3dbd1a0d512dbbd41b6545d2b1f9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 16:44:53 +0000 Subject: [PATCH 266/630] Add query_user/alias APIs. --- synapse/appservice/api.py | 47 +++++++++++++++++++++++++++++++--- synapse/handlers/appservice.py | 26 +++++++++++-------- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 158aded66..799ada96d 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -12,25 +12,64 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer +from twisted.web.client import PartialDownloadError + +from synapse.http.client import SimpleHttpClient + +import logging +import urllib + +logger = logging.getLogger(__name__) -class ApplicationServiceApi(object): +class ApplicationServiceApi(SimpleHttpClient): """This class manages HS -> AS communications, including querying and pushing. """ def __init__(self, hs): + super(ApplicationServiceApi, self).__init__(hs) self.hs_token = "_hs_token_" # TODO extract hs token + @defer.inlineCallbacks def query_user(self, service, user_id): - pass + uri = service.url + ("/users/%s" % urllib.quote(user_id)) + response = None + try: + response = yield self.get_json(uri, { + "access_token": self.hs_token + }) + if response: # just an empty json object + defer.returnValue(True) + except PartialDownloadError as e: + if e.status == 404: + defer.returnValue(False) + return + logger.warning("query_user to %s received %s", (uri, e.status)) + @defer.inlineCallbacks def query_alias(self, service, alias): - pass + uri = service.url + ("/rooms/%s" % urllib.quote(alias)) + response = None + try: + response = yield self.get_json(uri, { + "access_token": self.hs_token + }) + logger.info("%s", response[0]) + if response: # just an empty json object + defer.returnValue(True) + except PartialDownloadError as e: + if e.status == 404: + defer.returnValue(False) + return + logger.warning("query_alias to %s received %s", (uri, e.status)) def push_bulk(self, service, events): pass + @defer.inlineCallbacks def push(self, service, event): - pass + response = yield self.push_bulk(service, [event]) + defer.returnValue(response) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 3188c60f3..dd860b224 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -55,10 +55,14 @@ class ApplicationServicesHandler(object): logger.info("Updating application service info...") yield self.store.update_app_service(app_service) + logger.info("Sending ping to %s...", app_service.url) + yield self.appservice_api.query_alias(app_service, "ping") + def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) + @defer.inlineCallbacks def get_services_for_event(self, event, restrict_to=""): """Retrieve a list of application services interested in this event. @@ -71,14 +75,15 @@ class ApplicationServicesHandler(object): """ # We need to know the aliases associated with this event.room_id, if any alias_list = [] # TODO - + services = yield self.store.get_app_services() interested_list = [ - s for s in self.store.get_app_services() if ( + s for s in services if ( s.is_interested(event, restrict_to, alias_list) ) ] - return interested_list + defer.returnValue(interested_list) + @defer.inlineCallbacks def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. @@ -89,7 +94,7 @@ class ApplicationServicesHandler(object): event(Event): The event to push out to interested services. """ # Gather interested services - services = self.get_services_for_event(event) + services = yield self.get_services_for_event(event) if len(services) == 0: return # no services need notifying @@ -97,14 +102,14 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = False # TODO check if unknown_user: - user_query_services = self.get_services_for_event( + user_query_services = yield self.get_services_for_event( event=event, restrict_to=ApplicationService.NS_USERS ) for user_service in user_query_services: # this needs to block XXX: Need to feed response back to caller - is_known_user = self.appservice_api.query_user( - user_service, event + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender ) if is_known_user: # the user exists now,so don't query more ASes. @@ -114,14 +119,15 @@ class ApplicationServicesHandler(object): # API for all services which match that room alias regex. unknown_room_alias = False # TODO check if unknown_room_alias: - alias_query_services = self.get_services_for_event( + alias = "something" # TODO + alias_query_services = yield self.get_services_for_event( event=event, restrict_to=ApplicationService.NS_ALIASES ) for alias_service in alias_query_services: # this needs to block XXX: Need to feed response back to caller - is_known_alias = self.appservice_api.query_alias( - alias_service, event + is_known_alias = yield self.appservice_api.query_alias( + alias_service, alias ) if is_known_alias: # the alias exists now so don't query more ASes. From 96d4bf90120a07faa5163c2e5af542358554dd61 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:07:31 +0000 Subject: [PATCH 267/630] Modify API for SimpleHttpClient.get_json and update usages. Previously, this would only return the HTTP body as JSON, and discard other response information (e.g. the HTTP response code). This has now been changed to throw a CodeMessageException on a non-2xx response, with the response code and body, which can then be parsed as JSON. Affected modules include: - Registration/Login (when using an email for IS auth) --- synapse/handlers/login.py | 33 ++++++++++++--------- synapse/handlers/register.py | 56 +++++++++++++++++++++--------------- synapse/http/client.py | 19 ++++++------ 3 files changed, 61 insertions(+), 47 deletions(-) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index d297d71c0..744780046 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -16,12 +16,13 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import LoginError, Codes +from synapse.api.errors import LoginError, Codes, CodeMessageException from synapse.http.client import SimpleHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils import bcrypt +import json import logging logger = logging.getLogger(__name__) @@ -96,16 +97,20 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = SimpleHttpClient(self.hs) - data = yield httpCli.get_json( - # TODO FIXME This should be configurable. - # XXX: ID servers need to use HTTPS - "http://%s%s" % ( - "matrix.org:8090", "/_matrix/identity/api/v1/lookup" - ), - { - 'medium': 'email', - 'address': email - } - ) - defer.returnValue(data) + http_client = SimpleHttpClient(self.hs) + try: + data = yield http_client.get_json( + # TODO FIXME This should be configurable. + # XXX: ID servers need to use HTTPS + "http://%s%s" % ( + "matrix.org:8090", "/_matrix/identity/api/v1/lookup" + ), + { + 'medium': 'email', + 'address': email + } + ) + defer.returnValue(data) + except CodeMessageException as e: + data = json.loads(e.msg) + defer.returnValue(data) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 66a89c10b..08cd5fd72 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - SynapseError, RegistrationError, InvalidCaptchaError + SynapseError, RegistrationError, InvalidCaptchaError, CodeMessageException ) from ._base import BaseHandler import synapse.util.stringutils as stringutils @@ -28,6 +28,7 @@ from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt +import json import logging logger = logging.getLogger(__name__) @@ -161,21 +162,26 @@ class RegistrationHandler(BaseHandler): def _threepid_from_creds(self, creds): # TODO: get this from the homeserver rather than creating a new one for # each request - httpCli = SimpleHttpClient(self.hs) + http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090', 'matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) defer.returnValue(None) - data = yield httpCli.get_json( - # XXX: This should be HTTPS - "http://%s%s" % ( - creds['idServer'], - "/_matrix/identity/api/v1/3pid/getValidated3pid" - ), - {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} - ) + + data = {} + try: + data = yield http_client.get_json( + # XXX: This should be HTTPS + "http://%s%s" % ( + creds['idServer'], + "/_matrix/identity/api/v1/3pid/getValidated3pid" + ), + {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} + ) + except CodeMessageException as e: + data = json.loads(e.msg) if 'medium' in data: defer.returnValue(data) @@ -185,19 +191,23 @@ class RegistrationHandler(BaseHandler): def _bind_threepid(self, creds, mxid): yield logger.debug("binding threepid") - httpCli = SimpleHttpClient(self.hs) - data = yield httpCli.post_urlencoded_get_json( - # XXX: Change when ID servers are all HTTPS - "http://%s%s" % ( - creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" - ), - { - 'sid': creds['sid'], - 'clientSecret': creds['clientSecret'], - 'mxid': mxid, - } - ) - logger.debug("bound threepid") + http_client = SimpleHttpClient(self.hs) + data = None + try: + data = yield http_client.post_urlencoded_get_json( + # XXX: Change when ID servers are all HTTPS + "http://%s%s" % ( + creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" + ), + { + 'sid': creds['sid'], + 'clientSecret': creds['clientSecret'], + 'mxid': mxid, + } + ) + logger.debug("bound threepid") + except CodeMessageException as e: + data = json.loads(e.msg) defer.returnValue(data) @defer.inlineCallbacks diff --git a/synapse/http/client.py b/synapse/http/client.py index 198f575cf..5f4558be4 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from synapse.api.errors import CodeMessageException from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Get's some json from the given host and path + """ Gets some json from the given host and path Args: uri (str): The URI to request, not including query parameters @@ -91,15 +91,11 @@ class SimpleHttpClient(object): None. **Note**: The value of each key is assumed to be an iterable and *not* a string. - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. + Deferred: Succeeds when we get *any* 2xx HTTP response. + Raises: + On a non-2xx HTTP response. """ - - yield if len(args): query_bytes = urllib.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) @@ -114,7 +110,10 @@ class SimpleHttpClient(object): body = yield readBody(response) - defer.returnValue(json.loads(body)) + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + raise CodeMessageException(response.code, body) class CaptchaServerHttpClient(SimpleHttpClient): From 6d3e4f4d0aad4ad9b44b28349838ff48aef39440 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:32:44 +0000 Subject: [PATCH 268/630] Update user/alias query APIs to use new format of SimpleHttpClient.get_json --- synapse/appservice/api.py | 15 +++++++-------- synapse/http/client.py | 3 +++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 799ada96d..74508ecdd 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from twisted.internet import defer -from twisted.web.client import PartialDownloadError +from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient import logging @@ -42,11 +42,11 @@ class ApplicationServiceApi(SimpleHttpClient): }) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_user to %s received %s", (uri, e.status)) + logger.warning("query_user to %s received %s", uri, e.code) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -56,14 +56,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": self.hs_token }) - logger.info("%s", response[0]) if response: # just an empty json object defer.returnValue(True) - except PartialDownloadError as e: - if e.status == 404: + except CodeMessageException as e: + if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", (uri, e.status)) + logger.warning("query_alias to %s received %s", uri, e.code) def push_bulk(self, service, events): pass diff --git a/synapse/http/client.py b/synapse/http/client.py index 5f4558be4..fee8c901a 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -113,6 +113,9 @@ class SimpleHttpClient(object): if 200 <= response.code < 300: defer.returnValue(json.loads(body)) else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. raise CodeMessageException(response.code, body) From 6de799422db434bab4c687cbb465cfb730601d86 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 17:39:38 +0000 Subject: [PATCH 269/630] Mention new pydenticon dep. --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 297ae914f..922fa5b03 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +Changes in develop +================== + + * pydenticon support -- adds dep on pydenticon + Changes in synapse 0.6.1 (2015-01-07) ===================================== From 543e84fe70bdc0af6be5feb237de2d18adf352ab Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 4 Feb 2015 17:39:51 +0000 Subject: [PATCH 270/630] Add SimpleHttpClient.put_json with the same semantics as get_json. --- synapse/http/client.py | 48 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index fee8c901a..575510637 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -83,7 +83,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def get_json(self, uri, args={}): - """ Gets some json from the given host and path + """ Gets some json from the given URI. Args: uri (str): The URI to request, not including query parameters @@ -92,7 +92,8 @@ class SimpleHttpClient(object): **Note**: The value of each key is assumed to be an iterable and *not* a string. Returns: - Deferred: Succeeds when we get *any* 2xx HTTP response. + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. Raises: On a non-2xx HTTP response. """ @@ -118,6 +119,49 @@ class SimpleHttpClient(object): # load it into JSON if they want. raise CodeMessageException(response.code, body) + @defer.inlineCallbacks + def put_json(self, uri, json_body, args={}): + """ Puts some json to the given URI. + + Args: + uri (str): The URI to request, not including query parameters + json_body (dict): The JSON to put in the HTTP body, + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + Returns: + Deferred: Succeeds when we get *any* 2xx HTTP response, with the + HTTP body as JSON. + Raises: + On a non-2xx HTTP response. + """ + if len(args): + query_bytes = urllib.urlencode(args, True) + uri = "%s?%s" % (uri, query_bytes) + + json_str = json.dumps(json_body) + + response = yield self.agent.request( + "PUT", + uri.encode("ascii"), + headers=Headers({ + b"User-Agent": [AGENT_NAME], + "Content-Type": ["application/json"] + }), + bodyProducer=FileBodyProducer(StringIO(json_str)) + ) + + body = yield readBody(response) + + if 200 <= response.code < 300: + defer.returnValue(json.loads(body)) + else: + # NB: This is explicitly not json.loads(body)'d because the contract + # of CodeMessageException is a *string* message. Callers can always + # load it into JSON if they want. + raise CodeMessageException(response.code, body) + class CaptchaServerHttpClient(SimpleHttpClient): """ From f292ad4b2bc8fbd3d86f26236714cff53c47e9c2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 18:09:18 +0000 Subject: [PATCH 271/630] Add script to check and auth chain and current state of a room --- scripts/check_auth.py | 65 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 scripts/check_auth.py diff --git a/scripts/check_auth.py b/scripts/check_auth.py new file mode 100644 index 000000000..341f00e71 --- /dev/null +++ b/scripts/check_auth.py @@ -0,0 +1,65 @@ +from synapse.events import FrozenEvent +from synapse.api.auth import Auth + +from mock import Mock + +import argparse +import itertools +import json +import sys + + + +def check_auth(auth, auth_chain, events): + auth_chain.sort(key=lambda e: e.depth) + + auth_map = { + e.event_id: e + for e in auth_chain + } + + create_events = {} + for e in auth_chain: + if e.type == "m.room.create": + create_events[e.room_id] = e + + for e in itertools.chain(auth_chain, events): + auth_events_list = [auth_map[i] for i, _ in e.auth_events] + + auth_events = { + (e.type, e.state_key): e + for e in auth_events_list + } + + auth_events[("m.room.create", "")] = create_events[e.room_id] + + try: + auth.check(e, auth_events=auth_events) + except Exception as ex: + print "Failed:", e.event_id, e.type, e.state_key + print ex + print json.dumps(e.get_dict(), sort_keys=True, indent=4) + # raise + print "Success:", e.event_id, e.type, e.state_key + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument( + 'json', + nargs='?', + type=argparse.FileType('r'), + default=sys.stdin, + ) + + args = parser.parse_args() + + js = json.load(args.json) + + + auth = Auth(Mock()) + check_auth( + auth, + [FrozenEvent(d) for d in js["auth_chain"]], + [FrozenEvent(d) for d in js["pdus"]], + ) From 3b5e8125ebf435ad33012489bdd225285bfbc1d2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 23:48:32 +0000 Subject: [PATCH 272/630] Bluntly port changes of README from develop to master --- README.rst | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index ccbee989c..e1fa4d75f 100644 --- a/README.rst +++ b/README.rst @@ -111,6 +111,15 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into a virtual environment under ``~/.synapse``. +To set up your homeserver, run (in your virtualenv, as before):: + + $ python -m synapse.app.homeserver \ + --server-name machine.my.domain.name \ + --config-path homeserver.yaml \ + --generate-config + +Substituting your host and domain name as appropriate. + For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. @@ -213,7 +222,7 @@ to install using pip and a virtualenv:: $ virtualenv env $ source env/bin/activate - $ python synapse/dependencies | xargs -i pip install + $ python synapse/python_dependencies.py | xargs -n1 pip install $ pip install setuptools_trial mock This will run a process of downloading and installing all the needed @@ -265,9 +274,9 @@ For the first form, simply pass the required hostname (of the machine) as the $ python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ - --config-path homeserver.config \ + --config-path homeserver.yaml \ --generate-config - $ python -m synapse.app.homeserver --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.yaml Alternatively, you can run ``synctl start`` to guide you through the process. @@ -287,9 +296,9 @@ SRV record, as that is the name other machines will expect it to have:: $ python -m synapse.app.homeserver \ --server-name YOURDOMAIN \ --bind-port 8448 \ - --config-path homeserver.config \ + --config-path homeserver.yaml \ --generate-config - $ python -m synapse.app.homeserver --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.yaml You may additionally want to pass one or more "-v" options, in order to From 47b1e1491f7f98b57f4b6894f0d4d333e64bf721 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 23:49:40 +0000 Subject: [PATCH 273/630] Pull in python_dependencies.py from develop --- synapse/python_dependencies.py | 122 +++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 synapse/python_dependencies.py diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py new file mode 100644 index 000000000..a89d61860 --- /dev/null +++ b/synapse/python_dependencies.py @@ -0,0 +1,122 @@ +import logging +from distutils.version import LooseVersion + +logger = logging.getLogger(__name__) + +REQUIREMENTS = { + "syutil==0.0.2": ["syutil"], + "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], + "Twisted==14.0.2": ["twisted==14.0.2"], + "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "pyopenssl>=0.14": ["OpenSSL>=0.14"], + "pyyaml": ["yaml"], + "pyasn1": ["pyasn1"], + "pynacl": ["nacl"], + "daemonize": ["daemonize"], + "py-bcrypt": ["bcrypt"], + "frozendict>=0.4": ["frozendict"], + "pillow": ["PIL"], + "pydenticon": ["pydenticon"], +} + +def github_link(project, version, egg): + return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) + +DEPENDENCY_LINKS=[ + github_link( + project="matrix-org/syutil", + version="v0.0.2", + egg="syutil-0.0.2", + ), + github_link( + project="matrix-org/matrix-angular-sdk", + version="v0.6.0", + egg="matrix_angular_sdk-0.6.0", + ), + github_link( + project="pyca/pynacl", + version="d4d3175589b892f6ea7c22f466e0e223853516fa", + egg="pynacl-0.3.0", + ) +] + + +class MissingRequirementError(Exception): + pass + + +def check_requirements(): + """Checks that all the modules needed by synapse have been correctly + installed and are at the correct version""" + for dependency, module_requirements in REQUIREMENTS.items(): + for module_requirement in module_requirements: + if ">=" in module_requirement: + module_name, required_version = module_requirement.split(">=") + version_test = ">=" + elif "==" in module_requirement: + module_name, required_version = module_requirement.split("==") + version_test = "==" + else: + module_name = module_requirement + version_test = None + + try: + module = __import__(module_name) + except ImportError: + logging.exception( + "Can't import %r which is part of %r", + module_name, dependency + ) + raise MissingRequirementError( + "Can't import %r which is part of %r" + % (module_name, dependency) + ) + version = getattr(module, "__version__", None) + file_path = getattr(module, "__file__", None) + logger.info( + "Using %r version %r from %r to satisfy %r", + module_name, version, file_path, dependency + ) + + if version_test == ">=": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) < LooseVersion(required_version): + raise MissingRequirementError( + "Version of %r in %r is too old. %r < %r" + % (dependency, file_path, version, required_version) + ) + elif version_test == "==": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) != LooseVersion(required_version): + raise MissingRequirementError( + "Unexpected version of %r in %r. %r != %r" + % (dependency, file_path, version, required_version) + ) + +def list_requirements(): + result = [] + linked = [] + for link in DEPENDENCY_LINKS: + egg = link.split("#egg=")[1] + linked.append(egg.split('-')[0]) + result.append(link) + for requirement in REQUIREMENTS: + is_linked = False + for link in linked: + if requirement.replace('-','_').startswith(link): + is_linked = True + if not is_linked: + result.append(requirement) + return result + +if __name__ == "__main__": + import sys + sys.stdout.writelines(req + "\n" for req in list_requirements()) From 92c43e4a0ef17ae7b3c9bda88143a7351958972a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 23:50:25 +0000 Subject: [PATCH 274/630] Revert "Pull in python_dependencies.py from develop" This reverts commit 47b1e1491f7f98b57f4b6894f0d4d333e64bf721. --- synapse/python_dependencies.py | 122 --------------------------------- 1 file changed, 122 deletions(-) delete mode 100644 synapse/python_dependencies.py diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py deleted file mode 100644 index a89d61860..000000000 --- a/synapse/python_dependencies.py +++ /dev/null @@ -1,122 +0,0 @@ -import logging -from distutils.version import LooseVersion - -logger = logging.getLogger(__name__) - -REQUIREMENTS = { - "syutil==0.0.2": ["syutil"], - "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.2": ["twisted==14.0.2"], - "service_identity>=1.0.0": ["service_identity>=1.0.0"], - "pyopenssl>=0.14": ["OpenSSL>=0.14"], - "pyyaml": ["yaml"], - "pyasn1": ["pyasn1"], - "pynacl": ["nacl"], - "daemonize": ["daemonize"], - "py-bcrypt": ["bcrypt"], - "frozendict>=0.4": ["frozendict"], - "pillow": ["PIL"], - "pydenticon": ["pydenticon"], -} - -def github_link(project, version, egg): - return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) - -DEPENDENCY_LINKS=[ - github_link( - project="matrix-org/syutil", - version="v0.0.2", - egg="syutil-0.0.2", - ), - github_link( - project="matrix-org/matrix-angular-sdk", - version="v0.6.0", - egg="matrix_angular_sdk-0.6.0", - ), - github_link( - project="pyca/pynacl", - version="d4d3175589b892f6ea7c22f466e0e223853516fa", - egg="pynacl-0.3.0", - ) -] - - -class MissingRequirementError(Exception): - pass - - -def check_requirements(): - """Checks that all the modules needed by synapse have been correctly - installed and are at the correct version""" - for dependency, module_requirements in REQUIREMENTS.items(): - for module_requirement in module_requirements: - if ">=" in module_requirement: - module_name, required_version = module_requirement.split(">=") - version_test = ">=" - elif "==" in module_requirement: - module_name, required_version = module_requirement.split("==") - version_test = "==" - else: - module_name = module_requirement - version_test = None - - try: - module = __import__(module_name) - except ImportError: - logging.exception( - "Can't import %r which is part of %r", - module_name, dependency - ) - raise MissingRequirementError( - "Can't import %r which is part of %r" - % (module_name, dependency) - ) - version = getattr(module, "__version__", None) - file_path = getattr(module, "__file__", None) - logger.info( - "Using %r version %r from %r to satisfy %r", - module_name, version, file_path, dependency - ) - - if version_test == ">=": - if version is None: - raise MissingRequirementError( - "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name) - ) - if LooseVersion(version) < LooseVersion(required_version): - raise MissingRequirementError( - "Version of %r in %r is too old. %r < %r" - % (dependency, file_path, version, required_version) - ) - elif version_test == "==": - if version is None: - raise MissingRequirementError( - "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name) - ) - if LooseVersion(version) != LooseVersion(required_version): - raise MissingRequirementError( - "Unexpected version of %r in %r. %r != %r" - % (dependency, file_path, version, required_version) - ) - -def list_requirements(): - result = [] - linked = [] - for link in DEPENDENCY_LINKS: - egg = link.split("#egg=")[1] - linked.append(egg.split('-')[0]) - result.append(link) - for requirement in REQUIREMENTS: - is_linked = False - for link in linked: - if requirement.replace('-','_').startswith(link): - is_linked = True - if not is_linked: - result.append(requirement) - return result - -if __name__ == "__main__": - import sys - sys.stdout.writelines(req + "\n" for req in list_requirements()) From 19ebdc321d07e7f4a2946a284885e45e5434cb44 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 23:51:02 +0000 Subject: [PATCH 275/630] Pull in python_dependencies.py from develop --- synapse/python_dependencies.py | 122 +++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 synapse/python_dependencies.py diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py new file mode 100644 index 000000000..a89d61860 --- /dev/null +++ b/synapse/python_dependencies.py @@ -0,0 +1,122 @@ +import logging +from distutils.version import LooseVersion + +logger = logging.getLogger(__name__) + +REQUIREMENTS = { + "syutil==0.0.2": ["syutil"], + "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], + "Twisted==14.0.2": ["twisted==14.0.2"], + "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "pyopenssl>=0.14": ["OpenSSL>=0.14"], + "pyyaml": ["yaml"], + "pyasn1": ["pyasn1"], + "pynacl": ["nacl"], + "daemonize": ["daemonize"], + "py-bcrypt": ["bcrypt"], + "frozendict>=0.4": ["frozendict"], + "pillow": ["PIL"], + "pydenticon": ["pydenticon"], +} + +def github_link(project, version, egg): + return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) + +DEPENDENCY_LINKS=[ + github_link( + project="matrix-org/syutil", + version="v0.0.2", + egg="syutil-0.0.2", + ), + github_link( + project="matrix-org/matrix-angular-sdk", + version="v0.6.0", + egg="matrix_angular_sdk-0.6.0", + ), + github_link( + project="pyca/pynacl", + version="d4d3175589b892f6ea7c22f466e0e223853516fa", + egg="pynacl-0.3.0", + ) +] + + +class MissingRequirementError(Exception): + pass + + +def check_requirements(): + """Checks that all the modules needed by synapse have been correctly + installed and are at the correct version""" + for dependency, module_requirements in REQUIREMENTS.items(): + for module_requirement in module_requirements: + if ">=" in module_requirement: + module_name, required_version = module_requirement.split(">=") + version_test = ">=" + elif "==" in module_requirement: + module_name, required_version = module_requirement.split("==") + version_test = "==" + else: + module_name = module_requirement + version_test = None + + try: + module = __import__(module_name) + except ImportError: + logging.exception( + "Can't import %r which is part of %r", + module_name, dependency + ) + raise MissingRequirementError( + "Can't import %r which is part of %r" + % (module_name, dependency) + ) + version = getattr(module, "__version__", None) + file_path = getattr(module, "__file__", None) + logger.info( + "Using %r version %r from %r to satisfy %r", + module_name, version, file_path, dependency + ) + + if version_test == ">=": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) < LooseVersion(required_version): + raise MissingRequirementError( + "Version of %r in %r is too old. %r < %r" + % (dependency, file_path, version, required_version) + ) + elif version_test == "==": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) != LooseVersion(required_version): + raise MissingRequirementError( + "Unexpected version of %r in %r. %r != %r" + % (dependency, file_path, version, required_version) + ) + +def list_requirements(): + result = [] + linked = [] + for link in DEPENDENCY_LINKS: + egg = link.split("#egg=")[1] + linked.append(egg.split('-')[0]) + result.append(link) + for requirement in REQUIREMENTS: + is_linked = False + for link in linked: + if requirement.replace('-','_').startswith(link): + is_linked = True + if not is_linked: + result.append(requirement) + return result + +if __name__ == "__main__": + import sys + sys.stdout.writelines(req + "\n" for req in list_requirements()) From 77e5ae22a99e6668e76b530735e594509dcc3ea7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 23:51:34 +0000 Subject: [PATCH 276/630] Ver bump --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 1c29ff4d3..1648b8029 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1d +0.6.1e diff --git a/synapse/__init__.py b/synapse/__init__.py index dacda8986..d3f1c33d3 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1d" +__version__ = "0.6.1e" From 559a26b025133bd36016ab8314f8d92b52d20e25 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 Feb 2015 23:54:53 +0000 Subject: [PATCH 277/630] Pin the twisted version so that it doesn't pull in twisted 15. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bde26c0b9..28e8188f9 100755 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ setup( "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.1/#egg=matrix_angular_sdk-0.6.1", ], setup_requires=[ - "Twisted==14.0.2", + "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 "setuptools_trial", "setuptools>=1.0.0", # Needs setuptools that supports git+ssh. # TODO: Do we need this now? we don't use git+ssh. From a1a4960baf78c1a24f76c603076c832a1947360f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 09:43:22 +0000 Subject: [PATCH 278/630] Impl push_bulk function --- synapse/appservice/api.py | 19 ++++++++++++++++++- synapse/handlers/appservice.py | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 74508ecdd..fbf4abc52 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -64,8 +64,25 @@ class ApplicationServiceApi(SimpleHttpClient): return logger.warning("query_alias to %s received %s", uri, e.code) + @defer.inlineCallbacks def push_bulk(self, service, events): - pass + uri = service.url + ("/transactions/%s" % + urllib.quote(str(0))) # TODO txn_ids + response = None + try: + response = yield self.put_json( + uri, + { + "events": events + }, + { + "access_token": self.hs_token + }) + if response: # just an empty json object + defer.returnValue(True) + except CodeMessageException as e: + logger.warning("push_bulk to %s received %s", uri, e.code) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index dd860b224..2b2761682 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -56,7 +56,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.query_alias(app_service, "ping") + yield self.appservice_api.push(app_service, "pinger") def unregister(self, token): logger.info("Unregister as_token=%s", token) From 27091f146a0ebdbfe1ae7c5cd30de51515cfbebc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 10:08:12 +0000 Subject: [PATCH 279/630] Add hs_token column and generate a different token f.e application service. --- synapse/appservice/__init__.py | 6 ++++-- synapse/appservice/api.py | 8 ++++---- synapse/handlers/appservice.py | 9 ++++++--- synapse/rest/appservice/v1/register.py | 4 ++-- synapse/storage/appservice.py | 17 ++++++++++++----- synapse/storage/schema/application_services.sql | 1 + tests/storage/test_appservice.py | 10 ++++++---- 7 files changed, 35 insertions(+), 20 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 0c7f58574..f7baf578f 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -34,11 +34,13 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - def __init__(self, token, url=None, namespaces=None, txn_id=None): + def __init__(self, token, url=None, namespaces=None, hs_token=None, + txn_id=None): self.token = token self.url = url + self.hs_token = hs_token self.namespaces = self._check_namespaces(namespaces) - self.txn_id = None + self.txn_id = txn_id def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index fbf4abc52..29bb35d61 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -30,7 +30,6 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) - self.hs_token = "_hs_token_" # TODO extract hs token @defer.inlineCallbacks def query_user(self, service, user_id): @@ -38,7 +37,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -54,7 +53,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.get_json(uri, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object defer.returnValue(True) @@ -76,9 +75,10 @@ class ApplicationServiceApi(SimpleHttpClient): "events": events }, { - "access_token": self.hs_token + "access_token": service.hs_token }) if response: # just an empty json object + # TODO: Mark txn as sent successfully defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2b2761682..7b0599c71 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -19,6 +19,7 @@ from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi +import synapse.util.stringutils as stringutils import logging @@ -53,10 +54,9 @@ class ApplicationServicesHandler(object): errcode=Codes.FORBIDDEN ) logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() yield self.store.update_app_service(app_service) - - logger.info("Sending ping to %s...", app_service.url) - yield self.appservice_api.push(app_service, "pinger") + defer.returnValue(app_service) def unregister(self, token): logger.info("Unregister as_token=%s", token) @@ -136,3 +136,6 @@ class ApplicationServicesHandler(object): # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + + def _generate_hs_token(self): + return stringutils.random_string(18) diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index e374d538e..d3d5aef22 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -61,8 +61,8 @@ class RegisterRestServlet(AppServiceRestServlet): app_service = ApplicationService(as_token, as_url, namespaces) - yield self.handler.register(app_service) - hs_token = "_not_implemented_yet" # TODO: Pull this from self.hs? + app_service = yield self.handler.register(app_service) + hs_token = app_service.hs_token defer.returnValue((200, { "hs_token": hs_token diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index b64416de2..3c8bf9ad0 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -60,6 +60,7 @@ class ApplicationServiceStore(SQLBaseStore): if service.token == token: service.url = None service.namespaces = None + service.hs_token = None def _unregister_app_service_txn(self, txn, token): # kill the url to prevent pushes @@ -100,6 +101,9 @@ class ApplicationServiceStore(SQLBaseStore): if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") + if not service.hs_token: + raise StoreError(500, "No HS token") + yield self.runInteraction( "update_app_service", self._update_app_service_txn, @@ -126,8 +130,8 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=? WHERE id=?", - (service.url, as_id,) + "UPDATE application_services SET url=?, hs_token=? WHERE id=?", + (service.url, service.hs_token, as_id,) ) # cleanup regex txn.execute( @@ -196,6 +200,7 @@ class ApplicationServiceStore(SQLBaseStore): # 'namespace': enum, # 'as_id': 0, # 'token': "something", + # 'hs_token': "otherthing", # 'id': 0 # } # ] @@ -208,6 +213,7 @@ class ApplicationServiceStore(SQLBaseStore): services[as_token] = { "url": res["url"], "token": as_token, + "hs_token": res["hs_token"], "namespaces": { ApplicationService.NS_USERS: [], ApplicationService.NS_ALIASES: [], @@ -230,8 +236,9 @@ class ApplicationServiceStore(SQLBaseStore): for service in services.values(): logger.info("Found application service: %s", service) self.cache.services.append(ApplicationService( - service["token"], - service["url"], - service["namespaces"] + token=service["token"], + url=service["url"], + namespaces=service["namespaces"], + hs_token=service["hs_token"] )) diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 6d245fc80..03b5a10c8 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS application_services( id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT, token TEXT, + hs_token TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 56fdda377..b9ecfb338 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -46,13 +46,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_update_and_retrieval_of_service(self): url = "https://matrix.org/appservices/foobar" + hs_token = "hstok" user_regex = ["@foobar_.*:matrix.org"] alias_regex = ["#foobar_.*:matrix.org"] room_regex = [] - service = ApplicationService(url=url, token=self.as_token, namespaces={ - ApplicationService.NS_USERS: user_regex, - ApplicationService.NS_ALIASES: alias_regex, - ApplicationService.NS_ROOMS: room_regex + service = ApplicationService( + url=url, hs_token=hs_token, token=self.as_token, namespaces={ + ApplicationService.NS_USERS: user_regex, + ApplicationService.NS_ALIASES: alias_regex, + ApplicationService.NS_ROOMS: room_regex }) yield self.store.update_app_service(service) From 6a7e168009b6631fb7deb6bac5351085e993e620 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 11:25:20 +0000 Subject: [PATCH 280/630] Print out the auth events on failure --- scripts/check_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_auth.py b/scripts/check_auth.py index 341f00e71..b889ac7fa 100644 --- a/scripts/check_auth.py +++ b/scripts/check_auth.py @@ -9,7 +9,6 @@ import json import sys - def check_auth(auth, auth_chain, events): auth_chain.sort(key=lambda e: e.depth) @@ -37,6 +36,7 @@ def check_auth(auth, auth_chain, events): auth.check(e, auth_events=auth_events) except Exception as ex: print "Failed:", e.event_id, e.type, e.state_key + print "Auth_events:", auth_events print ex print json.dumps(e.get_dict(), sort_keys=True, indent=4) # raise From f0c730252fcb198ad925d6406f0f37ccee29e720 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 11:25:32 +0000 Subject: [PATCH 281/630] Add unknown user ID check. Use store.get_aliases_for_room(room_id) when searching for services by alias. --- synapse/handlers/appservice.py | 24 +++++++++++++++++++----- tests/handlers/test_appservice.py | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 7b0599c71..39976b962 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,10 +15,10 @@ from twisted.internet import defer -from ._base import BaseHandler from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi +from synapse.types import UserID import synapse.util.stringutils as stringutils import logging @@ -74,7 +74,7 @@ class ApplicationServicesHandler(object): event based on the service regex. """ # We need to know the aliases associated with this event.room_id, if any - alias_list = [] # TODO + alias_list = yield self.store.get_aliases_for_room(event.room_id) services = yield self.store.get_app_services() interested_list = [ s for s in services if ( @@ -100,7 +100,7 @@ class ApplicationServicesHandler(object): # Do we know this user exists? If not, poke the user query API for # all services which match that user regex. - unknown_user = False # TODO check + unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: user_query_services = yield self.get_services_for_event( event=event, @@ -117,7 +117,7 @@ class ApplicationServicesHandler(object): # Do we know this room alias exists? If not, poke the room alias query # API for all services which match that room alias regex. - unknown_room_alias = False # TODO check + unknown_room_alias = False # TODO if unknown_room_alias: alias = "something" # TODO alias_query_services = yield self.get_services_for_event( @@ -137,5 +137,19 @@ class ApplicationServicesHandler(object): for service in services: self.appservice_api.push(service, event) + @defer.inlineCallbacks + def _is_unknown_user(self, user_id): + user = UserID.from_string(user_id) + if not self.hs.is_mine(user): + # we don't know if they are unknown or not since it isn't one of our + # users. We can't poke ASes. + defer.returnValue(False) + return + + user_info = yield self.store.get_user_by_id(user_id) + defer.returnValue(len(user_info) == 0) + + + def _generate_hs_token(self): - return stringutils.random_string(18) + return stringutils.random_string(24) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 1daa314f2..f92dc1c89 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -46,6 +46,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): ] self.mock_store.get_app_services = Mock(return_value=services) + self.mock_store.get_user_by_id = Mock(return_value=[]) event = Mock( sender="@someone:anywhere", From b932600653a4585968a7d177b7ea7cb2ca33642a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 11:47:11 +0000 Subject: [PATCH 282/630] Add unknown room alias check. Call it from directory_handler.get_association --- synapse/handlers/appservice.py | 97 +++++++++++++++++++--------------- synapse/handlers/directory.py | 7 +++ 2 files changed, 61 insertions(+), 43 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 39976b962..2c6d4e281 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -62,27 +62,6 @@ class ApplicationServicesHandler(object): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) - @defer.inlineCallbacks - def get_services_for_event(self, event, restrict_to=""): - """Retrieve a list of application services interested in this event. - - Args: - event(Event): The event to check. - restrict_to(str): The namespace to restrict regex tests to. - Returns: - list: A list of services interested in this - event based on the service regex. - """ - # We need to know the aliases associated with this event.room_id, if any - alias_list = yield self.store.get_aliases_for_room(event.room_id) - services = yield self.store.get_app_services() - interested_list = [ - s for s in services if ( - s.is_interested(event, restrict_to, alias_list) - ) - ] - defer.returnValue(interested_list) - @defer.inlineCallbacks def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. @@ -94,7 +73,7 @@ class ApplicationServicesHandler(object): event(Event): The event to push out to interested services. """ # Gather interested services - services = yield self.get_services_for_event(event) + services = yield self._get_services_for_event(event) if len(services) == 0: return # no services need notifying @@ -102,7 +81,7 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: - user_query_services = yield self.get_services_for_event( + user_query_services = yield self._get_services_for_event( event=event, restrict_to=ApplicationService.NS_USERS ) @@ -115,28 +94,62 @@ class ApplicationServicesHandler(object): # the user exists now,so don't query more ASes. break - # Do we know this room alias exists? If not, poke the room alias query - # API for all services which match that room alias regex. - unknown_room_alias = False # TODO - if unknown_room_alias: - alias = "something" # TODO - alias_query_services = yield self.get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_ALIASES - ) - for alias_service in alias_query_services: - # this needs to block XXX: Need to feed response back to caller - is_known_alias = yield self.appservice_api.query_alias( - alias_service, alias - ) - if is_known_alias: - # the alias exists now so don't query more ASes. - break - # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + + @defer.inlineCallbacks + def query_room_alias_exists(self, room_alias): + """Check if an application service knows this room alias exists. + + Args: + room_alias(str): The room alias to query. + Returns: + namedtuple: with keys "room_id" and "servers" or None if no + association can be found. + """ + alias_query_services = yield self._get_services_for_event( + event=None, + restrict_to=ApplicationService.NS_ALIASES, + alias_list=[room_alias] + ) + for alias_service in alias_query_services: + is_known_alias = yield self.appservice_api.query_alias( + alias_service, room_alias + ) + if is_known_alias: + # the alias exists now so don't query more ASes. + result = yield self.store.get_association_from_room_alias( + room_alias + ) + defer.returnValue(result) + + @defer.inlineCallbacks + def _get_services_for_event(self, event, restrict_to="", alias_list=None): + """Retrieve a list of application services interested in this event. + + Args: + event(Event): The event to check. Can be None if alias_list is not. + restrict_to(str): The namespace to restrict regex tests to. + alias_list: A list of aliases to get services for. If None, this + list is obtained from the database. + Returns: + list: A list of services interested in this + event based on the service regex. + """ + # We need to know the aliases associated with this event.room_id, if any + if not alias_list: + alias_list = yield self.store.get_aliases_for_room(event.room_id) + + services = yield self.store.get_app_services() + interested_list = [ + s for s in services if ( + s.is_interested(event, restrict_to, alias_list) + ) + ] + defer.returnValue(interested_list) + @defer.inlineCallbacks def _is_unknown_user(self, user_id): user = UserID.from_string(user_id) @@ -149,7 +162,5 @@ class ApplicationServicesHandler(object): user_info = yield self.store.get_user_by_id(user_id) defer.returnValue(len(user_info) == 0) - - def _generate_hs_token(self): return stringutils.random_string(24) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 58e9a9156..000bf5793 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -84,6 +84,13 @@ class DirectoryHandler(BaseHandler): if result: room_id = result.room_id servers = result.servers + else: + # Query AS to see if it exists + as_handler = self.hs.get_handlers().appservice_handler + result = yield as_handler.query_room_alias_exists(room_alias) + if result: + room_id = result.room_id + servers = result.servers else: try: result = yield self.federation.make_query( From bc658907f0195f4f5f3ec9f7730884bbd516ce79 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 11:54:36 +0000 Subject: [PATCH 283/630] Add unit test for appservice_handler.query_room_alias_exists --- tests/handlers/test_appservice.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index f92dc1c89..fd6bcbbce 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -37,6 +37,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): raise Exception("Test expected handler.appservice_api to exist.") self.handler.appservice_api = self.mock_as_api + @defer.inlineCallbacks def test_notify_interested_services(self): interested_service = self._mkservice(is_interested=True) services = [ @@ -54,9 +55,37 @@ class AppServiceHandlerTestCase(unittest.TestCase): room_id="!foo:bar" ) self.mock_as_api.push = Mock() - self.handler.notify_interested_services(event) + yield self.handler.notify_interested_services(event) self.mock_as_api.push.assert_called_once_with(interested_service, event) + @defer.inlineCallbacks + def test_query_room_alias_exists(self): + room_alias = "#foo:bar" + room_id = "!alpha:bet" + servers = ["aperture"] + interested_service = self._mkservice(is_interested=True) + services = [ + self._mkservice(is_interested=False), + interested_service, + self._mkservice(is_interested=False) + ] + + self.mock_store.get_app_services = Mock(return_value=services) + self.mock_store.get_association_from_room_alias = Mock( + return_value=Mock(room_id=room_id, servers=servers) + ) + + result = yield self.handler.query_room_alias_exists(room_alias) + + self.mock_as_api.query_alias.assert_called_once_with( + interested_service, + room_alias + ) + self.assertEquals(result.room_id, room_id) + self.assertEquals(result.servers, servers) + + + def _mkservice(self, is_interested): service = Mock() service.is_interested = Mock(return_value=is_interested) From 26a041541baed887dc069c3667a86ddef81802bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 13:17:05 +0000 Subject: [PATCH 284/630] SYN-202: Log as WARN the 404 'Presence information not visible' errors instead of as ERROR since they were spamming the logs --- synapse/handlers/message.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6fbd2af4a..3f51f38f1 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import RoomError +from synapse.api.errors import RoomError, SynapseError from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -372,10 +372,17 @@ class MessageHandler(BaseHandler): as_event=True, ) presence.append(member_presence) - except Exception: - logger.exception( - "Failed to get member presence of %r", m.user_id - ) + except SynapseError as e: + if e.code == 404: + # FIXME: We are doing this as a warn since this gets hit a + # lot and spams the logs. Why is this happening? + logger.warn( + "Failed to get member presence of %r", m.user_id + ) + else: + logger.exception( + "Failed to get member presence of %r", m.user_id + ) time_now = self.clock.time_msec() From 51d63ac329a2613c0a7195e6183d70f789b7d823 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:19:46 +0000 Subject: [PATCH 285/630] Glue AS work to general event notifications. Add more exception handling when poking ASes. --- synapse/appservice/__init__.py | 2 +- synapse/appservice/api.py | 11 +++++++++- synapse/handlers/appservice.py | 39 ++++++++++++++++++++++------------ synapse/notifier.py | 6 ++++++ 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f7baf578f..d9ca856c8 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -62,7 +62,7 @@ class ApplicationService(object): def _matches_regex(self, test_string, namespace_key): if not isinstance(test_string, basestring): - logger.warning( + logger.error( "Expected a string to test regex against, but got %s", test_string ) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 29bb35d61..d96caf7f5 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -46,6 +46,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_user to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_user to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def query_alias(self, service, alias): @@ -62,6 +65,10 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) return logger.warning("query_alias to %s received %s", uri, e.code) + except Exception as ex: + logger.warning("query_alias to %s threw exception %s", uri, ex) + defer.returnValue(False) + @defer.inlineCallbacks def push_bulk(self, service, events): @@ -82,7 +89,9 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(True) except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) - defer.returnValue(False) + except Exception as ex: + logger.warning("push_bulk to %s threw exception %s", uri, ex) + defer.returnValue(False) @defer.inlineCallbacks def push(self, service, event): diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2c6d4e281..ef9421513 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -23,7 +23,6 @@ import synapse.util.stringutils as stringutils import logging - logger = logging.getLogger(__name__) @@ -58,6 +57,7 @@ class ApplicationServicesHandler(object): yield self.store.update_app_service(app_service) defer.returnValue(app_service) + @defer.inlineCallbacks def unregister(self, token): logger.info("Unregister as_token=%s", token) yield self.store.unregister_app_service(token) @@ -81,34 +81,45 @@ class ApplicationServicesHandler(object): # all services which match that user regex. unknown_user = yield self._is_unknown_user(event.sender) if unknown_user: - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS - ) - for user_service in user_query_services: - # this needs to block XXX: Need to feed response back to caller - is_known_user = yield self.appservice_api.query_user( - user_service, event.sender - ) - if is_known_user: - # the user exists now,so don't query more ASes. - break + yield self.query_user_exists(event) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) + @defer.inlineCallbacks + def query_user_exists(self, event): + """Check if an application services knows this event.sender exists. + + Args: + event: An event sent by the user to query + Returns: + True if this user exists. + """ + # TODO Would be nice for this to accept a user ID instead of an event. + user_query_services = yield self._get_services_for_event( + event=event, + restrict_to=ApplicationService.NS_USERS + ) + for user_service in user_query_services: + is_known_user = yield self.appservice_api.query_user( + user_service, event.sender + ) + if is_known_user: + defer.returnValue(True) + defer.returnValue(False) @defer.inlineCallbacks def query_room_alias_exists(self, room_alias): """Check if an application service knows this room alias exists. Args: - room_alias(str): The room alias to query. + room_alias(RoomAlias): The room alias to query. Returns: namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ + room_alias = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, diff --git a/synapse/notifier.py b/synapse/notifier.py index e3b6ead62..c7f75ab80 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -99,6 +99,12 @@ class Notifier(object): `extra_users` param. """ yield run_on_reactor() + + # poke any interested application service. + self.hs.get_handlers().appservice_handler.notify_interested_services( + event + ) + room_id = event.room_id room_source = self.event_sources.sources["room"] From 131e0364022735ab63d749c158e6875b7b11700e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:22:20 +0000 Subject: [PATCH 286/630] Fix unit tests. --- tests/handlers/test_appservice.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index fd6bcbbce..e16e51158 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -60,7 +60,10 @@ class AppServiceHandlerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_query_room_alias_exists(self): - room_alias = "#foo:bar" + room_alias_str = "#foo:bar" + room_alias = Mock() + room_alias.to_string = Mock(return_value=room_alias_str) + room_id = "!alpha:bet" servers = ["aperture"] interested_service = self._mkservice(is_interested=True) @@ -79,7 +82,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): self.mock_as_api.query_alias.assert_called_once_with( interested_service, - room_alias + room_alias_str ) self.assertEquals(result.room_id, room_id) self.assertEquals(result.servers, servers) From 0613666d9c417005393636b935593c423f4417b9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 13:42:35 +0000 Subject: [PATCH 287/630] Serialize events before sending to ASes --- synapse/appservice/api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index d96caf7f5..9cce4e097 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -16,6 +16,7 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient +from synapse.events.utils import serialize_event import logging import urllib @@ -30,6 +31,7 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) + self.clock = hs.get_clock() @defer.inlineCallbacks def query_user(self, service, user_id): @@ -72,6 +74,8 @@ class ApplicationServiceApi(SimpleHttpClient): @defer.inlineCallbacks def push_bulk(self, service, events): + events = self._serialize(events) + uri = service.url + ("/transactions/%s" % urllib.quote(str(0))) # TODO txn_ids response = None @@ -98,3 +102,9 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.push_bulk(service, [event]) defer.returnValue(response) + def _serialize(self, events): + time_now = self.clock.time_msec() + return [ + serialize_event(e, time_now, as_client_event=True) for e in events + ] + From e1515c3e91f2117adc3976b5e606728560ce9e96 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 13:43:28 +0000 Subject: [PATCH 288/630] Pass through list of room hosts from room alias query to federation so that it can retry against different room hosts --- synapse/federation/federation_client.py | 5 ++++- synapse/handlers/federation.py | 20 +++++++++++++------- synapse/handlers/room.py | 12 +++++------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index eb36ec040..9923b3fc0 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -264,7 +264,9 @@ class FederationClient(FederationBase): logger.debug("Got response to make_join: %s", pdu_dict) - defer.returnValue(self.event_from_pdu_json(pdu_dict)) + defer.returnValue( + (destination, self.event_from_pdu_json(pdu_dict)) + ) break except CodeMessageException: raise @@ -313,6 +315,7 @@ class FederationClient(FederationBase): defer.returnValue({ "state": signed_state, "auth_chain": signed_auth, + "origin": destination, }) except CodeMessageException: raise diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 04a468948..aba266c2b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -273,7 +273,7 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def do_invite_join(self, target_host, room_id, joinee, content, snapshot): + def do_invite_join(self, target_hosts, room_id, joinee, content, snapshot): """ Attempts to join the `joinee` to the room `room_id` via the server `target_host`. @@ -287,8 +287,8 @@ class FederationHandler(BaseHandler): """ logger.debug("Joining %s to %s", joinee, room_id) - pdu = yield self.replication_layer.make_join( - [target_host], + origin, pdu = yield self.replication_layer.make_join( + target_hosts, room_id, joinee ) @@ -330,11 +330,17 @@ class FederationHandler(BaseHandler): new_event = builder.build() + # Try the host we successfully got a response to /make_join/ + # request first. + target_hosts.remove(origin) + target_hosts.insert(0, origin) + ret = yield self.replication_layer.send_join( - [target_host], + target_hosts, new_event ) + origin = ret["origin"] state = ret["state"] auth_chain = ret["auth_chain"] auth_chain.sort(key=lambda e: e.depth) @@ -371,7 +377,7 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } yield self._handle_new_event( - target_host, e, auth_events=auth + origin, e, auth_events=auth ) except: logger.exception( @@ -391,7 +397,7 @@ class FederationHandler(BaseHandler): if e.event_id in auth_ids } yield self._handle_new_event( - target_host, e, auth_events=auth + origin, e, auth_events=auth ) except: logger.exception( @@ -406,7 +412,7 @@ class FederationHandler(BaseHandler): } yield self._handle_new_event( - target_host, + origin, new_event, state=state, current_state=state, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 23821d321..0369b907a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -389,8 +389,6 @@ class RoomMemberHandler(BaseHandler): if not hosts: raise SynapseError(404, "No known servers") - host = hosts[0] - # If event doesn't include a display name, add one. yield self.distributor.fire( "collect_presencelike_data", joinee, content @@ -407,12 +405,12 @@ class RoomMemberHandler(BaseHandler): }) event, context = yield self._create_new_client_event(builder) - yield self._do_join(event, context, room_host=host, do_auth=True) + yield self._do_join(event, context, room_hosts=hosts, do_auth=True) defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks - def _do_join(self, event, context, room_host=None, do_auth=True): + def _do_join(self, event, context, room_hosts=None, do_auth=True): joinee = UserID.from_string(event.state_key) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -441,7 +439,7 @@ class RoomMemberHandler(BaseHandler): if is_host_in_room: should_do_dance = False - elif room_host: # TODO: Shouldn't this be remote_room_host? + elif room_hosts: # TODO: Shouldn't this be remote_room_host? should_do_dance = True else: # TODO(markjh): get prev_state from snapshot @@ -453,7 +451,7 @@ class RoomMemberHandler(BaseHandler): inviter = UserID.from_string(prev_state.user_id) should_do_dance = not self.hs.is_mine(inviter) - room_host = inviter.domain + room_hosts = [inviter.domain] else: # return the same error as join_room_alias does raise SynapseError(404, "No known servers") @@ -461,7 +459,7 @@ class RoomMemberHandler(BaseHandler): if should_do_dance: handler = self.hs.get_handlers().federation_handler yield handler.do_invite_join( - room_host, + room_hosts, room_id, event.user_id, event.get_dict()["content"], # FIXME To get a non-frozen dict From e9c85a4d5ab332021f93634182ad8ed93bd0091c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 Feb 2015 13:50:15 +0000 Subject: [PATCH 289/630] Connection errors in twisted aren't RuntimeErrors --- synapse/federation/federation_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 9923b3fc0..70c9a6f46 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -270,7 +270,7 @@ class FederationClient(FederationBase): break except CodeMessageException: raise - except RuntimeError as e: + except Exception as e: logger.warn( "Failed to make_join via %s: %s", destination, e.message @@ -319,7 +319,7 @@ class FederationClient(FederationBase): }) except CodeMessageException: raise - except RuntimeError as e: + except Exception as e: logger.warn( "Failed to send_join via %s: %s", destination, e.message From c71456117dc70dfe0bfa15e3f655a3ac1dfc66ee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 14:17:08 +0000 Subject: [PATCH 290/630] Fix user query checks. HS>AS pushing now works. --- synapse/appservice/__init__.py | 21 +++++++++------- synapse/handlers/appservice.py | 44 ++++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index d9ca856c8..46d46a5a4 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -75,27 +75,23 @@ class ApplicationService(object): def _matches_user(self, event): if (hasattr(event, "sender") and - self._matches_regex( - event.sender, ApplicationService.NS_USERS)): + self.is_interested_in_user(event.sender)): return True # also check m.room.member state key if (hasattr(event, "type") and event.type == EventTypes.Member and hasattr(event, "state_key") - and self._matches_regex( - event.state_key, ApplicationService.NS_USERS)): + and self.is_interested_in_user(event.state_key)): return True return False def _matches_room_id(self, event): if hasattr(event, "room_id"): - return self._matches_regex( - event.room_id, ApplicationService.NS_ROOMS - ) + return self.is_interested_in_room(event.room_id) return False def _matches_aliases(self, event, alias_list): for alias in alias_list: - if self._matches_regex(alias, ApplicationService.NS_ALIASES): + if self.is_interested_in_alias(alias): return True return False @@ -128,5 +124,14 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_USERS: return self._matches_user(event) + def is_interested_in_user(self, user_id): + return self._matches_regex(user_id, ApplicationService.NS_USERS) + + def is_interested_in_alias(self, alias): + return self._matches_regex(alias, ApplicationService.NS_ALIASES) + + def is_interested_in_room(self, room_id): + return self._matches_regex(room_id, ApplicationService.NS_ROOMS) + def __str__(self): return "ApplicationService: %s" % (self.__dict__,) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index ef9421513..8d0cdd528 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,6 +15,7 @@ from twisted.internet import defer +from synapse.api.constants import EventTypes from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.appservice.api import ApplicationServiceApi @@ -78,32 +79,31 @@ class ApplicationServicesHandler(object): return # no services need notifying # Do we know this user exists? If not, poke the user query API for - # all services which match that user regex. - unknown_user = yield self._is_unknown_user(event.sender) - if unknown_user: - yield self.query_user_exists(event) + # all services which match that user regex. This needs to block as these + # user queries need to be made BEFORE pushing the event. + yield self._check_user_exists(event.sender) + if event.type == EventTypes.Member: + yield self._check_user_exists(event.state_key) # Fork off pushes to these services - XXX First cut, best effort for service in services: self.appservice_api.push(service, event) @defer.inlineCallbacks - def query_user_exists(self, event): - """Check if an application services knows this event.sender exists. + def query_user_exists(self, user_id): + """Check if any application service knows this user_id exists. Args: - event: An event sent by the user to query + user_id(str): The user to query if they exist on any AS. Returns: - True if this user exists. + True if this user exists on at least one application service. """ - # TODO Would be nice for this to accept a user ID instead of an event. - user_query_services = yield self._get_services_for_event( - event=event, - restrict_to=ApplicationService.NS_USERS + user_query_services = yield self._get_services_for_user( + user_id=user_id ) for user_service in user_query_services: is_known_user = yield self.appservice_api.query_user( - user_service, event.sender + user_service, user_id ) if is_known_user: defer.returnValue(True) @@ -161,6 +161,16 @@ class ApplicationServicesHandler(object): ] defer.returnValue(interested_list) + @defer.inlineCallbacks + def _get_services_for_user(self, user_id): + services = yield self.store.get_app_services() + interested_list = [ + s for s in services if ( + s.is_interested_in_user(user_id) + ) + ] + defer.returnValue(interested_list) + @defer.inlineCallbacks def _is_unknown_user(self, user_id): user = UserID.from_string(user_id) @@ -173,5 +183,13 @@ class ApplicationServicesHandler(object): user_info = yield self.store.get_user_by_id(user_id) defer.returnValue(len(user_info) == 0) + @defer.inlineCallbacks + def _check_user_exists(self, user_id): + unknown_user = yield self._is_unknown_user(user_id) + if unknown_user: + exists = yield self.query_user_exists(user_id) + defer.returnValue(exists) + defer.returnValue(True) + def _generate_hs_token(self): return stringutils.random_string(24) From f90782a6589ee567d6dc9ead19451cc8a0aabdd3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 5 Feb 2015 09:35:21 +0000 Subject: [PATCH 291/630] namespace rule IDs to be unique within their scope and rule type. --- synapse/rest/client/v1/push_rule.py | 252 ++++++++++++++-------------- 1 file changed, 129 insertions(+), 123 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 5582f33c8..eaef55cc1 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -38,100 +38,9 @@ class PushRuleRestServlet(ClientV1RestServlet): SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( "Unrecognised request: You probably wanted a trailing slash") - def rule_spec_from_path(self, path): - if len(path) < 2: - raise UnrecognizedRequestError() - if path[0] != 'pushrules': - raise UnrecognizedRequestError() - - scope = path[1] - path = path[2:] - if scope not in ['global', 'device']: - raise UnrecognizedRequestError() - - device = None - if scope == 'device': - if len(path) == 0: - raise UnrecognizedRequestError() - device = path[0] - path = path[1:] - - if len(path) == 0: - raise UnrecognizedRequestError() - - template = path[0] - path = path[1:] - - if len(path) == 0: - raise UnrecognizedRequestError() - - rule_id = path[0] - - spec = { - 'scope': scope, - 'template': template, - 'rule_id': rule_id - } - if device: - spec['profile_tag'] = device - return spec - - def rule_tuple_from_request_object(self, rule_template, rule_id, req_obj, device=None): - if rule_template in ['override', 'underride']: - if 'conditions' not in req_obj: - raise InvalidRuleException("Missing 'conditions'") - conditions = req_obj['conditions'] - for c in conditions: - if 'kind' not in c: - raise InvalidRuleException("Condition without 'kind'") - elif rule_template == 'room': - conditions = [{ - 'kind': 'event_match', - 'key': 'room_id', - 'pattern': rule_id - }] - elif rule_template == 'sender': - conditions = [{ - 'kind': 'event_match', - 'key': 'user_id', - 'pattern': rule_id - }] - elif rule_template == 'content': - if 'pattern' not in req_obj: - raise InvalidRuleException("Content rule missing 'pattern'") - pat = req_obj['pattern'] - - conditions = [{ - 'kind': 'event_match', - 'key': 'content.body', - 'pattern': pat - }] - else: - raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) - - if device: - conditions.append({ - 'kind': 'device', - 'profile_tag': device - }) - - if 'actions' not in req_obj: - raise InvalidRuleException("No actions found") - actions = req_obj['actions'] - - for a in actions: - if a in ['notify', 'dont_notify', 'coalesce']: - pass - elif isinstance(a, dict) and 'set_sound' in a: - pass - else: - raise InvalidRuleException("Unrecognised action") - - return conditions, actions - @defer.inlineCallbacks def on_PUT(self, request): - spec = self.rule_spec_from_path(request.postpath) + spec = _rule_spec_from_path(request.postpath) try: priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: @@ -142,10 +51,13 @@ class PushRuleRestServlet(ClientV1RestServlet): if spec['template'] == 'default': raise SynapseError(403, "The default rules are immutable.") + if not spec['rule_id'].isalnum(): + raise SynapseError(400, "rule_id may only contain alphanumeric characters") + content = _parse_json(request) try: - (conditions, actions) = self.rule_tuple_from_request_object( + (conditions, actions) = _rule_tuple_from_request_object( spec['template'], spec['rule_id'], content, @@ -164,7 +76,7 @@ class PushRuleRestServlet(ClientV1RestServlet): try: yield self.hs.get_datastore().add_push_rule( user_name=user.to_string(), - rule_id=spec['rule_id'], + rule_id=_namespaced_rule_id_from_spec(spec), priority_class=priority_class, conditions=conditions, actions=actions, @@ -180,7 +92,7 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request): - spec = self.rule_spec_from_path(request.postpath) + spec = _rule_spec_from_path(request.postpath) try: priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: @@ -188,32 +100,18 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) - if 'profile_tag' in spec: - rules = yield self.hs.get_datastore().get_push_rules_for_user_name( - user.to_string() - ) + namespaced_rule_id = _namespaced_rule_id_from_spec(spec) - for r in rules: - conditions = json.loads(r['conditions']) - pt = _profile_tag_from_conditions(conditions) - if pt == spec['profile_tag'] and r['priority_class'] == priority_class: - yield self.hs.get_datastore().delete_push_rule( - user.to_string(), spec['rule_id'] - ) - defer.returnValue((200, {})) - raise NotFoundError() - else: - try: - yield self.hs.get_datastore().delete_push_rule( - user.to_string(), spec['rule_id'], - priority_class=priority_class - ) - defer.returnValue((200, {})) - except StoreError as e: - if e.code == 404: - raise NotFoundError() - else: - raise + try: + yield self.hs.get_datastore().delete_push_rule( + user.to_string(), namespaced_rule_id + ) + defer.returnValue((200, {})) + except StoreError as e: + if e.code == 404: + raise NotFoundError() + else: + raise @defer.inlineCallbacks def on_GET(self, request): @@ -298,6 +196,99 @@ class PushRuleRestServlet(ClientV1RestServlet): return 200, {} +def _rule_spec_from_path(path): + if len(path) < 2: + raise UnrecognizedRequestError() + if path[0] != 'pushrules': + raise UnrecognizedRequestError() + + scope = path[1] + path = path[2:] + if scope not in ['global', 'device']: + raise UnrecognizedRequestError() + + device = None + if scope == 'device': + if len(path) == 0: + raise UnrecognizedRequestError() + device = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + template = path[0] + path = path[1:] + + if len(path) == 0: + raise UnrecognizedRequestError() + + rule_id = path[0] + + spec = { + 'scope': scope, + 'template': template, + 'rule_id': rule_id + } + if device: + spec['profile_tag'] = device + return spec + + +def _rule_tuple_from_request_object(rule_template, rule_id, req_obj, device=None): + if rule_template in ['override', 'underride']: + if 'conditions' not in req_obj: + raise InvalidRuleException("Missing 'conditions'") + conditions = req_obj['conditions'] + for c in conditions: + if 'kind' not in c: + raise InvalidRuleException("Condition without 'kind'") + elif rule_template == 'room': + conditions = [{ + 'kind': 'event_match', + 'key': 'room_id', + 'pattern': rule_id + }] + elif rule_template == 'sender': + conditions = [{ + 'kind': 'event_match', + 'key': 'user_id', + 'pattern': rule_id + }] + elif rule_template == 'content': + if 'pattern' not in req_obj: + raise InvalidRuleException("Content rule missing 'pattern'") + pat = req_obj['pattern'] + + conditions = [{ + 'kind': 'event_match', + 'key': 'content.body', + 'pattern': pat + }] + else: + raise InvalidRuleException("Unknown rule template: %s" % (rule_template,)) + + if device: + conditions.append({ + 'kind': 'device', + 'profile_tag': device + }) + + if 'actions' not in req_obj: + raise InvalidRuleException("No actions found") + actions = req_obj['actions'] + + for a in actions: + if a in ['notify', 'dont_notify', 'coalesce']: + pass + elif isinstance(a, dict) and 'set_sound' in a: + pass + else: + raise InvalidRuleException("Unrecognised action") + + return conditions, actions + + def _add_empty_priority_class_arrays(d): for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): d[pc] = [] @@ -361,20 +352,24 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): + unscoped_rule_id = _rule_id_from_namespaced(rule['rule_id']) + template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['default']: return {k: rule[k] for k in ["conditions", "actions"]} elif template_name in ['override', 'underride']: - return {k: rule[k] for k in ["rule_id", "conditions", "actions"]} + ret = {k: rule[k] for k in ["conditions", "actions"]} + ret['rule_id'] = unscoped_rule_id + return ret elif template_name in ["sender", "room"]: - return {k: rule[k] for k in ["rule_id", "actions"]} + return {'rule_id': unscoped_rule_id, 'actions': rule['actions']} elif template_name == 'content': if len(rule["conditions"]) != 1: return None thecond = rule["conditions"][0] if "pattern" not in thecond: return None - ret = {k: rule[k] for k in ["rule_id", "actions"]} + ret = {'rule_id': unscoped_rule_id, 'actions': rule['actions']} ret["pattern"] = thecond["pattern"] return ret @@ -386,6 +381,17 @@ def _strip_device_condition(rule): return rule +def _namespaced_rule_id_from_spec(spec): + if spec['scope'] == 'global': + scope = 'global' + else: + scope = 'device.%s' % (spec['profile_tag']) + return "%s.%s.%s" % (scope, spec['template'], spec['rule_id']) + + +def _rule_id_from_namespaced(in_rule_id, spec): + return in_rule_id.split('.')[-1] + class InvalidRuleException(Exception): pass From 2df41aa1386545f4237c0141c19db1fef85e7161 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 5 Feb 2015 14:46:06 +0000 Subject: [PATCH 292/630] Server default rules now of all kinds rather than all being at lowest prio. --- synapse/push/__init__.py | 8 ++-- synapse/push/baserules.py | 62 +++++++++++++++++++++---- synapse/push/rulekinds.py | 8 ++++ synapse/rest/client/v1/push_rule.py | 71 +++++++++++++---------------- 4 files changed, 98 insertions(+), 51 deletions(-) create mode 100644 synapse/push/rulekinds.py diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8c6f0a657..729371529 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -77,15 +77,15 @@ class Pusher(object): if ev['state_key'] != self.user_name: defer.returnValue(['dont_notify']) - rules = yield self.store.get_push_rules_for_user_name(self.user_name) + rawrules = yield self.store.get_push_rules_for_user_name(self.user_name) - for r in rules: + for r in rawrules: r['conditions'] = json.loads(r['conditions']) r['actions'] = json.loads(r['actions']) - user_name_localpart = UserID.from_string(self.user_name).localpart + user = UserID.from_string(self.user_name) - rules.extend(baserules.make_base_rules(user_name_localpart)) + rules = baserules.list_with_base_rules(rawrules, user) # get *our* member event for display name matching member_events_for_room = yield self.store.get_current_state( diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 376d1d4d3..191909ad4 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,20 +1,68 @@ -def make_base_rules(user_name): - rules = [ +from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP + +def list_with_base_rules(rawrules, user_name): + ruleslist = [] + + # shove the server default rules for each kind onto the end of each + current_prio_class = 1 + for r in rawrules: + if r['priority_class'] > current_prio_class: + while current_prio_class < r['priority_class']: + ruleslist.extend(make_base_rules( + user_name, + PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) + ) + current_prio_class += 1 + + ruleslist.append(r) + + while current_prio_class <= PRIORITY_CLASS_INVERSE_MAP.keys()[-1]: + ruleslist.extend(make_base_rules( + user_name, + PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) + ) + current_prio_class += 1 + + return ruleslist + + +def make_base_rules(user, kind): + rules = [] + + if kind == 'override': + rules = make_base_override_rules() + elif kind == 'content': + rules = make_base_content_rules(user) + + for r in rules: + r['priority_class'] = PRIORITY_CLASS_MAP[kind] + + return rules + + +def make_base_content_rules(user): + return [ { 'conditions': [ { 'kind': 'event_match', 'key': 'content.body', - 'pattern': '*%s*' % (user_name,), # Matrix ID match + 'pattern': user.localpart, # Matrix ID match } ], 'actions': [ 'notify', { - 'set_sound': 'default' + 'set_tweak': 'sound', + 'value': 'default', } ] }, + ] + + +def make_base_override_rules(): + return [ { 'conditions': [ { @@ -24,7 +72,8 @@ def make_base_rules(user_name): 'actions': [ 'notify', { - 'set_sound': 'default' + 'set_tweak': 'sound', + 'value': 'default' } ] }, @@ -44,6 +93,3 @@ def make_base_rules(user_name): ] } ] - for r in rules: - r['priority_class'] = 0 - return rules \ No newline at end of file diff --git a/synapse/push/rulekinds.py b/synapse/push/rulekinds.py new file mode 100644 index 000000000..763bdee58 --- /dev/null +++ b/synapse/push/rulekinds.py @@ -0,0 +1,8 @@ +PRIORITY_CLASS_MAP = { + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, + } +PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index eaef55cc1..7ab167ce0 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -20,21 +20,13 @@ from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, No from .base import ClientV1RestServlet, client_path_pattern from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException import synapse.push.baserules as baserules +from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP import json class PushRuleRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/pushrules/.*$") - PRIORITY_CLASS_MAP = { - 'default': 0, - 'underride': 1, - 'sender': 2, - 'room': 3, - 'content': 4, - 'override': 5, - } - PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = ( "Unrecognised request: You probably wanted a trailing slash") @@ -48,11 +40,8 @@ class PushRuleRestServlet(ClientV1RestServlet): user, _ = yield self.auth.get_user_by_req(request) - if spec['template'] == 'default': - raise SynapseError(403, "The default rules are immutable.") - - if not spec['rule_id'].isalnum(): - raise SynapseError(400, "rule_id may only contain alphanumeric characters") + if '/' in spec['rule_id'] or '\\' in spec['rule_id']: + raise SynapseError(400, "rule_id may not contain slashes") content = _parse_json(request) @@ -121,21 +110,23 @@ class PushRuleRestServlet(ClientV1RestServlet): # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + for r in rawrules: r["conditions"] = json.loads(r["conditions"]) r["actions"] = json.loads(r["actions"]) - rawrules.extend(baserules.make_base_rules(user.to_string())) + + ruleslist = baserules.list_with_base_rules(rawrules, user) rules = {'global': {}, 'device': {}} rules['global'] = _add_empty_priority_class_arrays(rules['global']) - for r in rawrules: + for r in ruleslist: rulearray = None template_name = _priority_class_to_template_name(r['priority_class']) - if r['priority_class'] > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + if r['priority_class'] > PRIORITY_CLASS_MAP['override']: # per-device rule profile_tag = _profile_tag_from_conditions(r["conditions"]) r = _strip_device_condition(r) @@ -290,7 +281,7 @@ def _rule_tuple_from_request_object(rule_template, rule_id, req_obj, device=None def _add_empty_priority_class_arrays(d): - for pc in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + for pc in PRIORITY_CLASS_MAP.keys(): d[pc] = [] return d @@ -332,46 +323,48 @@ def _filter_ruleset_with_path(ruleset, path): def _priority_class_from_spec(spec): - if spec['template'] not in PushRuleRestServlet.PRIORITY_CLASS_MAP.keys(): + if spec['template'] not in PRIORITY_CLASS_MAP.keys(): raise InvalidRuleException("Unknown template: %s" % (spec['kind'])) - pc = PushRuleRestServlet.PRIORITY_CLASS_MAP[spec['template']] + pc = PRIORITY_CLASS_MAP[spec['template']] if spec['scope'] == 'device': - pc += len(PushRuleRestServlet.PRIORITY_CLASS_MAP) + pc += len(PRIORITY_CLASS_MAP) return pc def _priority_class_to_template_name(pc): - if pc > PushRuleRestServlet.PRIORITY_CLASS_MAP['override']: + if pc > PRIORITY_CLASS_MAP['override']: # per-device prio_class_index = pc - len(PushRuleRestServlet.PRIORITY_CLASS_MAP) - return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[prio_class_index] + return PRIORITY_CLASS_INVERSE_MAP[prio_class_index] else: - return PushRuleRestServlet.PRIORITY_CLASS_INVERSE_MAP[pc] + return PRIORITY_CLASS_INVERSE_MAP[pc] def _rule_to_template(rule): - unscoped_rule_id = _rule_id_from_namespaced(rule['rule_id']) + unscoped_rule_id = None + if 'rule_id' in rule: + _rule_id_from_namespaced(rule['rule_id']) template_name = _priority_class_to_template_name(rule['priority_class']) - if template_name in ['default']: - return {k: rule[k] for k in ["conditions", "actions"]} - elif template_name in ['override', 'underride']: - ret = {k: rule[k] for k in ["conditions", "actions"]} - ret['rule_id'] = unscoped_rule_id - return ret + if template_name in ['override', 'underride']: + templaterule = {k: rule[k] for k in ["conditions", "actions"]} elif template_name in ["sender", "room"]: - return {'rule_id': unscoped_rule_id, 'actions': rule['actions']} + templaterule = {'actions': rule['actions']} + unscoped_rule_id = rule['conditions'][0]['pattern'] elif template_name == 'content': if len(rule["conditions"]) != 1: return None thecond = rule["conditions"][0] if "pattern" not in thecond: return None - ret = {'rule_id': unscoped_rule_id, 'actions': rule['actions']} - ret["pattern"] = thecond["pattern"] - return ret + templaterule = {'actions': rule['actions']} + templaterule["pattern"] = thecond["pattern"] + + if unscoped_rule_id: + templaterule['rule_id'] = unscoped_rule_id + return templaterule def _strip_device_condition(rule): @@ -385,12 +378,12 @@ def _namespaced_rule_id_from_spec(spec): if spec['scope'] == 'global': scope = 'global' else: - scope = 'device.%s' % (spec['profile_tag']) - return "%s.%s.%s" % (scope, spec['template'], spec['rule_id']) + scope = 'device/%s' % (spec['profile_tag']) + return "%s/%s/%s" % (scope, spec['template'], spec['rule_id']) -def _rule_id_from_namespaced(in_rule_id, spec): - return in_rule_id.split('.')[-1] +def _rule_id_from_namespaced(in_rule_id): + return in_rule_id.split('/')[-1] class InvalidRuleException(Exception): pass From c163357f387788afa8fc1d4160e806eccbc5220a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 15:00:33 +0000 Subject: [PATCH 293/630] Add CS extension for masquerading as users within the namespaces specified by the AS. --- synapse/api/auth.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 7105ee21d..86f963c19 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -299,6 +299,29 @@ class Auth(object): # Can optionally look elsewhere in the request (e.g. headers) try: access_token = request.args["access_token"][0] + + # Check for application service tokens with a user_id override + try: + masquerade_user_id = request.args["user_id"][0] + app_service = yield self.store.get_app_service_by_token( + access_token + ) + if not app_service: + raise AuthError( + 403, "Invalid application service access token" + ) + if not app_service.is_interested_in_user(masquerade_user_id): + raise AuthError( + 403, + "Application service cannot masquerade as this user." + ) + defer.returnValue( + (UserID.from_string(masquerade_user_id), ClientInfo("", "")) + ) + return + except KeyError: + pass # normal users won't have this query parameter set + user_info = yield self.get_user_by_token(access_token) user = user_info["user"] device_id = user_info["device_id"] From aaf50bf6f3d6adee92fa4d5cb55dbf3c5a13dbe3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 5 Feb 2015 15:11:38 +0000 Subject: [PATCH 294/630] Give server default rules the 'default' attribute and fix various brokenness. --- synapse/push/baserules.py | 1 + synapse/rest/client/v1/push_rule.py | 4 +++- synapse/storage/push_rule.py | 7 +++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 191909ad4..8d4b806da 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -36,6 +36,7 @@ def make_base_rules(user, kind): for r in rules: r['priority_class'] = PRIORITY_CLASS_MAP[kind] + r['default'] = True return rules diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 7ab167ce0..80f116b1e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -345,7 +345,7 @@ def _priority_class_to_template_name(pc): def _rule_to_template(rule): unscoped_rule_id = None if 'rule_id' in rule: - _rule_id_from_namespaced(rule['rule_id']) + unscoped_rule_id = _rule_id_from_namespaced(rule['rule_id']) template_name = _priority_class_to_template_name(rule['priority_class']) if template_name in ['override', 'underride']: @@ -364,6 +364,8 @@ def _rule_to_template(rule): if unscoped_rule_id: templaterule['rule_id'] = unscoped_rule_id + if 'default' in rule: + templaterule['default'] = rule['default'] return templaterule diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 27502d239..30e23445d 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -176,7 +176,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, new_rule.values()) @defer.inlineCallbacks - def delete_push_rule(self, user_name, rule_id, **kwargs): + def delete_push_rule(self, user_name, rule_id): """ Delete a push rule. Args specify the row to be deleted and can be any of the columns in the push_rule table, but below are the @@ -186,7 +186,10 @@ class PushRuleStore(SQLBaseStore): user_name (str): The matrix ID of the push rule owner rule_id (str): The rule_id of the rule to be deleted """ - yield self._simple_delete_one(PushRuleTable.table_name, kwargs) + yield self._simple_delete_one( + PushRuleTable.table_name, + {'user_name': user_name, 'rule_id': rule_id} + ) class RuleNotFoundException(Exception): From 5b99b471b2ba9a9c9da801e9a9d6e1801f02068c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 15:12:36 +0000 Subject: [PATCH 295/630] Fix unit tests. --- synapse/api/auth.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 86f963c19..4f116184c 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -302,6 +302,12 @@ class Auth(object): # Check for application service tokens with a user_id override try: + if "user_id" not in request.args: + # This has to be done like this rather than relying on it + # natively throwing because tests use a Mock for the request + # object which doesn't throw :/ + raise KeyError + masquerade_user_id = request.args["user_id"][0] app_service = yield self.store.get_app_service_by_token( access_token From a93fa42bcef97b1e5f938a3a96f5afcbaeaef376 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 5 Feb 2015 15:45:16 +0000 Subject: [PATCH 296/630] priority class now dealt with in namespaced rule_id --- synapse/rest/client/v1/push_rule.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 80f116b1e..d43ade39d 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -82,10 +82,6 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request): spec = _rule_spec_from_path(request.postpath) - try: - priority_class = _priority_class_from_spec(spec) - except InvalidRuleException as e: - raise SynapseError(400, e.message) user, _ = yield self.auth.get_user_by_req(request) From e9484d6a9514d5c9aea3839bcc3fb25c9cb88cb0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 16:29:56 +0000 Subject: [PATCH 297/630] Prevent aliases in AS namespaces being created/deleted by users. Check with ASes when queried for room aliases via federation. --- synapse/handlers/directory.py | 42 +++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 24ea3573d..842f075fe 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -49,6 +49,12 @@ class DirectoryHandler(BaseHandler): # TODO(erikj): Check if there is a current association. + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service." + ) + if not servers: servers = yield self.store.get_joined_hosts_for_room(room_id) @@ -68,6 +74,12 @@ class DirectoryHandler(BaseHandler): if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service." + ) + room_id = yield self.store.delete_room_alias(room_alias) if room_id: @@ -77,20 +89,13 @@ class DirectoryHandler(BaseHandler): def get_association(self, room_alias): room_id = None if self.hs.is_mine(room_alias): - result = yield self.store.get_association_from_room_alias( + result = yield self.get_association_from_room_alias( room_alias ) if result: room_id = result.room_id servers = result.servers - else: - # Query AS to see if it exists - as_handler = self.hs.get_handlers().appservice_handler - result = yield as_handler.query_room_alias_exists(room_alias) - if result: - room_id = result.room_id - servers = result.servers else: try: result = yield self.federation.make_query( @@ -145,7 +150,7 @@ class DirectoryHandler(BaseHandler): 400, "Room Alias is not hosted on this Home Server" ) - result = yield self.store.get_association_from_room_alias( + result = yield self.get_association_from_room_alias( room_alias ) @@ -173,3 +178,22 @@ class DirectoryHandler(BaseHandler): "sender": user_id, "content": {"aliases": aliases}, }, ratelimit=False) + + @defer.inlineCallbacks + def get_association_from_room_alias(self, room_alias): + result = yield self.store.get_association_from_room_alias( + room_alias + ) + if not result: + # Query AS to see if it exists + as_handler = self.hs.get_handlers().appservice_handler + result = yield as_handler.query_room_alias_exists(room_alias) + defer.returnValue(result) + + @defer.inlineCallbacks + def is_alias_exclusive_to_appservices(self, alias): + services = yield self.store.get_app_services() + interested_services = [ + s for s in services if s.is_interested_in_alias(alias.to_string()) + ] + defer.returnValue(len(interested_services) > 0) From cab4c730885dcb5c95a50d425d6b6f655154a173 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 16:46:56 +0000 Subject: [PATCH 298/630] Prevent user IDs in AS namespaces being created/deleted by humans. --- synapse/handlers/register.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 08cd5fd72..b6e19d498 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -65,6 +65,8 @@ class RegistrationHandler(BaseHandler): user = UserID(localpart, self.hs.hostname) user_id = user.to_string() + yield self.check_user_id_is_valid(user_id) + token = self._generate_token(user_id) yield self.store.register( user_id=user_id, @@ -83,6 +85,7 @@ class RegistrationHandler(BaseHandler): localpart = self._generate_user_id() user = UserID(localpart, self.hs.hostname) user_id = user.to_string() + yield self.check_user_id_is_valid(user_id) token = self._generate_token(user_id) yield self.store.register( @@ -148,6 +151,19 @@ class RegistrationHandler(BaseHandler): # XXX: This should be a deferred list, shouldn't it? yield self._bind_threepid(c, user_id) + @defer.inlineCallbacks + def check_user_id_is_valid(self, user_id): + # valid user IDs must not clash with any user ID namespaces claimed by + # application services. + services = yield self.store.get_app_services() + interested_services = [ + s for s in services if s.is_interested_in_user(user_id) + ] + if len(interested_services) > 0: + raise SynapseError( + 400, "This user ID is reserved by an application service." + ) + def _generate_token(self, user_id): # urlsafe variant uses _ and - so use . as the separator and replace # all =s with .s so http clients don't quote =s when it is used as From a3c6010718c2749bd446bb63f3cf03bae09b0d20 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 16:48:57 +0000 Subject: [PATCH 299/630] Add delta sql file. --- synapse/storage/schema/delta/v14.sql | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 synapse/storage/schema/delta/v14.sql diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v14.sql new file mode 100644 index 000000000..03b5a10c8 --- /dev/null +++ b/synapse/storage/schema/delta/v14.sql @@ -0,0 +1,33 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + token TEXT, + hs_token TEXT, + UNIQUE(token) ON CONFLICT ROLLBACK +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id INTEGER PRIMARY KEY AUTOINCREMENT, + as_id INTEGER NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + + + From 11e6b3d18b5ae95857e01884960a01b9ea4d307d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 17:04:59 +0000 Subject: [PATCH 300/630] Dependency inject ApplicationServiceApi when creating ApplicationServicesHandler. --- synapse/handlers/__init__.py | 5 ++++- synapse/handlers/appservice.py | 5 ++--- tests/handlers/test_appservice.py | 10 +++------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index b31518bf6..8d345bf93 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.appservice.api import ApplicationServiceApi from .register import RegistrationHandler from .room import ( RoomCreationHandler, RoomMemberHandler, RoomListHandler @@ -53,5 +54,7 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) - self.appservice_handler = ApplicationServicesHandler(hs) + self.appservice_handler = ApplicationServicesHandler( + hs, ApplicationServiceApi(hs) + ) self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8d0cdd528..fa810b9a9 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -18,7 +18,6 @@ from twisted.internet import defer from synapse.api.constants import EventTypes from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService -from synapse.appservice.api import ApplicationServiceApi from synapse.types import UserID import synapse.util.stringutils as stringutils @@ -32,10 +31,10 @@ logger = logging.getLogger(__name__) # easier. class ApplicationServicesHandler(object): - def __init__(self, hs): + def __init__(self, hs, appservice_api): self.store = hs.get_datastore() self.hs = hs - self.appservice_api = ApplicationServiceApi(hs) + self.appservice_api = appservice_api @defer.inlineCallbacks def register(self, app_service): diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index e16e51158..a2c541317 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -29,13 +29,9 @@ class AppServiceHandlerTestCase(unittest.TestCase): self.mock_as_api = Mock() hs = Mock() hs.get_datastore = Mock(return_value=self.mock_store) - self.handler = ApplicationServicesHandler(hs) # thing being tested - - # FIXME Would be nice to DI this rather than monkey patch:( - if not hasattr(self.handler, "appservice_api"): - # someone probably updated the handler but not the tests. Fail fast. - raise Exception("Test expected handler.appservice_api to exist.") - self.handler.appservice_api = self.mock_as_api + self.handler = ApplicationServicesHandler( + hs, self.mock_as_api + ) @defer.inlineCallbacks def test_notify_interested_services(self): From 0227618d3c3bea6c85a922d5605f526719573121 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Feb 2015 17:29:27 +0000 Subject: [PATCH 301/630] Add m.login.application_service registration procedure. This allows known application services to register any user ID under their own user namespace(s). --- synapse/api/constants.py | 1 + synapse/handlers/register.py | 20 ++++++++++++++++++++ synapse/rest/client/v1/register.py | 24 +++++++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 0d3fc629a..420f963d9 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -59,6 +59,7 @@ class LoginType(object): EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" + APPLICATION_SERVICE = u"m.login.application_service" class EventTypes(object): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b6e19d498..60821edb0 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -105,6 +105,26 @@ class RegistrationHandler(BaseHandler): defer.returnValue((user_id, token)) + @defer.inlineCallbacks + def appservice_register(self, user_localpart, as_token): + user = UserID(user_localpart, self.hs.hostname) + user_id = user.to_string() + service = yield self.store.get_app_service_by_token(as_token) + if not service: + raise SynapseError(403, "Invalid application service token.") + if not service.is_interested_in_user(user_id): + raise SynapseError( + 400, "Invalid user localpart for this application service." + ) + token = self._generate_token(user_id) + yield self.store.register( + user_id=user_id, + token=token, + password_hash="" + ) + self.distributor.fire("registered_user", user) + defer.returnValue((user_id, token)) + @defer.inlineCallbacks def check_recaptcha(self, ip, private_key, challenge, response): """Checks a recaptcha is correct.""" diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index c0423c2d4..1ab32b53e 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -110,7 +110,8 @@ class RegisterRestServlet(ClientV1RestServlet): stages = { LoginType.RECAPTCHA: self._do_recaptcha, LoginType.PASSWORD: self._do_password, - LoginType.EMAIL_IDENTITY: self._do_email_identity + LoginType.EMAIL_IDENTITY: self._do_email_identity, + LoginType.APPLICATION_SERVICE: self._do_app_service } session_info = self._get_session_info(request, session) @@ -276,6 +277,27 @@ class RegisterRestServlet(ClientV1RestServlet): self._remove_session(session) defer.returnValue(result) + @defer.inlineCallbacks + def _do_app_service(self, request, register_json, session): + if "access_token" not in request.args: + raise SynapseError(400, "Expected application service token.") + if "user" not in register_json: + raise SynapseError(400, "Expected 'user' key.") + + as_token = request.args["access_token"][0] + user_localpart = register_json["user"].encode("utf-8") + + handler = self.handlers.registration_handler + (user_id, token) = yield handler.appservice_register( + user_localpart, as_token + ) + self._remove_session(session) + defer.returnValue({ + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + }) + def _parse_json(request): try: From 3737329d9b40b54bc4205c5f5e7e0946c5e51614 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 10:53:18 +0000 Subject: [PATCH 302/630] Handle the fact the list.remove raises if element doesn't exist --- synapse/handlers/federation.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index aba266c2b..9d0ce9aa5 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -332,8 +332,11 @@ class FederationHandler(BaseHandler): # Try the host we successfully got a response to /make_join/ # request first. - target_hosts.remove(origin) - target_hosts.insert(0, origin) + try: + target_hosts.remove(origin) + target_hosts.insert(0, origin) + except ValueError: + pass ret = yield self.replication_layer.send_join( target_hosts, @@ -521,7 +524,7 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) - destinations.remove(origin) + destinations.discard(origin) logger.debug( "on_send_join_request: Sending event: %s, signatures: %s", @@ -1013,7 +1016,10 @@ class FederationHandler(BaseHandler): for e in missing_remotes: for e_id, _ in e.auth_events: if e_id in missing_remote_ids: - base_remote_rejected.remove(e) + try: + base_remote_rejected.remove(e) + except ValueError: + pass reason_map = {} From 9f2573eea1641806c67d51b98013a17a686c0909 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 10:55:01 +0000 Subject: [PATCH 303/630] Return body of response in HttpResponseException --- synapse/http/matrixfederationclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8559d06b7..056d446e4 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -169,8 +169,9 @@ class MatrixFederationHttpClient(object): else: # :'( # Update transactions table? + body = yield readBody(response) raise HttpResponseException( - response.code, response.phrase, response + response.code, response.phrase, body ) defer.returnValue(response) From e426df8e106aef0b213928afb6189569474ac5d9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 10:57:14 +0000 Subject: [PATCH 304/630] Grant ASes the ability to create alias in their own namespace. Add a new errcode type M_EXCLUSIVE when users try to create aliases inside AS namespaces, and when ASes try to create aliases outside their own namespace. --- synapse/api/auth.py | 12 ++++++++ synapse/api/errors.py | 3 +- synapse/handlers/directory.py | 43 +++++++++++++++++++++-------- synapse/rest/client/v1/directory.py | 29 +++++++++++-------- 4 files changed, 63 insertions(+), 24 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 4f116184c..ea8c46172 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -380,6 +380,18 @@ class Auth(object): raise AuthError(403, "Unrecognised access token.", errcode=Codes.UNKNOWN_TOKEN) + @defer.inlineCallbacks + def get_appservice_by_req(self, request): + try: + token = request.args["access_token"][0] + service = yield self.store.get_app_service_by_token(token) + if not service: + raise AuthError(403, "Unrecognised access token.", + errcode=Codes.UNKNOWN_TOKEN) + defer.returnValue(service) + except KeyError: + raise AuthError(403, "Missing access token.") + def is_server_admin(self, user): return self.store.is_server_admin(user) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 5041828f1..eddd88977 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -36,7 +36,8 @@ class Codes(object): CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" MISSING_PARAM = "M_MISSING_PARAM", - TOO_LARGE = "M_TOO_LARGE" + TOO_LARGE = "M_TOO_LARGE", + EXCLUSIVE = "M_EXCLUSIVE" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 842f075fe..4c15e57fa 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -37,24 +37,15 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def create_association(self, user_id, room_alias, room_id, servers=None): - - # TODO(erikj): Do auth. + def _create_association(self, room_alias, room_id, servers=None): + # general association creation for both human users and app services if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") # TODO(erikj): Change this. # TODO(erikj): Add transactions. - # TODO(erikj): Check if there is a current association. - - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: - raise SynapseError( - 400, "This alias is reserved by an application service." - ) - if not servers: servers = yield self.store.get_joined_hosts_for_room(room_id) @@ -67,6 +58,33 @@ class DirectoryHandler(BaseHandler): servers ) + + @defer.inlineCallbacks + def create_association(self, user_id, room_alias, room_id, servers=None): + # association creation for human users + # TODO(erikj): Do user auth. + + is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) + if is_claimed: + raise SynapseError( + 400, "This alias is reserved by an application service.", + errcode=Codes.EXCLUSIVE + ) + yield self._create_association(room_alias, room_id, servers) + + + @defer.inlineCallbacks + def create_appservice_association(self, service, room_alias, room_id, + servers=None): + if not service.is_interested_in_alias(room_alias.to_string()): + raise SynapseError( + 400, "This application service has not reserved" + " this kind of alias.", errcode=Codes.EXCLUSIVE + ) + + # association creation for app services + yield self._create_association(room_alias, room_id, servers) + @defer.inlineCallbacks def delete_association(self, user_id, room_alias): # TODO Check if server admin @@ -77,7 +95,8 @@ class DirectoryHandler(BaseHandler): is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) if is_claimed: raise SynapseError( - 400, "This alias is reserved by an application service." + 400, "This alias is reserved by an application service.", + errcode=Codes.EXCLUSIVE ) room_id = yield self.store.delete_room_alias(room_alias) diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8f65efec5..f7e910bb4 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -45,8 +45,6 @@ class ClientDirectoryServer(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_alias): - user, client = yield self.auth.get_user_by_req(request) - content = _parse_json(request) if not "room_id" in content: raise SynapseError(400, "Missing room_id key", @@ -70,16 +68,25 @@ class ClientDirectoryServer(ClientV1RestServlet): dir_handler = self.handlers.directory_handler try: - user_id = user.to_string() - yield dir_handler.create_association( - user_id, room_alias, room_id, servers + # try to auth as a user + user, client = yield self.auth.get_user_by_req(request) + try: + user_id = user.to_string() + yield dir_handler.create_association( + user_id, room_alias, room_id, servers + ) + yield dir_handler.send_room_alias_update_event(user_id, room_id) + except SynapseError as e: + raise e + except: + logger.exception("Failed to create association") + raise + except AuthError: + # try to auth as an application service + service = yield self.auth.get_appservice_by_req(request) + yield dir_handler.create_appservice_association( + service, room_alias, room_id, servers ) - yield dir_handler.send_room_alias_update_event(user_id, room_id) - except SynapseError as e: - raise e - except: - logger.exception("Failed to create association") - raise defer.returnValue((200, {})) From c3ae8def755a043283e945e6653970599d227a43 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 11:32:07 +0000 Subject: [PATCH 305/630] Grant ASes the ability to delete aliases in their own namespace. --- synapse/handlers/directory.py | 28 +++++++++++++++++++++----- synapse/rest/client/v1/directory.py | 31 +++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 4c15e57fa..c3c95996e 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -87,10 +87,9 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def delete_association(self, user_id, room_alias): - # TODO Check if server admin + # association deletion for human users - if not self.hs.is_mine(room_alias): - raise SynapseError(400, "Room alias must be local") + # TODO Check if server admin is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) if is_claimed: @@ -99,10 +98,29 @@ class DirectoryHandler(BaseHandler): errcode=Codes.EXCLUSIVE ) + yield self._delete_association(room_alias) + + @defer.inlineCallbacks + def delete_appservice_association(self, service, room_alias): + if not service.is_interested_in_alias(room_alias.to_string()): + raise SynapseError( + 400, + "This application service has not reserved this kind of alias", + errcode=Codes.EXCLUSIVE + ) + yield self._delete_association(room_alias) + + @defer.inlineCallbacks + def _delete_association(self, room_alias): + if not self.hs.is_mine(room_alias): + raise SynapseError(400, "Room alias must be local") + room_id = yield self.store.delete_room_alias(room_alias) - if room_id: - yield self._update_room_alias_events(user_id, room_id) + # TODO - Looks like _update_room_alias_event has never been implemented + # if room_id: + # yield self._update_room_alias_events(user_id, room_id) + @defer.inlineCallbacks def get_association(self, room_alias): diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index f7e910bb4..8e83548bb 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -87,24 +87,51 @@ class ClientDirectoryServer(ClientV1RestServlet): yield dir_handler.create_appservice_association( service, room_alias, room_id, servers ) + logger.info( + "Application service at %s created alias %s pointing to %s", + service.url, + room_alias.to_string(), + room_id + ) defer.returnValue((200, {})) @defer.inlineCallbacks def on_DELETE(self, request, room_alias): + dir_handler = self.handlers.directory_handler + + try: + service = yield self.auth.get_appservice_by_req(request) + room_alias = RoomAlias.from_string(room_alias) + yield dir_handler.delete_appservice_association( + service, room_alias + ) + logger.info( + "Application service at %s deleted alias %s", + service.url, + room_alias.to_string() + ) + defer.returnValue((200, {})) + except AuthError: + # fallback to default user behaviour if they aren't an AS + pass + user, client = yield self.auth.get_user_by_req(request) is_admin = yield self.auth.is_server_admin(user) if not is_admin: raise AuthError(403, "You need to be a server admin") - dir_handler = self.handlers.directory_handler - room_alias = RoomAlias.from_string(room_alias) yield dir_handler.delete_association( user.to_string(), room_alias ) + logger.info( + "User %s deleted alias %s", + user.to_string(), + room_alias.to_string() + ) defer.returnValue((200, {})) From 0995810273ddbc7e5e89b56ad3768fa8f0583bc4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 11:45:19 +0000 Subject: [PATCH 306/630] Pyflakes: unused variable. --- synapse/handlers/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index c3c95996e..22c6391a1 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -115,7 +115,7 @@ class DirectoryHandler(BaseHandler): if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") - room_id = yield self.store.delete_room_alias(room_alias) + yield self.store.delete_room_alias(room_alias) # TODO - Looks like _update_room_alias_event has never been implemented # if room_id: From c78b5fb1f16957f5398517fa5a99dcb370992b26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 13:52:16 +0000 Subject: [PATCH 307/630] Make seen_ids a set --- synapse/handlers/federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9d0ce9aa5..1b709e5e2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -130,7 +130,9 @@ class FederationHandler(BaseHandler): if auth_chain: event_ids |= {e.event_id for e in auth_chain} - seen_ids = (yield self.store.have_events(event_ids)).keys() + seen_ids = set( + (yield self.store.have_events(event_ids)).keys() + ) if state and auth_chain is not None: # If we have any state or auth_chain given to us by the replication From e890ce223c0c7798efc9961ca1e4c4870cfc7e2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 14:16:50 +0000 Subject: [PATCH 308/630] Don't query auth if the only difference is events that were rejected due to auth. --- synapse/handlers/federation.py | 118 +++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 51 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1b709e5e2..b13b7c770 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -791,12 +791,12 @@ class FederationHandler(BaseHandler): @log_function def do_auth(self, origin, event, context, auth_events): # Check if we have all the auth events. - res = yield self.store.have_events( + have_events = yield self.store.have_events( [e_id for e_id, _ in event.auth_events] ) event_auth_events = set(e_id for e_id, _ in event.auth_events) - seen_events = set(res.keys()) + seen_events = set(have_events.keys()) missing_auth = event_auth_events - seen_events @@ -839,6 +839,11 @@ class FederationHandler(BaseHandler): auth_events[(e.type, e.state_key)] = e except AuthError: pass + + have_events = yield self.store.have_events( + [e_id for e_id, _ in event.auth_events] + ) + seen_events = set(have_events.keys()) except: # FIXME: logger.exception("Failed to get auth chain") @@ -852,64 +857,75 @@ class FederationHandler(BaseHandler): # Do auth conflict res. logger.debug("Different auth: %s", different_auth) - # 1. Get what we think is the auth chain. - auth_ids = self.auth.compute_auth_events( - event, context.current_state - ) - local_auth_chain = yield self.store.get_auth_chain(auth_ids) + # Only do auth resolution if we have something new to say. + # We can't rove an auth failure. + do_resolution = False + for e_id in different_auth: + if e_id in have_events: + if have_events[e_id] != RejectedReason.AUTH_ERROR: + do_resolution = True + break - try: - # 2. Get remote difference. - result = yield self.replication_layer.query_auth( - origin, - event.room_id, - event.event_id, - local_auth_chain, + if do_resolution: + # 1. Get what we think is the auth chain. + auth_ids = self.auth.compute_auth_events( + event, context.current_state ) + local_auth_chain = yield self.store.get_auth_chain(auth_ids) - seen_remotes = yield self.store.have_events( - [e.event_id for e in result["auth_chain"]] - ) + try: + # 2. Get remote difference. + result = yield self.replication_layer.query_auth( + origin, + event.room_id, + event.event_id, + local_auth_chain, + ) - # 3. Process any remote auth chain events we haven't seen. - for ev in result["auth_chain"]: - if ev.event_id in seen_remotes.keys(): - continue + seen_remotes = yield self.store.have_events( + [e.event_id for e in result["auth_chain"]] + ) - if ev.event_id == event.event_id: - continue + # 3. Process any remote auth chain events we haven't seen. + for ev in result["auth_chain"]: + if ev.event_id in seen_remotes.keys(): + continue - try: - auth_ids = [e_id for e_id, _ in ev.auth_events] - auth = { - (e.type, e.state_key): e for e in result["auth_chain"] - if e.event_id in auth_ids - } - ev.internal_metadata.outlier = True + if ev.event_id == event.event_id: + continue - logger.debug( - "do_auth %s different_auth: %s", - event.event_id, e.event_id - ) + try: + auth_ids = [e_id for e_id, _ in ev.auth_events] + auth = { + (e.type, e.state_key): e + for e in result["auth_chain"] + if e.event_id in auth_ids + } + ev.internal_metadata.outlier = True - yield self._handle_new_event( - origin, ev, auth_events=auth - ) + logger.debug( + "do_auth %s different_auth: %s", + event.event_id, e.event_id + ) - if ev.event_id in event_auth_events: - auth_events[(ev.type, ev.state_key)] = ev - except AuthError: - pass + yield self._handle_new_event( + origin, ev, auth_events=auth + ) - except: - # FIXME: - logger.exception("Failed to query auth chain") + if ev.event_id in event_auth_events: + auth_events[(ev.type, ev.state_key)] = ev + except AuthError: + pass - # 4. Look at rejects and their proofs. - # TODO. + except: + # FIXME: + logger.exception("Failed to query auth chain") - context.current_state.update(auth_events) - context.state_group = None + # 4. Look at rejects and their proofs. + # TODO. + + context.current_state.update(auth_events) + context.state_group = None try: self.auth.check(event, auth_events=auth_events) @@ -1028,9 +1044,9 @@ class FederationHandler(BaseHandler): for e in base_remote_rejected: reason = yield self.store.get_rejection_reason(e.event_id) if reason is None: - # FIXME: ERRR?! - logger.warn("Could not find reason for %s", e.event_id) - raise RuntimeError("Could not find reason for %s" % e.event_id) + # TODO: e is not in the current state, so we should + # construct some proof of that. + continue reason_map[e.event_id] = reason From 0cd66885e3ff7828282cc03dd8189763fdb7b927 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 14:38:04 +0000 Subject: [PATCH 309/630] Move delta/v13.sql to delta/v12.sql --- synapse/storage/schema/delta/v12.sql | 11 +++++++++++ synapse/storage/schema/delta/v13.sql | 24 ------------------------ 2 files changed, 11 insertions(+), 24 deletions(-) delete mode 100644 synapse/storage/schema/delta/v13.sql diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index 16c2258ca..302d958db 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -52,3 +52,14 @@ CREATE TABLE IF NOT EXISTS push_rules ( ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); + +CREATE TABLE IF NOT EXISTS user_filters( + user_id TEXT, + filter_id INTEGER, + filter_json TEXT, + FOREIGN KEY(user_id) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); diff --git a/synapse/storage/schema/delta/v13.sql b/synapse/storage/schema/delta/v13.sql deleted file mode 100644 index beb39ca20..000000000 --- a/synapse/storage/schema/delta/v13.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright 2015 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS user_filters( - user_id TEXT, - filter_id INTEGER, - filter_json TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) -); - -CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id -); From cc0532a4bf328478d508242d830433a33c4f22b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 15:16:26 +0000 Subject: [PATCH 310/630] Explicitly list the RejectedReasons that we can prove --- synapse/handlers/federation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b13b7c770..0f9c82fd0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -860,9 +860,14 @@ class FederationHandler(BaseHandler): # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False + + provable = [ + RejectedReason.NOT_ANCESTOR, RejectedReason.NOT_ANCESTOR, + ] + for e_id in different_auth: if e_id in have_events: - if have_events[e_id] != RejectedReason.AUTH_ERROR: + if have_events[e_id] in provable: do_resolution = True break From 55a186485c8def40e578201bb33457c9b7df9dde Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 6 Feb 2015 15:58:40 +0000 Subject: [PATCH 311/630] SYN-258: get_recent_events_for_room only accepts stream tokens, convert the topological token to a stream token before passing it to get_recent_events_for_room --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 962686f4b..439164ae3 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,15 +298,17 @@ class SyncHandler(BaseHandler): load_limit = max(sync_config.limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key + end_key = room_key while limited and len(recents) < sync_config.limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, - end_token=room_key, + end_token=end_key, ) (room_key, _) = keys + end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents From 20db147ef377cefb2b5cc138c2bd95291e26f3cc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 6 Feb 2015 15:58:40 +0000 Subject: [PATCH 312/630] SYN-258: get_recent_events_for_room only accepts stream tokens, convert the topological token to a stream token before passing it to get_recent_events_for_room --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 962686f4b..439164ae3 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,15 +298,17 @@ class SyncHandler(BaseHandler): load_limit = max(sync_config.limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key + end_key = room_key while limited and len(recents) < sync_config.limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, - end_token=room_key, + end_token=end_key, ) (room_key, _) = keys + end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents From c4e3029d55c228375725ccacd326e1ee5cc8dd73 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 16:08:13 +0000 Subject: [PATCH 313/630] Add cache layer to state group resolution --- synapse/state.py | 63 ++++++++++++++++++++++++++++++++++++++++ synapse/util/__init__.py | 10 ++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index 695a5e7ac..c45bab585 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -43,14 +43,30 @@ AuthEventTypes = ( ) +class _StateCacheEntry(object): + def __init__(self, state, state_group, ts): + self.state = state + self.state_group = state_group + self.ts = ts + + class StateHandler(object): """ Responsible for doing state conflict resolution. """ def __init__(self, hs): + self.clock = hs.get_clock() self.store = hs.get_datastore() self.hs = hs + # set of event_ids -> _StateCacheEntry. + self._state_cache = {} + + def f(): + self._prune_cache() + + self.clock.looping_call(f, 10*1000) + @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): """ Returns the current state for the room as a list. This is done by @@ -70,6 +86,11 @@ class StateHandler(object): for e_id, _, _ in events ] + cache = self._state_cache.get(set(event_ids), None) + if cache: + cache.ts = self.clock.time_msec() + defer.returnValue(cache.state_group, cache.state) + res = yield self.resolve_state_groups(event_ids) if event_type: @@ -177,6 +198,11 @@ class StateHandler(object): """ logger.debug("resolve_state_groups event_ids %s", event_ids) + cache = self._state_cache.get(set(event_ids), None) + if cache and cache.state_group: + cache.ts = self.clock.time_msec() + defer.returnValue(cache.state_group, cache.state) + state_groups = yield self.store.get_state_groups( event_ids ) @@ -200,6 +226,14 @@ class StateHandler(object): else: prev_states = [] + cache = _StateCacheEntry( + state=state, + state_group=name, + ts=self.clock.time_msec() + ) + + self._state_cache[set(event_ids)] = cache + defer.returnValue((name, state, prev_states)) state = {} @@ -245,6 +279,14 @@ class StateHandler(object): new_state = unconflicted_state new_state.update(resolved_state) + cache = _StateCacheEntry( + state=new_state, + state_group=None, + ts=self.clock.time_msec() + ) + + self._state_cache[set(event_ids)] = cache + defer.returnValue((None, new_state, prev_states)) @log_function @@ -328,3 +370,24 @@ class StateHandler(object): return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() return sorted(events, key=key_func) + + def _prune_cache(self): + now = self.clock.time_msec() + + if len(self._state_cache) > 100: + sorted_entries = sorted( + self._state_cache.items(), + key=lambda k, v: v.ts, + ) + + for k, _ in sorted_entries[100:]: + self._state_cache.pop(k) + + keys_to_delete = set() + + for key, cache_entry in self._state_cache.items(): + if now - cache_entry.ts > 60*1000: + keys_to_delete.add(key) + + for k in keys_to_delete: + self._state_cache.pop(k) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 4e837a918..1fd5ba578 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -15,7 +15,7 @@ from synapse.util.logcontext import LoggingContext -from twisted.internet import reactor +from twisted.internet import reactor, task import time @@ -35,6 +35,14 @@ class Clock(object): """Returns the current system time in miliseconds since epoch.""" return self.time() * 1000 + def looping_call(self, f, msec): + l = task.LoopingCall(f) + l.start(msec/1000.0, now=False) + return l + + def looping_call(self, loop): + loop.stop() + def call_later(self, delay, callback): current_context = LoggingContext.current_context() From b4886264a36ae85bcf922f7d2bc56f0ae59b95e6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 16:17:05 +0000 Subject: [PATCH 314/630] Bugfix cache layer --- synapse/state.py | 31 +++++++++++++++++++++---------- synapse/util/__init__.py | 2 +- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index c45bab585..7523573f2 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -86,18 +86,19 @@ class StateHandler(object): for e_id, _, _ in events ] - cache = self._state_cache.get(set(event_ids), None) + cache = self._state_cache.get(frozenset(event_ids), None) if cache: cache.ts = self.clock.time_msec() - defer.returnValue(cache.state_group, cache.state) - - res = yield self.resolve_state_groups(event_ids) + state = cache.state + else: + res = yield self.resolve_state_groups(event_ids) + state = res[1] if event_type: - defer.returnValue(res[1].get((event_type, state_key))) + defer.returnValue(state.get((event_type, state_key))) return - defer.returnValue(res[1].values()) + defer.returnValue(state.values()) @defer.inlineCallbacks def compute_event_context(self, event, old_state=None): @@ -198,10 +199,16 @@ class StateHandler(object): """ logger.debug("resolve_state_groups event_ids %s", event_ids) - cache = self._state_cache.get(set(event_ids), None) + cache = self._state_cache.get(frozenset(event_ids), None) if cache and cache.state_group: cache.ts = self.clock.time_msec() - defer.returnValue(cache.state_group, cache.state) + prev_state = cache.state.get((event_type, state_key), None) + if prev_state: + prev_state = prev_state.event_id + prev_states = [prev_state] + else: + prev_states = [] + defer.returnValue((cache.state_group, cache.state, prev_states)) state_groups = yield self.store.get_state_groups( event_ids @@ -232,7 +239,7 @@ class StateHandler(object): ts=self.clock.time_msec() ) - self._state_cache[set(event_ids)] = cache + self._state_cache[frozenset(event_ids)] = cache defer.returnValue((name, state, prev_states)) @@ -285,7 +292,7 @@ class StateHandler(object): ts=self.clock.time_msec() ) - self._state_cache[set(event_ids)] = cache + self._state_cache[frozenset(event_ids)] = cache defer.returnValue((None, new_state, prev_states)) @@ -372,6 +379,8 @@ class StateHandler(object): return sorted(events, key=key_func) def _prune_cache(self): + logger.debug("_prune_cache. before len: ", len(self._state_cache)) + now = self.clock.time_msec() if len(self._state_cache) > 100: @@ -391,3 +400,5 @@ class StateHandler(object): for k in keys_to_delete: self._state_cache.pop(k) + + logger.debug("_prune_cache. after len: ", len(self._state_cache)) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 1fd5ba578..fee76b0a9 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -40,7 +40,7 @@ class Clock(object): l.start(msec/1000.0, now=False) return l - def looping_call(self, loop): + def stop_looping_call(self, loop): loop.stop() def call_later(self, delay, callback): From 5bf318e9a60dd05120e9b78b49042337d38ec414 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 16:52:22 +0000 Subject: [PATCH 315/630] Bug fixes. --- synapse/app/homeserver.py | 2 + synapse/state.py | 82 ++++++++++++++++++++++++++------------- 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f20ccfb5b..3a128af5f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -274,6 +274,8 @@ def setup(): hs.get_pusherpool().start() + hs.get_state_handler().start_caching() + if config.daemonize: print config.pid_file daemon = Daemonize( diff --git a/synapse/state.py b/synapse/state.py index 7523573f2..49ac09863 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -43,6 +43,10 @@ AuthEventTypes = ( ) +SIZE_OF_CACHE = 1000 +EVICTION_TIMEOUT_SECONDS = 20 + + class _StateCacheEntry(object): def __init__(self, state, state_group, ts): self.state = state @@ -59,13 +63,22 @@ class StateHandler(object): self.store = hs.get_datastore() self.hs = hs - # set of event_ids -> _StateCacheEntry. + # dict of set of event_ids -> _StateCacheEntry. + self._state_cache = None + + def start_caching(self): + logger.debug("start_caching") + self._state_cache = {} def f(): - self._prune_cache() + logger.debug("Pruning") + try: + self._prune_cache() + except: + logger.exception("Prune") - self.clock.looping_call(f, 10*1000) + self.clock.looping_call(f, 5*1000) @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): @@ -86,7 +99,10 @@ class StateHandler(object): for e_id, _, _ in events ] - cache = self._state_cache.get(frozenset(event_ids), None) + cache = None + if self._state_cache is not None: + cache = self._state_cache.get(frozenset(event_ids), None) + if cache: cache.ts = self.clock.time_msec() state = cache.state @@ -199,16 +215,19 @@ class StateHandler(object): """ logger.debug("resolve_state_groups event_ids %s", event_ids) - cache = self._state_cache.get(frozenset(event_ids), None) - if cache and cache.state_group: - cache.ts = self.clock.time_msec() - prev_state = cache.state.get((event_type, state_key), None) - if prev_state: - prev_state = prev_state.event_id - prev_states = [prev_state] - else: - prev_states = [] - defer.returnValue((cache.state_group, cache.state, prev_states)) + if self._state_cache is not None: + cache = self._state_cache.get(frozenset(event_ids), None) + if cache and cache.state_group: + cache.ts = self.clock.time_msec() + prev_state = cache.state.get((event_type, state_key), None) + if prev_state: + prev_state = prev_state.event_id + prev_states = [prev_state] + else: + prev_states = [] + defer.returnValue( + (cache.state_group, cache.state, prev_states) + ) state_groups = yield self.store.get_state_groups( event_ids @@ -233,15 +252,16 @@ class StateHandler(object): else: prev_states = [] - cache = _StateCacheEntry( - state=state, - state_group=name, - ts=self.clock.time_msec() - ) + if self._state_cache is not None: + cache = _StateCacheEntry( + state=state, + state_group=name, + ts=self.clock.time_msec() + ) - self._state_cache[frozenset(event_ids)] = cache + self._state_cache[frozenset(event_ids)] = cache - defer.returnValue((name, state, prev_states)) + defer.returnValue((name, state, prev_states)) state = {} for group, g_state in state_groups.items(): @@ -292,7 +312,8 @@ class StateHandler(object): ts=self.clock.time_msec() ) - self._state_cache[frozenset(event_ids)] = cache + if self._state_cache is not None: + self._state_cache[frozenset(event_ids)] = cache defer.returnValue((None, new_state, prev_states)) @@ -379,26 +400,33 @@ class StateHandler(object): return sorted(events, key=key_func) def _prune_cache(self): - logger.debug("_prune_cache. before len: ", len(self._state_cache)) + logger.debug("_prune_cache") + logger.debug( + "_prune_cache. before len: %d", + len(self._state_cache.keys()) + ) now = self.clock.time_msec() - if len(self._state_cache) > 100: + if len(self._state_cache.keys()) > SIZE_OF_CACHE: sorted_entries = sorted( self._state_cache.items(), key=lambda k, v: v.ts, ) - for k, _ in sorted_entries[100:]: + for k, _ in sorted_entries[SIZE_OF_CACHE:]: self._state_cache.pop(k) keys_to_delete = set() for key, cache_entry in self._state_cache.items(): - if now - cache_entry.ts > 60*1000: + if now - cache_entry.ts > EVICTION_TIMEOUT_SECONDS*1000: keys_to_delete.add(key) for k in keys_to_delete: self._state_cache.pop(k) - logger.debug("_prune_cache. after len: ", len(self._state_cache)) + logger.debug( + "_prune_cache. after len: %d", + len(self._state_cache.keys()) + ) From 73a680b2a87b309bf05b4afe2685aaba70201305 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Feb 2015 17:10:04 +0000 Subject: [PATCH 316/630] Add errcodes for appservice registrations. --- synapse/handlers/register.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 60821edb0..10ab9bc99 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,7 +18,8 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - SynapseError, RegistrationError, InvalidCaptchaError, CodeMessageException + AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError, + CodeMessageException ) from ._base import BaseHandler import synapse.util.stringutils as stringutils @@ -111,10 +112,11 @@ class RegistrationHandler(BaseHandler): user_id = user.to_string() service = yield self.store.get_app_service_by_token(as_token) if not service: - raise SynapseError(403, "Invalid application service token.") + raise AuthError(403, "Invalid application service token.") if not service.is_interested_in_user(user_id): raise SynapseError( - 400, "Invalid user localpart for this application service." + 400, "Invalid user localpart for this application service.", + errcode=Codes.EXCLUSIVE ) token = self._generate_token(user_id) yield self.store.register( @@ -181,7 +183,8 @@ class RegistrationHandler(BaseHandler): ] if len(interested_services) > 0: raise SynapseError( - 400, "This user ID is reserved by an application service." + 400, "This user ID is reserved by an application service.", + errcode=Codes.EXCLUSIVE ) def _generate_token(self, user_id): From 03c25ebeaeceb737d35db480ac8d1e3ed5f81988 Mon Sep 17 00:00:00 2001 From: TurnedToDust Date: Fri, 6 Feb 2015 22:28:21 -0700 Subject: [PATCH 317/630] Update to README.rst Added Documentation regarding ArchLinux --- README.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/README.rst b/README.rst index e1fa4d75f..dc1c14daa 100644 --- a/README.rst +++ b/README.rst @@ -96,6 +96,11 @@ Installing prerequisites on Ubuntu or Debian:: $ sudo apt-get install build-essential python2.7-dev libffi-dev \ python-pip python-setuptools sqlite3 \ libssl-dev python-virtualenv libjpeg-dev + +Installing prerequisites on ArchLinux:: + + $ sudo pacman -S base-devel python2 python-pip \ + python-setuptools python-virtualenv sqlite3 Installing prerequisites on Mac OS X:: @@ -148,6 +153,31 @@ failing, e.g.:: On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you will need to export CFLAGS=-Qunused-arguments. +ArchLinux +-------------- +ArchLinux with the default installation of prerequisites, and your System itself. The installation may encounter a few Hiccups. + +python2.7 is Needed and I believe by default Arch uses Python3. + +pip is outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ): + - $ sudo pip2.7 install --upgrade pip + +You also may need to call 2.7 again during the install request: + - $ sudo pip2.7 install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + +If you encounter an error with lib bcrypt Causing an Wrong Elf Class: ELFCLASS32 (x64 Systems): +you need to remove py-bcrypt itself and then reinstall it to correctly compile under your architecture + - $ sudo pip2.7 uninstall py-bcrypt + - $ sudo pip2.7 install py-bcrypt + +During setup of homeserver you need to call (depending) python2.7 directly again: + - $ sudo python2.7 -m synapse.app.homeserver \ + --server-name machine.my.domain.name \ + --config-path homeserver.yaml \ + --generate-config + +Substituting your host and domain name as appropriate. + Windows Install --------------- Synapse can be installed on Cygwin. It requires the following Cygwin packages: @@ -207,6 +237,14 @@ fix try re-installing from PyPI or directly from $ # Install from github $ pip install --user https://github.com/pyca/pynacl/tarball/master +ArchLinux +--------- +If running $ synctl start , causes the following error + "subprocess.CalledProcessError: Command '['python', '-m', 'synapse.app.homeserver', '--daemonize', '-c', 'homeserver.yaml', '--pid-file', 'homeserver.pid']' returned non-zero exit status 1" + +You need to call 2.7 again by directly using + - $ python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml --pid-file homeserver.pid + Homeserver Development ====================== From 34c39398faddae414f50838c744cbc2e4d934a56 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 12:55:13 +0000 Subject: [PATCH 318/630] i hate weakly typed languages --- synapse/rest/media/v1/base_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 688e7376a..d44d5f129 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -82,7 +82,7 @@ class BaseMediaResource(Resource): raise SynapseError( 404, "Invalid media id token %r" % (request.postpath,), - Codes.UNKKOWN, + Codes.UNKNOWN, ) @staticmethod From e117bc3fc582520e7d7adb86015ad814195cd286 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 12:56:21 +0000 Subject: [PATCH 319/630] thou shalt specify a content-length --- synapse/rest/media/v1/media_repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 61ed90f39..9ca4d884d 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -34,6 +34,7 @@ class MediaRepositoryResource(Resource): => POST /_matrix/media/v1/upload HTTP/1.1 Content-Type: + Content-Length: From f02bf64d0e25cc3264c7f6767e9fbc008d7cc3d3 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 12:59:09 +0000 Subject: [PATCH 320/630] create identicons for new users by default as default avatars, and provide script to update existing avatarless users --- scripts/make_identicons.pl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 scripts/make_identicons.pl diff --git a/scripts/make_identicons.pl b/scripts/make_identicons.pl new file mode 100755 index 000000000..172f63eba --- /dev/null +++ b/scripts/make_identicons.pl @@ -0,0 +1,24 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use DBI; +use DBD::SQLite; +use JSON; + +my $dbh = DBI->connect("dbi:SQLite:dbname=homeserver.db","","") || die DBI->error; + +my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die DBI->error; + +foreach (@$res) { + my ($token, $mxid) = ($_->[0], $_->[1]); + my ($user_id) = ($mxid =~ m/@(.*):/); + my ($url) = $dbh->selectrow_array("select avatar_url from profiles where user_id=?", undef, $user_id); + if (!$url || $url =~ /#auto$/) { + `curl -o tmp.png "http://localhost:8008/_matrix/media/v1/identicon?name=${mxid}&width=320&height=320"`; + my $json = `curl -X POST -H "Content-Type: image/png" -T "tmp.png" http://localhost:8008/_matrix/media/v1/upload?access_token=$token`; + my $content_uri = from_json($json)->{content_uri}; + `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' http://localhost:8008/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`; + } +} From 582019f870adbc4a8a8a9ef97b527e0fead77761 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 13:32:14 +0000 Subject: [PATCH 321/630] ...and here's the actual impl. git fail. --- synapse/handlers/register.py | 14 ++++++ synapse/rest/media/v1/upload_resource.py | 57 +++++++++++++----------- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 66a89c10b..2b9d86008 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -99,6 +99,20 @@ class RegistrationHandler(BaseHandler): raise RegistrationError( 500, "Cannot generate user ID.") + # create a default avatar for the user + # XXX: ideally clients would explicitly specify one, but given they don't + # and we want consistent and pretty identicons for random users, we'll + # do it here. + auth_user = UserID.from_string(user_id) + identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) + upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) + content_uri = yield upload_resource.create_content( + "image/png", None, identicon_bytes, len(identicon_bytes), auth_user + ) + profile_handler = self.hs.get_handlers().profile_handler + profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index b939a30e1..5b4278233 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -38,6 +38,35 @@ class UploadResource(BaseMediaResource): def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET + + @defer.inlineCallbacks + def create_content(self, media_type, upload_name, content, content_length, auth_user): + media_id = random_string(24) + + fname = self.filepaths.local_media_filepath(media_id) + self._makedirs(fname) + + # This shouldn't block for very long because the content will have + # already been uploaded at this point. + with open(fname, "wb") as f: + f.write(content) + + yield self.store.store_local_media( + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + ) + media_info = { + "media_type": media_type, + "media_length": content_length, + } + + yield self._generate_local_thumbnails(media_id, media_info) + + defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @defer.inlineCallbacks def _async_render_POST(self, request): @@ -70,32 +99,10 @@ class UploadResource(BaseMediaResource): # disposition = headers.getRawHeaders("Content-Disposition")[0] # TODO(markjh): parse content-dispostion - media_id = random_string(24) - - fname = self.filepaths.local_media_filepath(media_id) - self._makedirs(fname) - - # This shouldn't block for very long because the content will have - # already been uploaded at this point. - with open(fname, "wb") as f: - f.write(request.content.read()) - - yield self.store.store_local_media( - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=None, - media_length=content_length, - user_id=auth_user, + content_uri = yield self.create_content( + media_type, None, request.content.read(), + content_length, auth_user ) - media_info = { - "media_type": media_type, - "media_length": content_length, - } - - yield self._generate_local_thumbnails(media_id, media_info) - - content_uri = "mxc://%s/%s" % (self.server_name, media_id) respond_with_json( request, 200, {"content_uri": content_uri}, send_cors=True From 0c0ae2e886821d323f0dbd6254c2b8ed7c61e8be Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 20:24:46 +0000 Subject: [PATCH 322/630] clean up TurnedToDust's ArchLinux notes a bit --- README.rst | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index dc1c14daa..5eebe5b72 100644 --- a/README.rst +++ b/README.rst @@ -154,29 +154,37 @@ On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you will need to export CFLAGS=-Qunused-arguments. ArchLinux --------------- -ArchLinux with the default installation of prerequisites, and your System itself. The installation may encounter a few Hiccups. +--------- -python2.7 is Needed and I believe by default Arch uses Python3. +Installation on ArchLinux may encounter a few hiccups as Arch defaults to +python 3, but synapse currently assumes python 2.7 by default. -pip is outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ): - - $ sudo pip2.7 install --upgrade pip +pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: + + $ sudo pip2.7 install --upgrade pip -You also may need to call 2.7 again during the install request: - - $ sudo pip2.7 install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master +You also may need to explicitly specify python 2.7 again during the install +request:: + + $ pip2.7 install --process-dependency-links \ + https://github.com/matrix-org/synapse/tarball/master -If you encounter an error with lib bcrypt Causing an Wrong Elf Class: ELFCLASS32 (x64 Systems): -you need to remove py-bcrypt itself and then reinstall it to correctly compile under your architecture - - $ sudo pip2.7 uninstall py-bcrypt - - $ sudo pip2.7 install py-bcrypt +If you encounter an error with lib bcrypt causing an Wrong ELF Class: +ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly +compile it under the right architecture. (This should not be needed if +installing under virtualenv):: + + $ sudo pip2.7 uninstall py-bcrypt + $ sudo pip2.7 install py-bcrypt -During setup of homeserver you need to call (depending) python2.7 directly again: - - $ sudo python2.7 -m synapse.app.homeserver \ +During setup of homeserver you need to call python2.7 directly again:: + + $ python2.7 -m synapse.app.homeserver \ --server-name machine.my.domain.name \ --config-path homeserver.yaml \ --generate-config -Substituting your host and domain name as appropriate. +...substituting your host and domain name as appropriate. Windows Install --------------- @@ -239,12 +247,12 @@ fix try re-installing from PyPI or directly from ArchLinux --------- -If running $ synctl start , causes the following error - "subprocess.CalledProcessError: Command '['python', '-m', 'synapse.app.homeserver', '--daemonize', '-c', 'homeserver.yaml', '--pid-file', 'homeserver.pid']' returned non-zero exit status 1" -You need to call 2.7 again by directly using - - $ python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml --pid-file homeserver.pid +If running `$ synctl start` fails wit 'returned non-zero exit status 1', you will need to explicitly call Python2.7 - either running as:: + $ python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml --pid-file homeserver.pid + +...or by editing synctl with the correct python executable. Homeserver Development ====================== From adc4310a733044163f4ccea97fecc347193536e9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 21:13:57 +0000 Subject: [PATCH 323/630] add some options and doc --- scripts/make_identicons.pl | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/make_identicons.pl b/scripts/make_identicons.pl index 172f63eba..cbff63e29 100755 --- a/scripts/make_identicons.pl +++ b/scripts/make_identicons.pl @@ -6,19 +6,34 @@ use warnings; use DBI; use DBD::SQLite; use JSON; +use Getopt::Long; -my $dbh = DBI->connect("dbi:SQLite:dbname=homeserver.db","","") || die DBI->error; +my $db; # = "homeserver.db"; +my $server = "http://localhost:8008"; +my $size = 320; -my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die DBI->error; +GetOptions("db|d=s", \$db, + "server|s=s", \$server, + "width|w=i", \$size) or usage(); + +usage() unless $db; + +my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","") || die $DBI::errstr; + +my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die $DBI::errstr; foreach (@$res) { my ($token, $mxid) = ($_->[0], $_->[1]); my ($user_id) = ($mxid =~ m/@(.*):/); my ($url) = $dbh->selectrow_array("select avatar_url from profiles where user_id=?", undef, $user_id); if (!$url || $url =~ /#auto$/) { - `curl -o tmp.png "http://localhost:8008/_matrix/media/v1/identicon?name=${mxid}&width=320&height=320"`; - my $json = `curl -X POST -H "Content-Type: image/png" -T "tmp.png" http://localhost:8008/_matrix/media/v1/upload?access_token=$token`; + `curl -s -o tmp.png "$server/_matrix/media/v1/identicon?name=${mxid}&width=$size&height=$size"`; + my $json = `curl -s -X POST -H "Content-Type: image/png" -T "tmp.png" $server/_matrix/media/v1/upload?access_token=$token`; my $content_uri = from_json($json)->{content_uri}; - `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' http://localhost:8008/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`; + `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' $server/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`; } } + +sub usage { + die "usage: ./make-identicons.pl\n\t-d database [e.g. homeserver.db]\n\t-s homeserver (default: http://localhost:8008)\n\t-w identicon size in pixels (default 320)"; +} \ No newline at end of file From 37b6b880ef084f2cdd3cce1a275aa46764131c91 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 7 Feb 2015 21:24:08 +0000 Subject: [PATCH 324/630] don't give up if we can't create default avatars during tests --- synapse/handlers/register.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 2b9d86008..4f06c487b 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -103,15 +103,18 @@ class RegistrationHandler(BaseHandler): # XXX: ideally clients would explicitly specify one, but given they don't # and we want consistent and pretty identicons for random users, we'll # do it here. - auth_user = UserID.from_string(user_id) - identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) - upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) - identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) - content_uri = yield upload_resource.create_content( - "image/png", None, identicon_bytes, len(identicon_bytes), auth_user - ) - profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + try: + auth_user = UserID.from_string(user_id) + identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) + upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) + content_uri = yield upload_resource.create_content( + "image/png", None, identicon_bytes, len(identicon_bytes), auth_user + ) + profile_handler = self.hs.get_handlers().profile_handler + profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + except NotImplementedError: + pass # make tests pass without messing around creating default avatars defer.returnValue((user_id, token)) From 8be07e0db456d9c79d258c7a51482442445fad00 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 8 Feb 2015 00:34:11 +0000 Subject: [PATCH 325/630] kill off fnmatch in favour of word-boundary based push alerts (untested) --- synapse/push/__init__.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 729371529..8e11abfa5 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -22,7 +22,6 @@ import synapse.util.async import baserules import logging -import fnmatch import json import re @@ -130,26 +129,35 @@ class Pusher(object): defer.returnValue(Pusher.DEFAULT_ACTIONS) + @staticmethod + def _glob_to_regexp(glob): + r = re.escape(glob) + r = re.sub(r'\\\*', r'.*', r) + r = re.sub(r'\\\?', r'.', r) + + # handle [abc], [a-z] and [!a-z] style ranges. + r = re.sub(r'\\\[(\\\!|)(.*)\\\]', + lambda x: ('[%s%s]' % (x.group(1) and '^' or '', + re.sub(r'\\\-', '-', x.group(2)))), r) + return r + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: logger.warn("event_match condition with no pattern") return False - pat = condition['pattern'] - - if pat.strip("*?[]") == pat: - # no special glob characters so we assume the user means - # 'contains this string' rather than 'is this string' - pat = "*%s*" % (pat,) - + # XXX: optimisation: cache our pattern regexps + r = r'\b%s\b' % _glob_to_regexp(condition['pattern']) val = _value_for_dotted_key(condition['key'], ev) if val is None: return False - return fnmatch.fnmatch(val.upper(), pat.upper()) + return re.match(r, val, flags=re.IGNORECASE) != None + elif condition['kind'] == 'device': if 'profile_tag' not in condition: return True return condition['profile_tag'] == self.profile_tag + elif condition['kind'] == 'contains_display_name': # This is special because display names can be different # between rooms and so you can't really hard code it in a rule. @@ -159,9 +167,9 @@ class Pusher(object): return False if not display_name: return False - return fnmatch.fnmatch( - ev['content']['body'].upper(), "*%s*" % (display_name.upper(),) - ) + return re.match("\b%s\b" % re.escape(display_name), + ev['content']['body'], flags=re.IGNORECASE) != None + elif condition['kind'] == 'room_member_count': if 'is' not in condition: return False From c2afc2ad90b1121a072765237f5c4dc8d765ddf7 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 8 Feb 2015 00:37:03 +0000 Subject: [PATCH 326/630] oops --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8e11abfa5..7a41c5ece 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -147,7 +147,7 @@ class Pusher(object): logger.warn("event_match condition with no pattern") return False # XXX: optimisation: cache our pattern regexps - r = r'\b%s\b' % _glob_to_regexp(condition['pattern']) + r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) val = _value_for_dotted_key(condition['key'], ev) if val is None: return False From ecb0f7806305b3433d8c2a5a5bb413226f2e90f8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 8 Feb 2015 02:37:35 +0000 Subject: [PATCH 327/630] glob *s should probably be non-greedy --- synapse/push/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 7a41c5ece..07b5f0187 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -132,7 +132,7 @@ class Pusher(object): @staticmethod def _glob_to_regexp(glob): r = re.escape(glob) - r = re.sub(r'\\\*', r'.*', r) + r = re.sub(r'\\\*', r'.*?', r) r = re.sub(r'\\\?', r'.', r) # handle [abc], [a-z] and [!a-z] style ranges. From ac3183caaa66b750996d90c0ac9ed430f623909c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 12:03:37 +0000 Subject: [PATCH 328/630] Register a user account for the AS when the AS registers. Add 'sender' column to AS table. --- synapse/appservice/__init__.py | 3 ++- synapse/handlers/appservice.py | 8 +++++++- synapse/storage/appservice.py | 5 +++-- synapse/storage/schema/application_services.sql | 1 + synapse/storage/schema/delta/v14.sql | 1 + 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 46d46a5a4..fb9bfffe5 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -35,10 +35,11 @@ class ApplicationService(object): NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None, hs_token=None, - txn_id=None): + sender=None, txn_id=None): self.token = token self.url = url self.hs_token = hs_token + self.sender = sender self.namespaces = self._check_namespaces(namespaces) self.txn_id = txn_id diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index fa810b9a9..5071a12eb 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -52,8 +52,14 @@ class ApplicationServicesHandler(object): "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - logger.info("Updating application service info...") + app_service.hs_token = self._generate_hs_token() + + # create a sender for this application service which is used when + # creating rooms, etc.. + account = yield self.hs.get_handlers().registration_handler.register() + app_service.sender = account[0] + yield self.store.update_app_service(app_service) defer.returnValue(app_service) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 3c8bf9ad0..eef77e737 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -130,8 +130,9 @@ class ApplicationServiceStore(SQLBaseStore): return False txn.execute( - "UPDATE application_services SET url=?, hs_token=? WHERE id=?", - (service.url, service.hs_token, as_id,) + "UPDATE application_services SET url=?, hs_token=?, sender=? " + "WHERE id=?", + (service.url, service.hs_token, service.sender, as_id,) ) # cleanup regex txn.execute( diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/application_services.sql index 03b5a10c8..e491ad5ae 100644 --- a/synapse/storage/schema/application_services.sql +++ b/synapse/storage/schema/application_services.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v14.sql index 03b5a10c8..e491ad5ae 100644 --- a/synapse/storage/schema/delta/v14.sql +++ b/synapse/storage/schema/delta/v14.sql @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS application_services( url TEXT, token TEXT, hs_token TEXT, + sender TEXT, UNIQUE(token) ON CONFLICT ROLLBACK ); From 24cc6979fb384ef383309b27d06985ba3a845b2b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 13:46:22 +0000 Subject: [PATCH 329/630] Log when we receive a request, when we send a response and how long it took to process it. --- synapse/app/homeserver.py | 2 +- synapse/http/server.py | 23 +++++++++++++++++++++-- synapse/rest/client/v1/__init__.py | 2 +- synapse/rest/client/v2_alpha/__init__.py | 2 +- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f20ccfb5b..0f175ec3f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -67,7 +67,7 @@ class SynapseHomeServer(HomeServer): return ClientV2AlphaRestResource(self) def build_resource_for_federation(self): - return JsonResource() + return JsonResource(self) def build_resource_for_web_client(self): syweb_path = os.path.dirname(syweb.__file__) diff --git a/synapse/http/server.py b/synapse/http/server.py index 0f6539e1b..6d084fa33 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -69,9 +69,10 @@ class JsonResource(HttpServer, resource.Resource): _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"]) - def __init__(self): + def __init__(self, hs): resource.Resource.__init__(self) + self.clock = hs.get_clock() self.path_regexs = {} def register_path(self, method, path_pattern, callback): @@ -111,6 +112,7 @@ class JsonResource(HttpServer, resource.Resource): This checks if anyone has registered a callback for that method and path. """ + code = None try: # Just say yes to OPTIONS. if request.method == "OPTIONS": @@ -130,6 +132,13 @@ class JsonResource(HttpServer, resource.Resource): urllib.unquote(u).decode("UTF-8") for u in m.groups() ] + logger.info( + "Received request: %s %s", + request.method, request.path + ) + + start = self.clock.time_msec() + code, response = yield path_entry.callback( request, *args @@ -145,9 +154,11 @@ class JsonResource(HttpServer, resource.Resource): logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) else: logger.exception(e) + + code = e.code self._send_response( request, - e.code, + code, cs_exception(e), response_code_message=e.response_code_message ) @@ -158,6 +169,14 @@ class JsonResource(HttpServer, resource.Resource): 500, {"error": "Internal server error"} ) + finally: + code = str(code) if code else "-" + + end = self.clock.time_msec() + logger.info( + "Processed request: %dms %s %s %s", + end-start, code, request.method, request.path + ) def _send_response(self, request, code, response_json_object, response_code_message=None): diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index d8d01cdd1..21876b348 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -25,7 +25,7 @@ class ClientV1RestResource(JsonResource): """A resource for version 1 of the matrix client API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 8f611de3a..bca65f2a6 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -25,7 +25,7 @@ class ClientV2AlphaRestResource(JsonResource): """A resource for version 2 alpha of the matrix client API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod From 5a7dd058184613c70041a61fdbc2ccce104bb500 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 14:14:15 +0000 Subject: [PATCH 330/630] Modify auth.get_user_by_req for authing appservices directly. Add logic to map the appservice token to the autogenned appservice user ID. Add unit tests for all forms of get_user_by_req (user/appservice, valid/bad/missing tokens) --- synapse/api/auth.py | 34 +++--- synapse/storage/appservice.py | 4 +- tests/api/test_auth.py | 139 +++++++++++++++++++++++++ tests/rest/client/v1/test_presence.py | 3 + tests/rest/client/v2_alpha/__init__.py | 4 +- 5 files changed, 164 insertions(+), 20 deletions(-) create mode 100644 tests/api/test_auth.py diff --git a/synapse/api/auth.py b/synapse/api/auth.py index ea8c46172..310a42806 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -302,27 +302,26 @@ class Auth(object): # Check for application service tokens with a user_id override try: - if "user_id" not in request.args: - # This has to be done like this rather than relying on it - # natively throwing because tests use a Mock for the request - # object which doesn't throw :/ - raise KeyError - - masquerade_user_id = request.args["user_id"][0] app_service = yield self.store.get_app_service_by_token( access_token ) if not app_service: - raise AuthError( - 403, "Invalid application service access token" - ) - if not app_service.is_interested_in_user(masquerade_user_id): - raise AuthError( - 403, - "Application service cannot masquerade as this user." - ) + raise KeyError + + user_id = app_service.sender + if "user_id" in request.args: + user_id = request.args["user_id"][0] + if not app_service.is_interested_in_user(user_id): + raise AuthError( + 403, + "Application service cannot masquerade as this user." + ) + + if not user_id: + raise KeyError + defer.returnValue( - (UserID.from_string(masquerade_user_id), ClientInfo("", "")) + (UserID.from_string(user_id), ClientInfo("", "")) ) return except KeyError: @@ -366,8 +365,7 @@ class Auth(object): try: ret = yield self.store.get_user_by_token(token=token) if not ret: - raise StoreError() - + raise StoreError(400, "Unknown token") user_info = { "admin": bool(ret.get("admin", False)), "device_id": ret.get("device_id"), diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index eef77e737..ba31c6859 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -215,6 +215,7 @@ class ApplicationServiceStore(SQLBaseStore): "url": res["url"], "token": as_token, "hs_token": res["hs_token"], + "sender": res["sender"], "namespaces": { ApplicationService.NS_USERS: [], ApplicationService.NS_ALIASES: [], @@ -240,6 +241,7 @@ class ApplicationServiceStore(SQLBaseStore): token=service["token"], url=service["url"], namespaces=service["namespaces"], - hs_token=service["hs_token"] + hs_token=service["hs_token"], + sender=service["sender"] )) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py new file mode 100644 index 000000000..1d8367ce4 --- /dev/null +++ b/tests/api/test_auth.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from tests import unittest +from twisted.internet import defer + +from mock import Mock, NonCallableMock + +from synapse.api.auth import Auth +from synapse.api.errors import AuthError +from synapse.types import UserID + +class AuthTestCase(unittest.TestCase): + + def setUp(self): + self.state_handler = Mock() + self.store = Mock() + + self.hs = Mock() + self.hs.get_datastore = Mock(return_value=self.store) + self.hs.get_state_handler = Mock(return_value=self.state_handler) + self.auth = Auth(self.hs) + + self.test_user = "@foo:bar" + self.test_token = "_test_token_" + + @defer.inlineCallbacks + def test_get_user_by_req_user_valid_token(self): + self.store.get_app_service_by_token = Mock(return_value=None) + user_info = { + "name": self.test_user, + "device_id": "nothing", + "token_id": "ditto", + "admin": False + } + self.store.get_user_by_token = Mock(return_value=user_info) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + (user, info) = yield self.auth.get_user_by_req(request) + self.assertEquals(user.to_string(), self.test_user) + + def test_get_user_by_req_user_bad_token(self): + self.store.get_app_service_by_token = Mock(return_value=None) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) + + def test_get_user_by_req_user_missing_token(self): + self.store.get_app_service_by_token = Mock(return_value=None) + user_info = { + "name": self.test_user, + "device_id": "nothing", + "token_id": "ditto", + "admin": False + } + self.store.get_user_by_token = Mock(return_value=user_info) + + request = Mock(args={}) + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) + + @defer.inlineCallbacks + def test_get_user_by_req_appservice_valid_token(self): + app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + self.store.get_app_service_by_token = Mock(return_value=app_service) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + (user, info) = yield self.auth.get_user_by_req(request) + self.assertEquals(user.to_string(), self.test_user) + + def test_get_user_by_req_appservice_bad_token(self): + self.store.get_app_service_by_token = Mock(return_value=None) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) + + def test_get_user_by_req_appservice_missing_token(self): + app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + self.store.get_app_service_by_token = Mock(return_value=app_service) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) + + @defer.inlineCallbacks + def test_get_user_by_req_appservice_valid_token_valid_user_id(self): + masquerading_user_id = "@doppelganger:matrix.org" + app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + app_service.is_interested_in_user = Mock(return_value=True) + self.store.get_app_service_by_token = Mock(return_value=app_service) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.args["user_id"] = [masquerading_user_id] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + (user, info) = yield self.auth.get_user_by_req(request) + self.assertEquals(user.to_string(), masquerading_user_id) + + def test_get_user_by_req_appservice_valid_token_bad_user_id(self): + masquerading_user_id = "@doppelganger:matrix.org" + app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + app_service.is_interested_in_user = Mock(return_value=False) + self.store.get_app_service_by_token = Mock(return_value=app_service) + self.store.get_user_by_token = Mock(return_value=None) + + request = Mock(args={}) + request.args["access_token"] = [self.test_token] + request.args["user_id"] = [masquerading_user_id] + request.requestHeaders.getRawHeaders = Mock(return_value=[""]) + d = self.auth.get_user_by_req(request) + self.failureResultOf(d, AuthError) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index f849120a3..e5d876d89 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -65,6 +65,7 @@ class PresenceStateTestCase(unittest.TestCase): hs.handlers = JustPresenceHandlers(hs) self.datastore = hs.get_datastore() + self.datastore.get_app_service_by_token = Mock(return_value=None) def get_presence_list(*a, **kw): return defer.succeed([]) @@ -154,6 +155,7 @@ class PresenceListTestCase(unittest.TestCase): hs.handlers = JustPresenceHandlers(hs) self.datastore = hs.get_datastore() + self.datastore.get_app_service_by_token = Mock(return_value=None) def has_presence_state(user_localpart): return defer.succeed( @@ -303,6 +305,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): hs.handlers.room_member_handler.get_rooms_for_user = get_rooms_for_user self.mock_datastore = hs.get_datastore() + self.mock_datastore.get_app_service_by_token = Mock(return_value=None) def get_profile_displayname(user_id): return defer.succeed("Frank") diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index fa70575c5..7c2b0dfa0 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -59,6 +59,8 @@ class V2AlphaRestTestCase(unittest.TestCase): r.register_servlets(hs, self.mock_resource) def make_datastore_mock(self): - return Mock(spec=[ + store = Mock(spec=[ "insert_client_ip", ]) + store.get_app_service_by_token = Mock(return_value=None) + return store From ab3c897ce123506bfff8a6a67e0530ca90ae2f15 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 14:16:36 +0000 Subject: [PATCH 331/630] Remove unused imports. --- tests/api/test_auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 1d8367ce4..4f83db5e8 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -15,11 +15,11 @@ from tests import unittest from twisted.internet import defer -from mock import Mock, NonCallableMock +from mock import Mock from synapse.api.auth import Auth from synapse.api.errors import AuthError -from synapse.types import UserID + class AuthTestCase(unittest.TestCase): From 784d714a3f2be2cf15151eee8723377a0e3eea11 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 9 Feb 2015 14:17:52 +0000 Subject: [PATCH 332/630] Fix server default rule injection (downwards, not upwards!) --- synapse/push/baserules.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 8d4b806da..37878f1e0 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -4,24 +4,24 @@ def list_with_base_rules(rawrules, user_name): ruleslist = [] # shove the server default rules for each kind onto the end of each - current_prio_class = 1 + current_prio_class = PRIORITY_CLASS_INVERSE_MAP.keys()[-1] for r in rawrules: - if r['priority_class'] > current_prio_class: - while current_prio_class < r['priority_class']: + if r['priority_class'] < current_prio_class: + while r['priority_class'] < current_prio_class: ruleslist.extend(make_base_rules( user_name, PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) ) - current_prio_class += 1 + current_prio_class -= 1 ruleslist.append(r) - while current_prio_class <= PRIORITY_CLASS_INVERSE_MAP.keys()[-1]: + while current_prio_class > 0: ruleslist.extend(make_base_rules( user_name, PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) ) - current_prio_class += 1 + current_prio_class -= 1 return ruleslist From 75656712e34694460ce7b12fc5a467667e04ea21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 14:22:52 +0000 Subject: [PATCH 333/630] Time how long we're spending on the database thread --- synapse/app/homeserver.py | 2 ++ synapse/storage/_base.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0f175ec3f..8976ff2e8 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -274,6 +274,8 @@ def setup(): hs.get_pusherpool().start() + hs.get_datastore().start_profiling() + if config.daemonize: print config.pid_file daemon = Daemonize( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b350fd61f..0849c5f1b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -85,6 +85,28 @@ class SQLBaseStore(object): self._db_pool = hs.get_db_pool() self._clock = hs.get_clock() + self._previous_txn_total_time = 0 + self._current_txn_total_time = 0 + self._previous_loop_ts = 0 + + def start_profiling(self): + self._previous_loop_ts = self._clock.time_msec() + + def loop(): + curr = self._current_txn_total_time + prev = self._previous_txn_total_time + self._previous_txn_total_time = curr + + time_now = self._clock.time_msec() + time_then = self._previous_loop_ts + self._previous_loop_ts = time_now + + ratio = (curr - prev)/(time_now - time_then) + + logger.info("Total database time: %.3f", ratio) + + self._clock.looping_call(loop, 1000) + @defer.inlineCallbacks def runInteraction(self, desc, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" @@ -114,6 +136,9 @@ class SQLBaseStore(object): "[TXN END] {%s} %f", name, end - start ) + + self._current_txn_total_time += end - start + with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( inner_func, *args, **kwargs From 164f6b9256782a5040c1a770c0758a31cf88e6e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 14:23:57 +0000 Subject: [PATCH 334/630] Fix tests --- tests/test_state.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_state.py b/tests/test_state.py index 019e794aa..fea25f702 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -21,6 +21,8 @@ from synapse.api.auth import Auth from synapse.api.constants import EventTypes, Membership from synapse.state import StateHandler +from .utils import MockClock + from mock import Mock @@ -138,10 +140,13 @@ class StateTestCase(unittest.TestCase): "add_event_hashes", ] ) - hs = Mock(spec=["get_datastore", "get_auth", "get_state_handler"]) + hs = Mock(spec=[ + "get_datastore", "get_auth", "get_state_handler", "get_clock", + ]) hs.get_datastore.return_value = self.store hs.get_state_handler.return_value = None hs.get_auth.return_value = Auth(hs) + hs.get_clock.return_value = MockClock() self.state = StateHandler(hs) self.event_id = 0 From 66fde49f071d75ea8bfdfac02fd4fa6fab5a9bf4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 14:45:15 +0000 Subject: [PATCH 335/630] Log database time every 10s and log as percentage --- synapse/storage/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 0849c5f1b..f1df5d39f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -103,9 +103,9 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - logger.info("Total database time: %.3f", ratio) + logger.info("Total database time: %.3f%", ratio * 100) - self._clock.looping_call(loop, 1000) + self._clock.looping_call(loop, 10000) @defer.inlineCallbacks def runInteraction(self, desc, func, *args, **kwargs): From ef995e69460a117e78a72bcef285f9a0c7438487 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 14:47:59 +0000 Subject: [PATCH 336/630] Add looping_call to Clock --- synapse/util/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 4e837a918..fee76b0a9 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -15,7 +15,7 @@ from synapse.util.logcontext import LoggingContext -from twisted.internet import reactor +from twisted.internet import reactor, task import time @@ -35,6 +35,14 @@ class Clock(object): """Returns the current system time in miliseconds since epoch.""" return self.time() * 1000 + def looping_call(self, f, msec): + l = task.LoopingCall(f) + l.start(msec/1000.0, now=False) + return l + + def stop_looping_call(self, loop): + loop.stop() + def call_later(self, delay, callback): current_context = LoggingContext.current_context() From c4ee4ce93ec6075bc076b12520fd72769079f37c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 15:00:37 +0000 Subject: [PATCH 337/630] Fix typo --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f1df5d39f..310ee0104 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -103,7 +103,7 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - logger.info("Total database time: %.3f%", ratio * 100) + logger.info("Total database time: %.3f%%", ratio * 100) self._clock.looping_call(loop, 10000) From f7cac2f7b699fb3bb58c02fa1828aeec6c319f01 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Feb 2015 15:01:28 +0000 Subject: [PATCH 338/630] Fix bugs so lazy room joining works as intended. --- synapse/appservice/api.py | 6 +++--- synapse/handlers/appservice.py | 6 +++--- synapse/handlers/directory.py | 23 +++++++++++++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 9cce4e097..15ac1e27f 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -41,7 +41,7 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": service.hs_token }) - if response: # just an empty json object + if response is not None: # just an empty json object defer.returnValue(True) except CodeMessageException as e: if e.code == 404: @@ -60,13 +60,13 @@ class ApplicationServiceApi(SimpleHttpClient): response = yield self.get_json(uri, { "access_token": service.hs_token }) - if response: # just an empty json object + if response is not None: # just an empty json object defer.returnValue(True) except CodeMessageException as e: + logger.warning("query_alias to %s received %s", uri, e.code) if e.code == 404: defer.returnValue(False) return - logger.warning("query_alias to %s received %s", uri, e.code) except Exception as ex: logger.warning("query_alias to %s threw exception %s", uri, ex) defer.returnValue(False) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 5071a12eb..8591a77bf 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -124,15 +124,15 @@ class ApplicationServicesHandler(object): namedtuple: with keys "room_id" and "servers" or None if no association can be found. """ - room_alias = room_alias.to_string() + room_alias_str = room_alias.to_string() alias_query_services = yield self._get_services_for_event( event=None, restrict_to=ApplicationService.NS_ALIASES, - alias_list=[room_alias] + alias_list=[room_alias_str] ) for alias_service in alias_query_services: is_known_alias = yield self.appservice_api.query_alias( - alias_service, room_alias + alias_service, room_alias_str ) if is_known_alias: # the alias exists now so don't query more ASes. diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 22c6391a1..87bc12c98 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -64,8 +64,11 @@ class DirectoryHandler(BaseHandler): # association creation for human users # TODO(erikj): Do user auth. - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: + can_create = yield self.can_modify_alias( + room_alias, + user_id=user_id + ) + if not can_create: raise SynapseError( 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE @@ -91,8 +94,11 @@ class DirectoryHandler(BaseHandler): # TODO Check if server admin - is_claimed = yield self.is_alias_exclusive_to_appservices(room_alias) - if is_claimed: + can_delete = yield self.can_modify_alias( + room_alias, + user_id=user_id + ) + if not can_delete: raise SynapseError( 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE @@ -228,9 +234,14 @@ class DirectoryHandler(BaseHandler): defer.returnValue(result) @defer.inlineCallbacks - def is_alias_exclusive_to_appservices(self, alias): + def can_modify_alias(self, alias, user_id=None): services = yield self.store.get_app_services() interested_services = [ s for s in services if s.is_interested_in_alias(alias.to_string()) ] - defer.returnValue(len(interested_services) > 0) + for service in interested_services: + if user_id == service.sender: + # this user IS the app service + defer.returnValue(True) + return + defer.returnValue(len(interested_services) == 0) From a578251b4800f20424e8b294a42cc6c65ef568a2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:44:47 +0000 Subject: [PATCH 339/630] only do word-boundary patches on bodies for now --- synapse/push/__init__.py | 5 ++++- synapse/python_dependencies.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 07b5f0187..58c8cf700 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -147,7 +147,10 @@ class Pusher(object): logger.warn("event_match condition with no pattern") return False # XXX: optimisation: cache our pattern regexps - r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) + if condition['key'] == 'content.body': + r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) + else: + r = r'^%s$' % self._glob_to_regexp(condition['pattern']) val = _value_for_dotted_key(condition['key'], ev) if val is None: return False diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a89d61860..f429d1bed 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.2": ["twisted==14.0.2"], + "Twisted==14.0.0": ["twisted==14.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], From bd2373277de8fd571d8eb0af38b107065e567a02 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:48:09 +0000 Subject: [PATCH 340/630] oops --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index f429d1bed..44d1a0404 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.0": ["twisted==14.0.0"], + "Twisted==14.0.0": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], From 0b725f5c4f8f0550205c09bbdc775f6ad613a1fd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:48:31 +0000 Subject: [PATCH 341/630] oops --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 44d1a0404..a89d61860 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.0": ["twisted==14.0.2"], + "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], From 8f616684a3a712ccf06349c67bb64779f06d8da1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 9 Feb 2015 17:01:40 +0000 Subject: [PATCH 342/630] Need to use re.search if looking for matches not at the start of the string. Also comparisons with None should be 'is'. --- synapse/push/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 58c8cf700..6f143a5df 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -154,7 +154,7 @@ class Pusher(object): val = _value_for_dotted_key(condition['key'], ev) if val is None: return False - return re.match(r, val, flags=re.IGNORECASE) != None + return re.search(r, val, flags=re.IGNORECASE) is not None elif condition['kind'] == 'device': if 'profile_tag' not in condition: @@ -170,8 +170,8 @@ class Pusher(object): return False if not display_name: return False - return re.match("\b%s\b" % re.escape(display_name), - ev['content']['body'], flags=re.IGNORECASE) != None + return re.search("\b%s\b" % re.escape(display_name), + ev['content']['body'], flags=re.IGNORECASE) is not None elif condition['kind'] == 'room_member_count': if 'is' not in condition: From d94f682a4c3e7bab5079d516582f0ee44a3d3f06 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 17:41:29 +0000 Subject: [PATCH 343/630] During room intial sync, only calculate current state once. --- synapse/api/auth.py | 21 ++++++++++++++------- synapse/handlers/message.py | 32 ++++++++++++++++++++++---------- synapse/handlers/sync.py | 9 ++++++--- synapse/state.py | 2 +- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 7105ee21d..005474536 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -89,12 +89,19 @@ class Auth(object): raise @defer.inlineCallbacks - def check_joined_room(self, room_id, user_id): - member = yield self.state.get_current_state( - room_id=room_id, - event_type=EventTypes.Member, - state_key=user_id - ) + def check_joined_room(self, room_id, user_id, current_state=None): + if current_state: + member = current_state.get( + (EventTypes.Member, user_id), + None + ) + else: + member = yield self.state.get_current_state( + room_id=room_id, + event_type=EventTypes.Member, + state_key=user_id + ) + self._check_joined_room(member, user_id, room_id) defer.returnValue(member) @@ -102,7 +109,7 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) - for event in curr_state: + for event in curr_state.values(): if event.type == EventTypes.Member: try: if UserID.from_string(event.state_key).domain != host: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3f51f38f1..3355adefc 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -35,6 +35,7 @@ class MessageHandler(BaseHandler): def __init__(self, hs): super(MessageHandler, self).__init__(hs) self.hs = hs + self.state = hs.get_state_handler() self.clock = hs.get_clock() self.validator = EventValidator() @@ -225,7 +226,9 @@ class MessageHandler(BaseHandler): # TODO: This is duplicating logic from snapshot_all_rooms current_state = yield self.state_handler.get_current_state(room_id) now = self.clock.time_msec() - defer.returnValue([serialize_event(c, now) for c in current_state]) + defer.returnValue( + [serialize_event(c, now) for c in current_state.values()] + ) @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, @@ -313,7 +316,7 @@ class MessageHandler(BaseHandler): ) d["state"] = [ serialize_event(c, time_now, as_client_event) - for c in current_state + for c in current_state.values() ] except: logger.exception("Failed to get snapshot") @@ -329,7 +332,14 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def room_initial_sync(self, user_id, room_id, pagin_config=None, feedback=False): - yield self.auth.check_joined_room(room_id, user_id) + current_state = yield self.state.get_current_state( + room_id=room_id, + ) + + yield self.auth.check_joined_room( + room_id, user_id, + current_state=current_state + ) # TODO(paul): I wish I was called with user objects not user_id # strings... @@ -337,13 +347,12 @@ class MessageHandler(BaseHandler): # TODO: These concurrently time_now = self.clock.time_msec() - state_tuples = yield self.state_handler.get_current_state(room_id) - state = [serialize_event(x, time_now) for x in state_tuples] + state = [ + serialize_event(x, time_now) + for x in current_state.values() + ] - member_event = (yield self.store.get_room_member( - user_id=user_id, - room_id=room_id - )) + member_event = current_state.get((EventTypes.Member, user_id,)) now_token = yield self.hs.get_event_sources().get_current_token() @@ -360,7 +369,10 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) - room_members = yield self.store.get_room_members(room_id) + room_members = [ + m for m in current_state.values() + if m.type == EventTypes.Member + ] presence_handler = self.hs.get_handlers().presence_handler presence = [] diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 439164ae3..5af90cc5d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -175,9 +175,10 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, ) - current_state_events = yield self.state_handler.get_current_state( + current_state = yield self.state_handler.get_current_state( room_id ) + current_state_events = current_state.values() defer.returnValue(RoomSyncResult( room_id=room_id, @@ -347,9 +348,10 @@ class SyncHandler(BaseHandler): # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is - current_state_events = yield self.state_handler.get_current_state( + current_state = yield self.state_handler.get_current_state( room_id ) + current_state_events = current_state.values() state_at_previous_sync = yield self.get_state_at_previous_sync( room_id, since_token=since_token @@ -431,6 +433,7 @@ class SyncHandler(BaseHandler): joined = True if joined: - state_delta = yield self.state_handler.get_current_state(room_id) + res = yield self.state_handler.get_current_state(room_id) + state_delta = res.values() defer.returnValue(state_delta) diff --git a/synapse/state.py b/synapse/state.py index 695a5e7ac..54380b9e5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -76,7 +76,7 @@ class StateHandler(object): defer.returnValue(res[1].get((event_type, state_key))) return - defer.returnValue(res[1].values()) + defer.returnValue(res[1]) @defer.inlineCallbacks def compute_event_context(self, event, old_state=None): From 3a5ad7dbd5a375023c96ee65c901f8be5ab02341 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 17:55:56 +0000 Subject: [PATCH 344/630] Performance counters for database transaction names --- synapse/storage/_base.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 310ee0104..bcb03cbdc 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,6 +39,7 @@ class LoggingTransaction(object): passed to the constructor. Adds logging to the .execute() method.""" __slots__ = ["txn", "name"] + def __init__(self, txn, name): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) @@ -88,6 +89,8 @@ class SQLBaseStore(object): self._previous_txn_total_time = 0 self._current_txn_total_time = 0 self._previous_loop_ts = 0 + self._txn_perf_counters = {} + self._previous_txn_perf_counters = {} def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -103,7 +106,29 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - logger.info("Total database time: %.3f%%", ratio * 100) + txn_counters = [] + for name, (count, cum_time) in self._txn_perf_counters.items(): + prev_count, prev_time = self._previous_txn_perf_counters.get( + name, (0,0) + ) + txn_counters.append(( + (cum_time - prev_time) / (time_now - time_then), + count - prev_count, + name + )) + + self._previous_txn_perf_counters = dict(self._txn_perf_counters) + + txn_counters.sort(reverse=True) + top_three_counters = ", ".join( + "%s(%d): %.3f%%" % (name, count, 100 * ratio) + for ratio, count, name in txn_counters[:3] + ) + + logger.info( + "Total database time: %.3f%% {%s}", + ratio * 100, top_three_counters + ) self._clock.looping_call(loop, 10000) @@ -139,6 +164,11 @@ class SQLBaseStore(object): self._current_txn_total_time += end - start + count, cum_time = self._txn_perf_counters.get(name, (0,0)) + count += 1 + cum_time += end - start + self._txn_perf_counters[name] = (count, cum_time) + with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( inner_func, *args, **kwargs From 347b497db0355fe4e26ae3a51967aa91bec090d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 17:57:09 +0000 Subject: [PATCH 345/630] Formatting --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index bcb03cbdc..45f4b994e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,7 +39,6 @@ class LoggingTransaction(object): passed to the constructor. Adds logging to the .execute() method.""" __slots__ = ["txn", "name"] - def __init__(self, txn, name): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) From 0c4536da8fe75a207052fb558414b4408aa857ec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 18:06:31 +0000 Subject: [PATCH 346/630] Use the transaction 'desc' rather than 'name', increment the txn_ids in txn names --- synapse/storage/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 45f4b994e..5ddd41060 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -140,7 +140,7 @@ class SQLBaseStore(object): with LoggingContext("runInteraction") as context: current_context.copy_to(context) start = time.time() * 1000 - txn_id = SQLBaseStore._TXN_ID + txn_id = self._TXN_ID # We don't really need these to be unique, so lets stop it from # growing really large. @@ -163,10 +163,10 @@ class SQLBaseStore(object): self._current_txn_total_time += end - start - count, cum_time = self._txn_perf_counters.get(name, (0,0)) + count, cum_time = self._txn_perf_counters.get(desc, (0,0)) count += 1 cum_time += end - start - self._txn_perf_counters[name] = (count, cum_time) + self._txn_perf_counters[desc] = (count, cum_time) with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( From 8ce100c7b48a38e5f74ed6ac560de1f50c288379 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 18:29:36 +0000 Subject: [PATCH 347/630] Convert directory paths to absolute paths before daemonizing --- synapse/config/_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index dfc115d8e..9b0f8c3c3 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -50,8 +50,9 @@ class Config(object): ) return cls.abspath(file_path) - @staticmethod - def ensure_directory(dir_path): + @classmethod + def ensure_directory(cls, dir_path): + dir_path = cls.abspath(dir_path) if not os.path.exists(dir_path): os.makedirs(dir_path) if not os.path.isdir(dir_path): From d19e2ed02fb545bde9c8d1a317f07b9fb786d2c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 11:01:15 +0000 Subject: [PATCH 348/630] Move construction of object within if block --- synapse/state.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 49ac09863..2dea8f552 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -306,13 +306,13 @@ class StateHandler(object): new_state = unconflicted_state new_state.update(resolved_state) - cache = _StateCacheEntry( - state=new_state, - state_group=None, - ts=self.clock.time_msec() - ) - if self._state_cache is not None: + cache = _StateCacheEntry( + state=new_state, + state_group=None, + ts=self.clock.time_msec() + ) + self._state_cache[frozenset(event_ids)] = cache defer.returnValue((None, new_state, prev_states)) From 2b042ad67fd16c43f3cfce472a8d1b31cff867da Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 11:03:16 +0000 Subject: [PATCH 349/630] Oops, we do want to defer.return regardless of whether we are caching or not --- synapse/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index 2dea8f552..31f503a1e 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -261,7 +261,7 @@ class StateHandler(object): self._state_cache[frozenset(event_ids)] = cache - defer.returnValue((name, state, prev_states)) + defer.returnValue((name, state, prev_states)) state = {} for group, g_state in state_groups.items(): From 30595b466f1f6b61eef15dc41562f6336e57e6fe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 13:50:33 +0000 Subject: [PATCH 350/630] Use yaml logging config format because it is much nicer --- synapse/config/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index f9568ebd2..b18d08ec6 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -18,6 +18,7 @@ from synapse.util.logcontext import LoggingContextFilter from twisted.python.log import PythonLoggingObserver import logging import logging.config +import yaml class LoggingConfig(Config): @@ -79,7 +80,7 @@ class LoggingConfig(Config): logger.addHandler(handler) logger.info("Test") else: - logging.config.fileConfig(self.log_config) + logging.config.dictConfig(yaml.load(self.log_config)) observer = PythonLoggingObserver() observer.start() From f91345bdb5b76ce76b99d4fc969ee4f41cbbba4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 13:57:31 +0000 Subject: [PATCH 351/630] yaml.load expects strings to be a yaml rather than file --- synapse/config/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index b18d08ec6..63c8e3693 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -80,7 +80,8 @@ class LoggingConfig(Config): logger.addHandler(handler) logger.info("Test") else: - logging.config.dictConfig(yaml.load(self.log_config)) + with open(self.log_config, 'r') as f: + logging.config.dictConfig(yaml.load(f)) observer = PythonLoggingObserver() observer.start() From d7c7efb691bd726ec3e8879e289546fbcfd7dabd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 14:50:53 +0000 Subject: [PATCH 352/630] Add performance counters for different stages of loading events --- synapse/storage/_base.py | 82 +++++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5ddd41060..c79399fe5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -77,6 +77,43 @@ class LoggingTransaction(object): sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) +class PerformanceCounters(object): + def __init__(self): + self.current_counters = {} + self.previous_counters = {} + + def update(self, key, start_time, end_time=None): + if end_time is None: + end_time = time.time() * 1000; + duration = end_time - start_time + count, cum_time = self.current_counters.get(key, (0, 0)) + count += 1 + cum_time += duration + self.current_counters[key] = (count, cum_time) + return end_time + + def interval(self, interval_duration, limit=3): + counters = [] + for name, (count, cum_time) in self.current_counters.items(): + prev_count, prev_time = self.previous_counters.get(name, (0, 0)) + counters.append(( + (cum_time - prev_time) / interval_duration, + count - prev_count, + name + )) + + self.previous_counters = dict(self.current_counters) + + counters.sort(reverse=True) + + top_n_counters = ", ".join( + "%s(%d): %.3f%%" % (name, count, 100 * ratio) + for ratio, count, name in txn_counters[:limit] + ) + + return top_n_counters + + class SQLBaseStore(object): _TXN_ID = 0 @@ -88,8 +125,8 @@ class SQLBaseStore(object): self._previous_txn_total_time = 0 self._current_txn_total_time = 0 self._previous_loop_ts = 0 - self._txn_perf_counters = {} - self._previous_txn_perf_counters = {} + self._txn_perf_counters = PerformanceCounters() + self._get_event_counters = PerformanceCounters() def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -105,23 +142,12 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - txn_counters = [] - for name, (count, cum_time) in self._txn_perf_counters.items(): - prev_count, prev_time = self._previous_txn_perf_counters.get( - name, (0,0) - ) - txn_counters.append(( - (cum_time - prev_time) / (time_now - time_then), - count - prev_count, - name - )) + top_three_counters = self._txn_perf_counters.interval( + time_now - time_then, limit=3 + ) - self._previous_txn_perf_counters = dict(self._txn_perf_counters) - - txn_counters.sort(reverse=True) - top_three_counters = ", ".join( - "%s(%d): %.3f%%" % (name, count, 100 * ratio) - for ratio, count, name in txn_counters[:3] + top_3_event_counters = self._get_event_counters.interval( + time_now - time_then, limit=3 ) logger.info( @@ -162,11 +188,7 @@ class SQLBaseStore(object): ) self._current_txn_total_time += end - start - - count, cum_time = self._txn_perf_counters.get(desc, (0,0)) - count += 1 - cum_time += end - start - self._txn_perf_counters[desc] = (count, cum_time) + self._txn_perf_counters.update(desc, start, end) with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( @@ -566,6 +588,8 @@ class SQLBaseStore(object): "LIMIT 1 " ) + start_time = time.time() * 1000; + txn.execute(sql, (event_id,)) res = txn.fetchone() @@ -575,6 +599,8 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res + self._get_event_counters.update("select_event", start_time) + if allow_rejected or not rejected_reason: return self._get_event_from_row_txn( txn, internal_metadata, js, redacted, @@ -586,10 +612,18 @@ class SQLBaseStore(object): def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): + + start_time = time.time() * 1000; + update_counter = self._get_event_counters.update + d = json.loads(js) + start_time = update_counter("decode_json", start_time) + internal_metadata = json.loads(internal_metadata) + start_time = update_counter("decode_internal", start_time) ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) + start_time = update_counter("build_frozen_event", start_time) if check_redacted and redacted: ev = prune_event(ev) @@ -605,6 +639,7 @@ class SQLBaseStore(object): if because: ev.unsigned["redacted_because"] = because + start_time = update_counter("redact_event", start_time) if get_prev_content and "replaces_state" in ev.unsigned: prev = self._get_event_txn( @@ -614,6 +649,7 @@ class SQLBaseStore(object): ) if prev: ev.unsigned["prev_content"] = prev.get_dict()["content"] + start_time = update_counter("get_prev_content", start_time) return ev From fda4422bc9d9f2974d7185011d6d905eea372b09 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 14:54:07 +0000 Subject: [PATCH 353/630] Fix pyflakes --- synapse/storage/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c79399fe5..36455ef93 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -108,7 +108,7 @@ class PerformanceCounters(object): top_n_counters = ", ".join( "%s(%d): %.3f%%" % (name, count, 100 * ratio) - for ratio, count, name in txn_counters[:limit] + for ratio, count, name in counters[:limit] ) return top_n_counters @@ -151,8 +151,8 @@ class SQLBaseStore(object): ) logger.info( - "Total database time: %.3f%% {%s}", - ratio * 100, top_three_counters + "Total database time: %.3f%% {%s} {%s}", + ratio * 100, top_three_counters, top_3_event_counters ) self._clock.looping_call(loop, 10000) From f8abbae99fe76409533518e828031de4933f38ab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 15:45:50 +0000 Subject: [PATCH 354/630] Remove unnecessary logging --- synapse/state.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 64c58a393..98aaa2be5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -72,11 +72,7 @@ class StateHandler(object): self._state_cache = {} def f(): - logger.debug("Pruning") - try: - self._prune_cache() - except: - logger.exception("Prune") + self._prune_cache() self.clock.looping_call(f, 5*1000) @@ -400,7 +396,6 @@ class StateHandler(object): return sorted(events, key=key_func) def _prune_cache(self): - logger.debug("_prune_cache") logger.debug( "_prune_cache. before len: %d", len(self._state_cache.keys()) From 697ab75a34b14da6d872d153f36017a6dc6d5b99 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 15:46:24 +0000 Subject: [PATCH 355/630] Sign auth_chains when returned by /state/ requests --- synapse/federation/federation_server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 4742ca939..b23f72c7f 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -25,6 +25,8 @@ from synapse.events import FrozenEvent from synapse.api.errors import FederationError, SynapseError +from synapse.crypto.event_signing import compute_event_signature + import logging @@ -156,6 +158,15 @@ class FederationServer(FederationBase): auth_chain = yield self.store.get_auth_chain( [pdu.event_id for pdu in pdus] ) + + for event in auth_chain: + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) else: raise NotImplementedError("Specify an event") From b085fac7353e1cd395b89f9334c8273a8e996f48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 16:30:48 +0000 Subject: [PATCH 356/630] Code-style fixes --- synapse/handlers/presence.py | 4 +++- synapse/handlers/register.py | 13 ++++++++----- synapse/push/__init__.py | 8 +++++--- synapse/push/baserules.py | 11 ++++++----- synapse/push/rulekinds.py | 12 ++++++------ synapse/python_dependencies.py | 6 ++++-- synapse/rest/client/v1/push_rule.py | 18 +++++++++++++----- synapse/rest/client/v1/pusher.py | 4 ++-- synapse/rest/media/v1/upload_resource.py | 7 ++++--- synapse/storage/_base.py | 6 +++--- synapse/storage/push_rule.py | 4 +++- 11 files changed, 57 insertions(+), 36 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index cd0798c2b..6a266ee0f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -658,7 +658,9 @@ class PresenceHandler(BaseHandler): observers = set(self._remote_recvmap.get(user, set())) if observers: - logger.debug(" | %d interested local observers %r", len(observers), observers) + logger.debug( + " | %d interested local observers %r", len(observers), observers + ) rm_handler = self.homeserver.get_handlers().room_member_handler room_ids = yield rm_handler.get_rooms_for_user(user) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 4f06c487b..0247327eb 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -105,17 +105,20 @@ class RegistrationHandler(BaseHandler): # do it here. try: auth_user = UserID.from_string(user_id) - identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) - upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + media_repository = self.hs.get_resource_for_media_repository() + identicon_resource = media_repository.getChildWithDefault("identicon", None) + upload_resource = media_repository.getChildWithDefault("upload", None) identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) content_uri = yield upload_resource.create_content( "image/png", None, identicon_bytes, len(identicon_bytes), auth_user ) profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + profile_handler.set_avatar_url( + auth_user, auth_user, ("%s#auto" % (content_uri,)) + ) except NotImplementedError: - pass # make tests pass without messing around creating default avatars - + pass # make tests pass without messing around creating default avatars + defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 6f143a5df..418a348a5 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -140,7 +140,7 @@ class Pusher(object): lambda x: ('[%s%s]' % (x.group(1) and '^' or '', re.sub(r'\\\-', '-', x.group(2)))), r) return r - + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: @@ -170,8 +170,10 @@ class Pusher(object): return False if not display_name: return False - return re.search("\b%s\b" % re.escape(display_name), - ev['content']['body'], flags=re.IGNORECASE) is not None + return re.search( + "\b%s\b" % re.escape(display_name), ev['content']['body'], + flags=re.IGNORECASE + ) is not None elif condition['kind'] == 'room_member_count': if 'is' not in condition: diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 37878f1e0..162d265f6 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,5 +1,6 @@ from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP + def list_with_base_rules(rawrules, user_name): ruleslist = [] @@ -9,9 +10,9 @@ def list_with_base_rules(rawrules, user_name): if r['priority_class'] < current_prio_class: while r['priority_class'] < current_prio_class: ruleslist.extend(make_base_rules( - user_name, - PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) - ) + user_name, + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + )) current_prio_class -= 1 ruleslist.append(r) @@ -19,8 +20,8 @@ def list_with_base_rules(rawrules, user_name): while current_prio_class > 0: ruleslist.extend(make_base_rules( user_name, - PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) - ) + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + )) current_prio_class -= 1 return ruleslist diff --git a/synapse/push/rulekinds.py b/synapse/push/rulekinds.py index 763bdee58..660aa4e10 100644 --- a/synapse/push/rulekinds.py +++ b/synapse/push/rulekinds.py @@ -1,8 +1,8 @@ PRIORITY_CLASS_MAP = { - 'underride': 1, - 'sender': 2, - 'room': 3, - 'content': 4, - 'override': 5, - } + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, +} PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a89d61860..fd68da9df 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -19,10 +19,11 @@ REQUIREMENTS = { "pydenticon": ["pydenticon"], } + def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) -DEPENDENCY_LINKS=[ +DEPENDENCY_LINKS = [ github_link( project="matrix-org/syutil", version="v0.0.2", @@ -101,6 +102,7 @@ def check_requirements(): % (dependency, file_path, version, required_version) ) + def list_requirements(): result = [] linked = [] @@ -111,7 +113,7 @@ def list_requirements(): for requirement in REQUIREMENTS: is_linked = False for link in linked: - if requirement.replace('-','_').startswith(link): + if requirement.replace('-', '_').startswith(link): is_linked = True if not is_linked: result.append(requirement) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index d43ade39d..c4e7dfcf0 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,12 +15,17 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ - StoreError +from synapse.api.errors import ( + SynapseError, Codes, UnrecognizedRequestError, NotFoundError, StoreError +) from .base import ClientV1RestServlet, client_path_pattern -from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException +from synapse.storage.push_rule import ( + InconsistentRuleException, RuleNotFoundException +) import synapse.push.baserules as baserules -from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP +from synapse.push.rulekinds import ( + PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP +) import json @@ -105,7 +110,9 @@ class PushRuleRestServlet(ClientV1RestServlet): # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference - rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name( + user.to_string() + ) for r in rawrules: r["conditions"] = json.loads(r["conditions"]) @@ -383,6 +390,7 @@ def _namespaced_rule_id_from_spec(spec): def _rule_id_from_namespaced(in_rule_id): return in_rule_id.split('/')[-1] + class InvalidRuleException(Exception): pass diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index e10d2576d..80e9939b7 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -34,8 +34,8 @@ class PusherRestServlet(ClientV1RestServlet): pusher_pool = self.hs.get_pusherpool() if ('pushkey' in content and 'app_id' in content - and 'kind' in content and - content['kind'] is None): + and 'kind' in content and + content['kind'] is None): yield pusher_pool.remove_pusher( content['app_id'], content['pushkey'] ) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 5b4278233..6df52ca43 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -38,9 +38,10 @@ class UploadResource(BaseMediaResource): def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET - + @defer.inlineCallbacks - def create_content(self, media_type, upload_name, content, content_length, auth_user): + def create_content(self, media_type, upload_name, content, content_length, + auth_user): media_id = random_string(24) fname = self.filepaths.local_media_filepath(media_id) @@ -65,7 +66,7 @@ class UploadResource(BaseMediaResource): } yield self._generate_local_thumbnails(media_id, media_info) - + defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @defer.inlineCallbacks diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 36455ef93..3e1ab0a15 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -84,7 +84,7 @@ class PerformanceCounters(object): def update(self, key, start_time, end_time=None): if end_time is None: - end_time = time.time() * 1000; + end_time = time.time() * 1000 duration = end_time - start_time count, cum_time = self.current_counters.get(key, (0, 0)) count += 1 @@ -588,7 +588,7 @@ class SQLBaseStore(object): "LIMIT 1 " ) - start_time = time.time() * 1000; + start_time = time.time() * 1000 txn.execute(sql, (event_id,)) @@ -613,7 +613,7 @@ class SQLBaseStore(object): def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): - start_time = time.time() * 1000; + start_time = time.time() * 1000 update_counter = self._get_event_counters.update d = json.loads(js) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 30e23445d..620de7139 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -91,7 +91,9 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() if not res: - raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) + raise RuleNotFoundException( + "before/after rule not found: %s" % (relative_to_rule,) + ) priority_class, base_rule_priority = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: From e8d4a314753bb49f91a2e51d0e442a153f519a64 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 16:36:51 +0000 Subject: [PATCH 357/630] Fix prune_events to work with nested dicts --- synapse/events/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 4ad37188b..5bfa820f9 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -45,12 +45,14 @@ def prune_event(event): "membership", ] + event_dict = event.get_dict() + new_content = {} def add_fields(*fields): for field in fields: if field in event.content: - new_content[field] = event.content[field] + new_content[field] = event_dict["content"][field] if event_type == EventTypes.Member: add_fields("membership") @@ -75,7 +77,7 @@ def prune_event(event): allowed_fields = { k: v - for k, v in event.get_dict().items() + for k, v in event_dict.items() if k in allowed_keys } From b61a308b2725c6e7ab12983d2d05461a532f6405 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 16:37:12 +0000 Subject: [PATCH 358/630] Bump version --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 1648b8029..f97571ce5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1e +0.6.1f diff --git a/synapse/__init__.py b/synapse/__init__.py index d3f1c33d3..329138734 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1e" +__version__ = "0.6.1f" From c8e1da930dd4ffbc4ca409208b2d715182cd100f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 17:30:46 +0000 Subject: [PATCH 359/630] Log all the exits from _attempt_new_transaction --- synapse/federation/transaction_queue.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index f38aeba7c..731019ad9 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -157,15 +157,19 @@ class TransactionQueue(object): else: logger.info("TX [%s] is ready for retry", destination) - logger.info("TX [%s] _attempt_new_transaction", destination) - if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. # we need application-layer timeouts of some flavour of these # requests + logger.info( + "TX [%s] Transaction already in progress", + destination + ) return + logger.info("TX [%s] _attempt_new_transaction", destination) + # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) pending_edus = self.pending_edus_by_dest.pop(destination, []) @@ -176,6 +180,7 @@ class TransactionQueue(object): destination, len(pending_pdus)) if not pending_pdus and not pending_edus and not pending_failures: + logger.info("TX [%s] Nothing to send", destination) return logger.debug( From 7ed971d9b206a70f44c55588bdb1a4a33906fbe1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:42:36 +0000 Subject: [PATCH 360/630] Single source version and python dependencies, prevent people accidentally installing with easy_install, use scripts rather than entry_points to install synctl --- VERSION | 1 - setup.py | 80 +++++++++++++++++----------------- synapse/python_dependencies.py | 6 +-- 3 files changed, 43 insertions(+), 44 deletions(-) delete mode 100644 VERSION diff --git a/VERSION b/VERSION deleted file mode 100644 index 1c29ff4d3..000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.6.1d diff --git a/setup.py b/setup.py index bd2766b24..0215fba4c 100755 --- a/setup.py +++ b/setup.py @@ -17,52 +17,52 @@ import os from setuptools import setup, find_packages +import setuptools.command.easy_install -# Utility function to read the README file. -# Used for the long_description. It's nice, because now 1) we have a top level -# README file and 2) it's easier to type in the README file than to put a raw -# string in below ... -def read(fname): - return open(os.path.join(os.path.dirname(__file__), fname)).read() + +def no_easy_install(self, spec, *args, **kargs): + raise RuntimeError( + "Missing requirement %r. Please use pip to install this dependency." + " See README.rst for instructions." + % (spec,) + ) + +# Patch the easy_install command to prevent people accidentally installing +# depedencies using it. +setuptools.command.easy_install.easy_install.easy_install = no_easy_install + + +here = os.path.abspath(os.path.dirname(__file__)) + + +def read_file(path): + """Read a file from the package. Takes a list of strings to join to + make the path""" + file_path = os.path.join(here, *path) + with open(file_path) as f: + return f.read() + + +def exec_file(path): + """Execute a single python file to get the variables defined in it""" + result = {} + code = read_file(path) + exec(code, result) + return result + +version = exec_file(("synapse", "__init__.py"))["__version__"] +dependencies = exec_file(("synapse", "python_dependencies.py")) +long_description = read_file(("README.rst",)) setup( name="matrix-synapse", - version=read("VERSION").strip(), + version=version, packages=find_packages(exclude=["tests", "tests.*"]), description="Reference Synapse Home Server", - install_requires=[ - "syutil==0.0.2", - "matrix_angular_sdk>=0.6.1", - "Twisted==14.0.2", - "service_identity>=1.0.0", - "pyopenssl>=0.14", - "pyyaml", - "pyasn1", - "pynacl", - "daemonize", - "py-bcrypt", - "frozendict>=0.4", - "pillow", - "pydenticon", - ], - dependency_links=[ - "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", - "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.1/#egg=matrix_angular_sdk-0.6.1", - ], - setup_requires=[ - "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 - "setuptools_trial", - "setuptools>=1.0.0", # Needs setuptools that supports git+ssh. - # TODO: Do we need this now? we don't use git+ssh. - "mock" - ], + install_requires=dependencies["REQUIREMENTS"].keys(), + dependency_links=dependencies["DEPENDENCY_LINKS"], include_package_data=True, zip_safe=False, - long_description=read("README.rst"), - entry_points=""" - [console_scripts] - synctl=synapse.app.synctl:main - synapse-homeserver=synapse.app.homeserver:main - """ + long_description=long_description, + scripts=["synctl"], ) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index fd68da9df..e2a9d1f6a 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], - "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], + "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -31,8 +31,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.0", - egg="matrix_angular_sdk-0.6.0", + version="v0.6.1", + egg="matrix_angular_sdk-0.6.1", ), github_link( project="pyca/pynacl", From a9684730acdb98ebe9102da985765d8f309388fa Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:48:16 +0000 Subject: [PATCH 361/630] Add the 'setup_requires' and allow easy_install since jenkins uses them --- setup.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 0215fba4c..cc7a3f5b6 100755 --- a/setup.py +++ b/setup.py @@ -17,20 +17,6 @@ import os from setuptools import setup, find_packages -import setuptools.command.easy_install - - -def no_easy_install(self, spec, *args, **kargs): - raise RuntimeError( - "Missing requirement %r. Please use pip to install this dependency." - " See README.rst for instructions." - % (spec,) - ) - -# Patch the easy_install command to prevent people accidentally installing -# depedencies using it. -setuptools.command.easy_install.easy_install.easy_install = no_easy_install - here = os.path.abspath(os.path.dirname(__file__)) @@ -60,6 +46,11 @@ setup( packages=find_packages(exclude=["tests", "tests.*"]), description="Reference Synapse Home Server", install_requires=dependencies["REQUIREMENTS"].keys(), + setup_requires=[ + "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 + "setuptools_trial", + "mock" + ], dependency_links=dependencies["DEPENDENCY_LINKS"], include_package_data=True, zip_safe=False, From 84a769cdb7321481954c386a040d7e7ff504d02b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:58:36 +0000 Subject: [PATCH 362/630] Fix code-style --- synapse/app/homeserver.py | 2 +- synapse/crypto/keyclient.py | 4 ++-- synapse/handlers/presence.py | 8 ++++---- synapse/handlers/sync.py | 2 +- synapse/rest/client/v1/directory.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/media/v1/upload_resource.py | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8976ff2e8..ff6680e7d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -134,7 +134,7 @@ class SynapseHomeServer(HomeServer): logger.info("Attaching %s to path %s", resource, full_path) last_resource = self.root_resource for path_seg in full_path.split('/')[1:-1]: - if not path_seg in last_resource.listNames(): + if path_seg not in last_resource.listNames(): # resource doesn't exist, so make a "dummy resource" child_resource = Resource() last_resource.putChild(path_seg, child_resource) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index cdb627976..cd12349f6 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -75,7 +75,7 @@ class SynapseKeyClientProtocol(HTTPClient): def handleStatus(self, version, status, message): if status != b"200": - #logger.info("Non-200 response from %s: %s %s", + # logger.info("Non-200 response from %s: %s %s", # self.transport.getHost(), status, message) self.transport.abortConnection() @@ -83,7 +83,7 @@ class SynapseKeyClientProtocol(HTTPClient): try: json_response = json.loads(response_body_bytes) except ValueError: - #logger.info("Invalid JSON response from %s", + # logger.info("Invalid JSON response from %s", # self.transport.getHost()) self.transport.abortConnection() return diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6a266ee0f..59287010e 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -457,9 +457,9 @@ class PresenceHandler(BaseHandler): if state is None: state = yield self.store.get_presence_state(user.localpart) else: -# statuscache = self._get_or_make_usercache(user) -# self._user_cachemap_latest_serial += 1 -# statuscache.update(state, self._user_cachemap_latest_serial) + # statuscache = self._get_or_make_usercache(user) + # self._user_cachemap_latest_serial += 1 + # statuscache.update(state, self._user_cachemap_latest_serial) pass yield self.push_update_to_local_and_remote( @@ -709,7 +709,7 @@ class PresenceHandler(BaseHandler): # TODO(paul) permissions checks - if not user in self._remote_sendmap: + if user not in self._remote_sendmap: self._remote_sendmap[user] = set() self._remote_sendmap[user].add(origin) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5af90cc5d..7883bbd83 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -114,7 +114,7 @@ class SyncHandler(BaseHandler): if sync_config.gap: return self.incremental_sync_with_gap(sync_config, since_token) else: - #TODO(mjark): Handle gapless sync + # TODO(mjark): Handle gapless sync raise NotImplementedError() @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8f65efec5..8ed7e2d66 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -48,7 +48,7 @@ class ClientDirectoryServer(ClientV1RestServlet): user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) - if not "room_id" in content: + if "room_id" not in content: raise SynapseError(400, "Missing room_id key", errcode=Codes.BAD_JSON) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 81d5cf8ea..3056ec45c 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -118,7 +118,7 @@ class SyncRestServlet(RestServlet): except: filter = Filter({}) # filter = filter.apply_overrides(http_request) - #if filter.matches(event): + # if filter.matches(event): # # stuff sync_config = SyncConfig( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 6df52ca43..e5aba3af4 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -96,8 +96,8 @@ class UploadResource(BaseMediaResource): code=400, ) - #if headers.hasHeader("Content-Disposition"): - # disposition = headers.getRawHeaders("Content-Disposition")[0] + # if headers.hasHeader("Content-Disposition"): + # disposition = headers.getRawHeaders("Content-Disposition")[0] # TODO(markjh): parse content-dispostion content_uri = yield self.create_content( From 0e6b3e4e40ae918eacdef87bb50ff1d19b304e7c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 18:17:27 +0000 Subject: [PATCH 363/630] Time out HTTP federation requests --- synapse/http/matrixfederationclient.py | 11 +++++++++-- synapse/util/async.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index aa14782b0..fdc1e2678 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint -from synapse.util.async import sleep +from synapse.util.async import sleep, time_bound_deferred from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json @@ -78,6 +78,7 @@ class MatrixFederationHttpClient(object): self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.agent = MatrixFederationHttpAgent(reactor) + self.clock = hs.get_clock() @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, @@ -117,7 +118,7 @@ class MatrixFederationHttpClient(object): try: with PreserveLoggingContext(): - response = yield self.agent.request( + request_deferred = self.agent.request( destination, endpoint, method, @@ -128,6 +129,12 @@ class MatrixFederationHttpClient(object): producer ) + response = yield time_bound_deferred( + request_deferred, + clock=self.clock, + time_out=60, + ) + logger.debug("Got response to %s", method) break except Exception as e: diff --git a/synapse/util/async.py b/synapse/util/async.py index c4fe5d522..d4d1d4b47 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,3 +32,23 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) + + +def time_bound_deferred(given_deferred, clock, time_out): + ret_deferred = defer.Deferred() + + def timed_out(): + if not given_deferred.called: + given_deferred.cancel() + ret_deferred.errback(RuntimeError("Timed out")) + + timer = clock.call_later(time_out, timed_out) + + def succeed(result): + clock.cancel_call_later(timer) + ret_deferred.callback(result) + + given_deferred.addCallback(succeed) + given_deferred.addErrback(ret_deferred.errback) + + return ret_deferred From eab141ee676c87f6d1ca3d853e3f9ef9d49e28ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 18:25:54 +0000 Subject: [PATCH 364/630] Rename path to path_segments to make it clearer that it is a list --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index cc7a3f5b6..2d812fa38 100755 --- a/setup.py +++ b/setup.py @@ -21,18 +21,18 @@ from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) -def read_file(path): +def read_file(path_segments): """Read a file from the package. Takes a list of strings to join to make the path""" - file_path = os.path.join(here, *path) + file_path = os.path.join(here, *path_segments) with open(file_path) as f: return f.read() -def exec_file(path): +def exec_file(path_segments): """Execute a single python file to get the variables defined in it""" result = {} - code = read_file(path) + code = read_file(path_segments) exec(code, result) return result From dcf52469e821b3b2b69a0610c2c4f025a5aac68f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:25:06 +0000 Subject: [PATCH 365/630] Move time_bound_deferred into Clock --- synapse/http/matrixfederationclient.py | 5 ++--- synapse/util/__init__.py | 21 ++++++++++++++++++++- synapse/util/async.py | 20 -------------------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index fdc1e2678..74e523960 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint -from synapse.util.async import sleep, time_bound_deferred +from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json @@ -129,9 +129,8 @@ class MatrixFederationHttpClient(object): producer ) - response = yield time_bound_deferred( + response = yield self.clock.time_bound_deferred( request_deferred, - clock=self.clock, time_out=60, ) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 4e837a918..2da8dfa71 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -15,7 +15,7 @@ from synapse.util.logcontext import LoggingContext -from twisted.internet import reactor +from twisted.internet import defer, reactor import time @@ -45,3 +45,22 @@ class Clock(object): def cancel_call_later(self, timer): timer.cancel() + + def time_bound_deferred(self, given_deferred, time_out): + ret_deferred = defer.Deferred() + + def timed_out(): + if not given_deferred.called: + given_deferred.cancel() + ret_deferred.errback(RuntimeError("Timed out")) + + timer = self.call_later(time_out, timed_out) + + def succeed(result): + self.cancel_call_later(timer) + ret_deferred.callback(result) + + given_deferred.addCallback(succeed) + given_deferred.addErrback(ret_deferred.errback) + + return ret_deferred diff --git a/synapse/util/async.py b/synapse/util/async.py index d4d1d4b47..c4fe5d522 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,23 +32,3 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) - - -def time_bound_deferred(given_deferred, clock, time_out): - ret_deferred = defer.Deferred() - - def timed_out(): - if not given_deferred.called: - given_deferred.cancel() - ret_deferred.errback(RuntimeError("Timed out")) - - timer = clock.call_later(time_out, timed_out) - - def succeed(result): - clock.cancel_call_later(timer) - ret_deferred.callback(result) - - given_deferred.addCallback(succeed) - given_deferred.addErrback(ret_deferred.errback) - - return ret_deferred From 05b961d7e3f6c926520c3200ae3031c77cebc212 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:28:46 +0000 Subject: [PATCH 366/630] PEP8 --- synapse/http/matrixfederationclient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 74e523960..92f887f77 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -72,7 +72,6 @@ class MatrixFederationHttpClient(object): requests. """ - def __init__(self, hs): self.hs = hs self.signing_key = hs.config.signing_key[0] From 6370cffbbfc0bcfa4465453c24c535ef755db4f1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 10:34:41 +0000 Subject: [PATCH 367/630] Fix bug where variable was not always defined --- synapse/http/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 6d084fa33..589c30c19 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -113,6 +113,7 @@ class JsonResource(HttpServer, resource.Resource): path. """ code = None + start = self.clock.time_msec() try: # Just say yes to OPTIONS. if request.method == "OPTIONS": @@ -137,8 +138,6 @@ class JsonResource(HttpServer, resource.Resource): request.method, request.path ) - start = self.clock.time_msec() - code, response = yield path_entry.callback( request, *args From c7783d6feec9b69c24f3303cbb51cce3e6b8ffb3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:36:08 +0000 Subject: [PATCH 368/630] Notify ASes for events sent by other users in a room which an AS user is a part of. --- synapse/appservice/__init__.py | 17 +++++++++++++---- synapse/handlers/appservice.py | 21 ++++++++++++++++----- synapse/rest/appservice/v1/__init__.py | 4 ++-- tests/appservice/test_appservice.py | 25 +++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index fb9bfffe5..381b4cfc4 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -74,7 +74,7 @@ class ApplicationService(object): return True return False - def _matches_user(self, event): + def _matches_user(self, event, member_list): if (hasattr(event, "sender") and self.is_interested_in_user(event.sender)): return True @@ -83,6 +83,10 @@ class ApplicationService(object): and hasattr(event, "state_key") and self.is_interested_in_user(event.state_key)): return True + # check joined member events + for member in member_list: + if self.is_interested_in_user(member.state_key): + return True return False def _matches_room_id(self, event): @@ -96,7 +100,8 @@ class ApplicationService(object): return True return False - def is_interested(self, event, restrict_to=None, aliases_for_event=None): + def is_interested(self, event, restrict_to=None, aliases_for_event=None, + member_list=None): """Check if this service is interested in this event. Args: @@ -104,18 +109,22 @@ class ApplicationService(object): restrict_to(str): The namespace to restrict regex tests to. aliases_for_event(list): A list of all the known room aliases for this event. + member_list(list): A list of all joined room members in this room. Returns: bool: True if this service would like to know about this event. """ if aliases_for_event is None: aliases_for_event = [] + if member_list is None: + member_list = [] + if restrict_to and restrict_to not in ApplicationService.NS_LIST: # this is a programming error, so fail early and raise a general # exception raise Exception("Unexpected restrict_to value: %s". restrict_to) if not restrict_to: - return (self._matches_user(event) + return (self._matches_user(event, member_list) or self._matches_aliases(event, aliases_for_event) or self._matches_room_id(event)) elif restrict_to == ApplicationService.NS_ALIASES: @@ -123,7 +132,7 @@ class ApplicationService(object): elif restrict_to == ApplicationService.NS_ROOMS: return self._matches_room_id(event) elif restrict_to == ApplicationService.NS_USERS: - return self._matches_user(event) + return self._matches_user(event, member_list) def is_interested_in_user(self, user_id): return self._matches_regex(user_id, ApplicationService.NS_USERS) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8591a77bf..2c488a46f 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.types import UserID @@ -154,14 +154,25 @@ class ApplicationServicesHandler(object): list: A list of services interested in this event based on the service regex. """ - # We need to know the aliases associated with this event.room_id, if any - if not alias_list: - alias_list = yield self.store.get_aliases_for_room(event.room_id) + member_list = None + if hasattr(event, "room_id"): + # We need to know the aliases associated with this event.room_id, + # if any. + if not alias_list: + alias_list = yield self.store.get_aliases_for_room( + event.room_id + ) + # We need to know the members associated with this event.room_id, + # if any. + member_list = yield self.store.get_room_members( + room_id=event.room_id, + membership=Membership.JOIN + ) services = yield self.store.get_app_services() interested_list = [ s for s in services if ( - s.is_interested(event, restrict_to, alias_list) + s.is_interested(event, restrict_to, alias_list, member_list) ) ] defer.returnValue(interested_list) diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index bf243b618..a7877609a 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -21,9 +21,9 @@ class AppServiceRestResource(JsonResource): """A resource for version 1 of the matrix application service API.""" def __init__(self, hs): - JsonResource.__init__(self) + JsonResource.__init__(self, hs) self.register_servlets(self, hs) @staticmethod def register_servlets(appservice_resource, hs): - register.register_servlets(hs, appservice_resource) \ No newline at end of file + register.register_servlets(hs, appservice_resource) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index c0aaf1278..d12e4f264 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -143,3 +143,28 @@ class ApplicationServiceTestCase(unittest.TestCase): restrict_to=ApplicationService.NS_USERS, aliases_for_event=["#xmpp_barfoo:matrix.org"] )) + + def test_member_list_match(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + "@irc_.*" + ) + join_list = [ + Mock( + type="m.room.member", room_id="!foo:bar", sender="@alice:here", + state_key="@alice:here" + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@irc_fo:here", + state_key="@irc_fo:here" # AS user + ), + Mock( + type="m.room.member", room_id="!foo:bar", sender="@bob:here", + state_key="@bob:here" + ) + ] + + self.event.sender = "@xmpp_foobar:matrix.org" + self.assertTrue(self.service.is_interested( + event=self.event, + member_list=join_list + )) From 8beb61391609d8dd55022a14e1ef11bf353c1fdc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:36:48 +0000 Subject: [PATCH 369/630] Add newline to EOF --- synapse/rest/appservice/v1/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index a7877609a..a9d6d7256 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -27,3 +27,4 @@ class AppServiceRestResource(JsonResource): @staticmethod def register_servlets(appservice_resource, hs): register.register_servlets(hs, appservice_resource) + From fd40d992adfb8b63f6e925dad030c63498501408 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:41:33 +0000 Subject: [PATCH 370/630] PEP8-ify --- synapse/appservice/api.py | 2 -- synapse/handlers/directory.py | 3 --- synapse/rest/appservice/v1/register.py | 2 +- synapse/storage/appservice.py | 3 --- 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 15ac1e27f..6192813c0 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -71,7 +71,6 @@ class ApplicationServiceApi(SimpleHttpClient): logger.warning("query_alias to %s threw exception %s", uri, ex) defer.returnValue(False) - @defer.inlineCallbacks def push_bulk(self, service, events): events = self._serialize(events) @@ -107,4 +106,3 @@ class ApplicationServiceApi(SimpleHttpClient): return [ serialize_event(e, time_now, as_client_event=True) for e in events ] - diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 87bc12c98..20ab9e269 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -58,7 +58,6 @@ class DirectoryHandler(BaseHandler): servers ) - @defer.inlineCallbacks def create_association(self, user_id, room_alias, room_id, servers=None): # association creation for human users @@ -75,7 +74,6 @@ class DirectoryHandler(BaseHandler): ) yield self._create_association(room_alias, room_id, servers) - @defer.inlineCallbacks def create_appservice_association(self, service, room_alias, room_id, servers=None): @@ -127,7 +125,6 @@ class DirectoryHandler(BaseHandler): # if room_id: # yield self._update_room_alias_events(user_id, room_id) - @defer.inlineCallbacks def get_association(self, room_alias): room_id = None diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py index d3d5aef22..3bd0c1220 100644 --- a/synapse/rest/appservice/v1/register.py +++ b/synapse/rest/appservice/v1/register.py @@ -65,7 +65,7 @@ class RegisterRestServlet(AppServiceRestServlet): hs_token = app_service.hs_token defer.returnValue((200, { - "hs_token": hs_token + "hs_token": hs_token })) def _parse_namespace(self, target_ns, origin_ns, ns): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index ba31c6859..d941b1f38 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -97,7 +97,6 @@ class ApplicationServiceStore(SQLBaseStore): # allocate new ASes. It relies on the server admin inserting the AS # token into the database manually. - if not service.token or not service.url: raise StoreError(400, "Token and url must be specified.") @@ -186,7 +185,6 @@ class ApplicationServiceStore(SQLBaseStore): # TODO: The from_cache=False impl # TODO: This should be JOINed with the application_services_regex table. - @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" @@ -244,4 +242,3 @@ class ApplicationServiceStore(SQLBaseStore): hs_token=service["hs_token"], sender=service["sender"] )) - From 14d413752bc7f656e9df5363a8aa29bae404d10a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 10:53:47 +0000 Subject: [PATCH 371/630] Fix newline on __init__ --- synapse/rest/appservice/v1/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py index a9d6d7256..a7877609a 100644 --- a/synapse/rest/appservice/v1/__init__.py +++ b/synapse/rest/appservice/v1/__init__.py @@ -27,4 +27,3 @@ class AppServiceRestResource(JsonResource): @staticmethod def register_servlets(appservice_resource, hs): register.register_servlets(hs, appservice_resource) - From 896253e085751df88e747b29e638916b15cf7a0e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 11:37:30 +0000 Subject: [PATCH 372/630] Factor out some of the common homeserver setup code into a setup_test_homeserver function in utils. --- tests/api/test_filtering.py | 14 +--- tests/federation/test_federation.py | 11 +-- tests/handlers/test_directory.py | 14 +--- tests/handlers/test_federation.py | 11 +-- tests/handlers/test_presence.py | 16 +---- tests/handlers/test_presencelike.py | 39 +++++------ tests/handlers/test_profile.py | 29 +++----- tests/handlers/test_room.py | 19 ++--- tests/handlers/test_typing.py | 47 ++++++------- tests/rest/client/v1/test_events.py | 14 +--- tests/rest/client/v1/test_presence.py | 25 ++----- tests/rest/client/v1/test_profile.py | 13 ++-- tests/rest/client/v1/test_rooms.py | 97 ++++---------------------- tests/rest/client/v1/test_typing.py | 13 +--- tests/rest/client/v2_alpha/__init__.py | 13 ++-- tests/storage/test_base.py | 1 + tests/storage/test_directory.py | 11 +-- tests/storage/test_presence.py | 11 +-- tests/storage/test_profile.py | 10 +-- tests/storage/test_redaction.py | 14 +--- tests/storage/test_registration.py | 10 +-- tests/storage/test_room.py | 17 +---- tests/storage/test_roommember.py | 14 +--- tests/storage/test_stream.py | 14 +--- tests/utils.py | 23 ++++++ 25 files changed, 140 insertions(+), 360 deletions(-) diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index babf4c37f..65b2f590c 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -19,11 +19,9 @@ from twisted.internet import defer from mock import Mock, NonCallableMock from tests.utils import ( - MockHttpResource, MockClock, DeferredMockCallable, SQLiteMemoryDbPool, - MockKey + MockHttpResource, DeferredMockCallable, setup_test_homeserver ) -from synapse.server import HomeServer from synapse.types import UserID from synapse.api.filtering import Filter @@ -34,22 +32,14 @@ class FilteringTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - self.mock_federation_resource = MockHttpResource() self.mock_http_client = Mock(spec=[]) self.mock_http_client.put_json = DeferredMockCallable() - hs = HomeServer("test", - db_pool=db_pool, + hs = yield setup_test_homeserver( handlers=None, http_client=self.mock_http_client, - config=self.mock_config, keyring=Mock(), ) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 3e484cd30..2ecd00d2a 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -19,9 +19,8 @@ from tests import unittest # python imports from mock import Mock, ANY -from ..utils import MockHttpResource, MockClock, MockKey +from ..utils import MockHttpResource, MockClock, setup_test_homeserver -from synapse.server import HomeServer from synapse.federation import initialize_http_replication from synapse.events import FrozenEvent @@ -40,6 +39,7 @@ def make_pdu(prev_pdus=[], **kwargs): class FederationTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.mock_resource = MockHttpResource() self.mock_http_client = Mock(spec=[ @@ -61,17 +61,12 @@ class FederationTestCase(unittest.TestCase): defer.succeed(DestinationsTable.EntryType("", 0, 0)) ) self.mock_persistence.get_auth_chain.return_value = [] - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] self.clock = MockClock() - hs = HomeServer( - "test", + hs = yield setup_test_homeserver( resource_for_federation=self.mock_resource, http_client=self.mock_http_client, - db_pool=None, datastore=self.mock_persistence, clock=self.clock, - config=self.mock_config, keyring=Mock(), ) self.federation = initialize_http_replication(hs) diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 22119de46..27306ba42 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -19,11 +19,10 @@ from twisted.internet import defer from mock import Mock -from synapse.server import HomeServer from synapse.handlers.directory import DirectoryHandler from synapse.types import RoomAlias -from tests.utils import SQLiteMemoryDbPool, MockKey +from tests.utils import setup_test_homeserver class DirectoryHandlers(object): @@ -46,19 +45,10 @@ class DirectoryTestCase(unittest.TestCase): self.query_handlers[query_type] = handler self.mock_federation.register_query_handler = register_query_handler - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer( - "test", - db_pool=db_pool, + hs = yield setup_test_homeserver( http_client=None, resource_for_federation=Mock(), replication_layer=self.mock_federation, - config=self.mock_config, ) hs.handlers = DirectoryHandlers(hs) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 427048113..c13ade328 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -19,20 +19,17 @@ from tests import unittest from synapse.api.constants import EventTypes from synapse.events import FrozenEvent from synapse.handlers.federation import FederationHandler -from synapse.server import HomeServer from mock import NonCallableMock, ANY, Mock -from ..utils import MockKey +from ..utils import setup_test_homeserver class FederationTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - self.state_handler = NonCallableMock(spec_set=[ "compute_event_context", ]) @@ -43,9 +40,8 @@ class FederationTestCase(unittest.TestCase): ]) self.hostname = "test" - hs = HomeServer( + hs = yield setup_test_homeserver( self.hostname, - db_pool=None, datastore=NonCallableMock(spec_set=[ "persist_event", "store_room", @@ -61,7 +57,6 @@ class FederationTestCase(unittest.TestCase): "room_member_handler", "federation_handler", ]), - config=self.mock_config, auth=self.auth, state_handler=self.state_handler, keyring=Mock(), diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 5621a8afa..d88a977be 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -21,11 +21,9 @@ from mock import Mock, call, ANY, NonCallableMock import json from tests.utils import ( - MockHttpResource, MockClock, DeferredMockCallable, SQLiteMemoryDbPool, - MockKey + MockHttpResource, MockClock, DeferredMockCallable, setup_test_homeserver ) -from synapse.server import HomeServer from synapse.api.constants import PresenceState from synapse.api.errors import SynapseError from synapse.handlers.presence import PresenceHandler, UserPresenceCache @@ -66,30 +64,20 @@ class PresenceTestCase(unittest.TestCase): def setUp(self): self.clock = MockClock() - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - self.mock_federation_resource = MockHttpResource() self.mock_http_client = Mock(spec=[]) self.mock_http_client.put_json = DeferredMockCallable() - db_pool = None hs_kwargs = {} - if hasattr(self, "make_datastore_mock"): hs_kwargs["datastore"] = self.make_datastore_mock() - else: - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - hs = HomeServer("test", + hs = yield setup_test_homeserver( clock=self.clock, - db_pool=db_pool, handlers=None, resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, - config=self.mock_config, keyring=Mock(), **hs_kwargs ) diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 3cdbb186a..18cac9a84 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -21,9 +21,8 @@ from twisted.internet import defer from mock import Mock, call, ANY, NonCallableMock -from ..utils import MockClock, MockKey +from ..utils import MockClock, setup_test_homeserver -from synapse.server import HomeServer from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler from synapse.handlers.profile import ProfileHandler @@ -57,29 +56,23 @@ class PresenceAndProfileHandlers(object): class PresenceProfilelikeDataTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - clock=MockClock(), - db_pool=None, - datastore=Mock(spec=[ - "set_presence_state", - "is_presence_visible", - - "set_profile_displayname", - - "get_rooms_for_user_where_membership_is", - ]), - handlers=None, - resource_for_federation=Mock(), - http_client=None, - replication_layer=MockReplication(), - ratelimiter=NonCallableMock(spec_set=[ + hs = yield setup_test_homeserver( + clock=MockClock(), + datastore=Mock(spec=[ + "set_presence_state", + "is_presence_visible", + "set_profile_displayname", + "get_rooms_for_user_where_membership_is", + ]), + handlers=None, + resource_for_federation=Mock(), + http_client=None, + replication_layer=MockReplication(), + ratelimiter=NonCallableMock(spec_set=[ "send_message", - ]), - config=self.mock_config + ]), ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 7b9590c11..31f03d73d 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -20,11 +20,10 @@ from twisted.internet import defer from mock import Mock, NonCallableMock from synapse.api.errors import AuthError -from synapse.server import HomeServer from synapse.handlers.profile import ProfileHandler from synapse.types import UserID -from tests.utils import SQLiteMemoryDbPool, MockKey +from tests.utils import setup_test_homeserver class ProfileHandlers(object): @@ -46,23 +45,15 @@ class ProfileTestCase(unittest.TestCase): self.query_handlers[query_type] = handler self.mock_federation.register_query_handler = register_query_handler - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - db_pool=db_pool, - http_client=None, - handlers=None, - resource_for_federation=Mock(), - replication_layer=self.mock_federation, - config=self.mock_config, - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]) - ) + hs = yield setup_test_homeserver( + http_client=None, + handlers=None, + resource_for_federation=Mock(), + replication_layer=self.mock_federation, + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]) + ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 9a23b3812..6417f7330 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -20,23 +20,19 @@ from .. import unittest from synapse.api.constants import EventTypes, Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler from synapse.handlers.profile import ProfileHandler -from synapse.server import HomeServer from synapse.types import UserID -from ..utils import MockKey +from ..utils import setup_test_homeserver from mock import Mock, NonCallableMock class RoomMemberHandlerTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - self.hostname = "red" - hs = HomeServer( + hs = yield setup_test_homeserver( self.hostname, - db_pool=None, ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -64,7 +60,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "compute_event_context", "get_current_state", ]), - config=self.mock_config, ) self.federation = NonCallableMock(spec_set=[ @@ -319,15 +314,12 @@ class RoomMemberHandlerTestCase(unittest.TestCase): class RoomCreationTest(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.hostname = "red" - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer( + hs = yield setup_test_homeserver( self.hostname, - db_pool=None, datastore=NonCallableMock(spec_set=[ "store_room", "snapshot_room", @@ -344,7 +336,6 @@ class RoomCreationTest(unittest.TestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=self.mock_config, ) self.federation = NonCallableMock(spec_set=[ diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 8a7fc028d..bf34b7ccb 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -20,10 +20,11 @@ from twisted.internet import defer from mock import Mock, call, ANY import json -from ..utils import MockHttpResource, MockClock, DeferredMockCallable, MockKey +from ..utils import ( + MockHttpResource, MockClock, DeferredMockCallable, setup_test_homeserver +) from synapse.api.errors import AuthError -from synapse.server import HomeServer from synapse.handlers.typing import TypingNotificationHandler from synapse.storage.transactions import DestinationsTable @@ -56,6 +57,7 @@ class JustTypingNotificationHandlers(object): class TypingNotificationsTestCase(unittest.TestCase): """Tests typing notifications to rooms.""" + @defer.inlineCallbacks def setUp(self): self.clock = MockClock() @@ -64,34 +66,29 @@ class TypingNotificationsTestCase(unittest.TestCase): self.mock_federation_resource = MockHttpResource() - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - mock_notifier = Mock(spec=["on_new_user_event"]) self.on_new_user_event = mock_notifier.on_new_user_event self.auth = Mock(spec=[]) - hs = HomeServer("test", - auth=self.auth, - clock=self.clock, - db_pool=None, - datastore=Mock(spec=[ - # Bits that Federation needs - "prep_send_transaction", - "delivered_txn", - "get_received_txn_response", - "set_received_txn_response", - "get_destination_retry_timings", - ]), - handlers=None, - notifier=mock_notifier, - resource_for_client=Mock(), - resource_for_federation=self.mock_federation_resource, - http_client=self.mock_http_client, - config=self.mock_config, - keyring=Mock(), - ) + hs = yield setup_test_homeserver( + auth=self.auth, + clock=self.clock, + datastore=Mock(spec=[ + # Bits that Federation needs + "prep_send_transaction", + "delivered_txn", + "get_received_txn_response", + "set_received_txn_response", + "get_destination_retry_timings", + ]), + handlers=None, + notifier=mock_notifier, + resource_for_client=Mock(), + resource_for_federation=self.mock_federation_resource, + http_client=self.mock_http_client, + keyring=Mock(), + ) hs.handlers = JustTypingNotificationHandlers(hs) self.handler = hs.get_handlers().typing_notification_handler diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index 0384ffbb3..f36340121 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -23,9 +23,8 @@ import synapse.rest.client.v1.events import synapse.rest.client.v1.register import synapse.rest.client.v1.room -from synapse.server import HomeServer -from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from ....utils import MockHttpResource, setup_test_homeserver from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -113,15 +112,7 @@ class EventStreamPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "test", - db_pool=db_pool, + hs = yield setup_test_homeserver( http_client=None, replication_layer=Mock(), clock=Mock(spec=[ @@ -133,7 +124,6 @@ class EventStreamPermissionsTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index f849120a3..c1acba5a0 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -20,11 +20,10 @@ from twisted.internet import defer from mock import Mock -from ....utils import MockHttpResource, MockKey +from ....utils import MockHttpResource, setup_test_homeserver from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler -from synapse.server import HomeServer from synapse.rest.client.v1 import presence from synapse.rest.client.v1 import events from synapse.types import UserID @@ -46,12 +45,10 @@ class JustPresenceHandlers(object): class PresenceStateTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - hs = HomeServer("test", - db_pool=None, + hs = yield setup_test_homeserver( datastore=Mock(spec=[ "get_presence_state", "set_presence_state", @@ -60,7 +57,6 @@ class PresenceStateTestCase(unittest.TestCase): http_client=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource, - config=self.mock_config, ) hs.handlers = JustPresenceHandlers(hs) @@ -128,13 +124,11 @@ class PresenceStateTestCase(unittest.TestCase): class PresenceListTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - hs = HomeServer("test", - db_pool=None, + hs = yield setup_test_homeserver( datastore=Mock(spec=[ "has_presence_state", "get_presence_state", @@ -149,7 +143,6 @@ class PresenceListTestCase(unittest.TestCase): http_client=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource, - config=self.mock_config, ) hs.handlers = JustPresenceHandlers(hs) @@ -242,12 +235,10 @@ class PresenceListTestCase(unittest.TestCase): class PresenceEventStreamTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - # HIDEOUS HACKERY # TODO(paul): This should be injected in via the HomeServer DI system from synapse.streams.events import ( @@ -264,8 +255,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): } EventSources.SOURCE_TYPES["presence"] = PresenceEventSource - hs = HomeServer("test", - db_pool=None, + hs = yield setup_test_homeserver( http_client=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource, @@ -278,7 +268,6 @@ class PresenceEventStreamTestCase(unittest.TestCase): "cancel_call_later", "time_msec", ]), - config=self.mock_config, ) hs.get_clock().time_msec.return_value = 1000000 diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 6a2085276..5cd5767f2 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -20,10 +20,9 @@ from twisted.internet import defer from mock import Mock, NonCallableMock -from ....utils import MockHttpResource, MockKey +from ....utils import MockHttpResource, setup_test_homeserver from synapse.api.errors import SynapseError, AuthError -from synapse.server import HomeServer from synapse.types import UserID from synapse.rest.client.v1 import profile @@ -35,6 +34,7 @@ PATH_PREFIX = "/_matrix/client/api/v1" class ProfileTestCase(unittest.TestCase): """ Tests profile management. """ + @defer.inlineCallbacks def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.mock_handler = Mock(spec=[ @@ -44,17 +44,12 @@ class ProfileTestCase(unittest.TestCase): "set_avatar_url", ]) - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - db_pool=None, + hs = yield setup_test_homeserver( + "test", http_client=None, resource_for_client=self.mock_resource, federation=Mock(), replication_layer=Mock(), - datastore=None, - config=self.mock_config, ) def _get_user_by_req(request=None): diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 81ead10e7..72fb4576b 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -21,13 +21,12 @@ from twisted.internet import defer import synapse.rest.client.v1.room from synapse.api.constants import Membership -from synapse.server import HomeServer from synapse.types import UserID import json import urllib -from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from ....utils import MockHttpResource, setup_test_homeserver from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -44,21 +43,11 @@ class RoomPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( + hs = yield setup_test_homeserver( "red", - db_pool=db_pool, http_client=None, replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, + ratelimiter=NonCallableMock(spec_set=["send_message"]), ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -439,21 +428,11 @@ class RoomsMemberListTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( + hs = yield setup_test_homeserver( "red", - db_pool=db_pool, http_client=None, replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, + ratelimiter=NonCallableMock(spec_set=["send_message"]), ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -531,21 +510,11 @@ class RoomsCreateTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( + hs = yield setup_test_homeserver( "red", - db_pool=db_pool, http_client=None, replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, + ratelimiter=NonCallableMock(spec_set=["send_message"]), ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -634,21 +603,11 @@ class RoomTopicTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( + hs = yield setup_test_homeserver( "red", - db_pool=db_pool, http_client=None, replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, + ratelimiter=NonCallableMock(spec_set=["send_message"]), ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -751,21 +710,11 @@ class RoomMemberStateTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( + hs = yield setup_test_homeserver( "red", - db_pool=db_pool, http_client=None, replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, + ratelimiter=NonCallableMock(spec_set=["send_message"]), ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -888,21 +837,11 @@ class RoomMessagesTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( + hs = yield setup_test_homeserver( "red", - db_pool=db_pool, http_client=None, replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, + ratelimiter=NonCallableMock(spec_set=["send_message"]), ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -993,21 +932,13 @@ class RoomInitialSyncTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( + hs = yield setup_test_homeserver( "red", - db_pool=db_pool, http_client=None, replication_layer=Mock(), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index c5d5b06da..80f2ec9dd 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -19,10 +19,9 @@ from twisted.internet import defer import synapse.rest.client.v1.room -from synapse.server import HomeServer from synapse.types import UserID -from ....utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey +from ....utils import MockHttpResource, MockClock, setup_test_homeserver from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -42,22 +41,14 @@ class RoomTypingTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( + hs = yield setup_test_homeserver( "red", clock=self.clock, - db_pool=db_pool, http_client=None, replication_layer=Mock(), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=self.mock_config, ) self.hs = hs diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index fa70575c5..f5a5f780b 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -17,11 +17,12 @@ from tests import unittest from mock import Mock -from ....utils import MockHttpResource, MockKey +from ....utils import MockHttpResource, setup_test_homeserver -from synapse.server import HomeServer from synapse.types import UserID +from twisted.internet import defer + PATH_PREFIX = "/_matrix/client/v2_alpha" @@ -31,19 +32,15 @@ class V2AlphaRestTestCase(unittest.TestCase): # USER_ID = # TO_REGISTER = [] + @defer.inlineCallbacks def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - mock_config = Mock() - mock_config.signing_key = [MockKey()] - - hs = HomeServer("test", - db_pool=None, + hs = yield setup_test_homeserver( datastore=self.make_datastore_mock(), http_client=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource, - config=mock_config, ) def _get_user_by_token(token=None): diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index a6f1d6a33..a0a24ce09 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -22,6 +22,7 @@ from mock import Mock, call from collections import OrderedDict from synapse.server import HomeServer + from synapse.storage._base import SQLBaseStore diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py index bc9ebf35e..b9bfbc00e 100644 --- a/tests/storage/test_directory.py +++ b/tests/storage/test_directory.py @@ -17,24 +17,17 @@ from tests import unittest from twisted.internet import defer -from synapse.server import HomeServer from synapse.storage.directory import DirectoryStore from synapse.types import RoomID, RoomAlias -from tests.utils import SQLiteMemoryDbPool +from tests.utils import setup_test_homeserver class DirectoryStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "test", - db_pool=db_pool, - ) + hs = yield setup_test_homeserver() self.store = DirectoryStore(hs) diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py index 1ab193736..065eebdbc 100644 --- a/tests/storage/test_presence.py +++ b/tests/storage/test_presence.py @@ -17,24 +17,17 @@ from tests import unittest from twisted.internet import defer -from synapse.server import HomeServer from synapse.storage.presence import PresenceStore from synapse.types import UserID -from tests.utils import SQLiteMemoryDbPool, MockClock +from tests.utils import setup_test_homeserver, MockClock class PresenceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer("test", - clock=MockClock(), - db_pool=db_pool, - ) + hs = yield setup_test_homeserver(clock=MockClock()) self.store = PresenceStore(hs) diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index 84381241b..1fa783f31 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -17,23 +17,17 @@ from tests import unittest from twisted.internet import defer -from synapse.server import HomeServer from synapse.storage.profile import ProfileStore from synapse.types import UserID -from tests.utils import SQLiteMemoryDbPool +from tests.utils import setup_test_homeserver class ProfileStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer("test", - db_pool=db_pool, - ) + hs = yield setup_test_homeserver() self.store = ProfileStore(hs) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 0713dfab6..b57006fcb 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -17,11 +17,10 @@ from tests import unittest from twisted.internet import defer -from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID -from tests.utils import SQLiteMemoryDbPool, MockKey +from tests.utils import setup_test_homeserver from mock import Mock @@ -30,16 +29,7 @@ class RedactionTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer( - "test", - db_pool=db_pool, - config=self.mock_config, + hs = yield setup_test_homeserver( resource_for_federation=Mock(), http_client=None, ) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 6f8bea2f6..e0b81f2b5 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -17,22 +17,16 @@ from tests import unittest from twisted.internet import defer -from synapse.server import HomeServer from synapse.storage.registration import RegistrationStore -from tests.utils import SQLiteMemoryDbPool +from tests.utils import setup_test_homeserver class RegistrationStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer("test", - db_pool=db_pool, - ) + hs = yield setup_test_homeserver() self.store = RegistrationStore(hs) diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index 71e5d3414..c88dd446f 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -17,23 +17,17 @@ from tests import unittest from twisted.internet import defer -from synapse.server import HomeServer from synapse.api.constants import EventTypes from synapse.types import UserID, RoomID, RoomAlias -from tests.utils import SQLiteMemoryDbPool +from tests.utils import setup_test_homeserver class RoomStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer("test", - db_pool=db_pool, - ) + hs = yield setup_test_homeserver() # We can't test RoomStore on its own without the DirectoryStore, for # management of the 'room_aliases' table @@ -86,12 +80,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer("test", - db_pool=db_pool, - ) + hs = setup_test_homeserver() # Room events need the full datastore, for persist_event() and # get_room_state() diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 2b9048e2a..811fea544 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -17,11 +17,10 @@ from tests import unittest from twisted.internet import defer -from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID -from tests.utils import SQLiteMemoryDbPool, MockKey +from tests.utils import setup_test_homeserver from mock import Mock @@ -30,16 +29,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer( - "test", - db_pool=db_pool, - config=self.mock_config, + hs = yield setup_test_homeserver( resource_for_federation=Mock(), http_client=None, ) diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index b7f6e2aa8..0c9b89d76 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -17,11 +17,10 @@ from tests import unittest from twisted.internet import defer -from synapse.server import HomeServer from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID -from tests.utils import SQLiteMemoryDbPool, MockKey +from tests.utils import setup_test_homeserver from mock import Mock @@ -30,16 +29,7 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - self.mock_config = Mock() - self.mock_config.signing_key = [MockKey()] - - hs = HomeServer( - "test", - db_pool=db_pool, - config=self.mock_config, + hs = yield setup_test_homeserver( resource_for_federation=Mock(), http_client=None, ) diff --git a/tests/utils.py b/tests/utils.py index 97fa8d818..25c33492a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,6 +17,7 @@ from synapse.http.server import HttpServer from synapse.api.errors import cs_error, CodeMessageException, StoreError from synapse.api.constants import EventTypes from synapse.storage import prepare_database +from synapse.server import HomeServer from synapse.util.logcontext import LoggingContext @@ -31,6 +32,28 @@ import urlparse from inspect import getcallargs +@defer.inlineCallbacks +def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): + """Setup a homeserver suitable for running tests against. Keyword arguments + are passed to the Homeserver constructor. If no datastore is supplied a + datastore backed by an in-memory sqlite db will be given to the HS. + """ + if config is None: + config = Mock() + config.signing_key = [MockKey()] + + if datastore is None: + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer(name, db_pool=db_pool, config=config, **kargs) + else: + hs = HomeServer( + name, db_pool=None, datastore=datastore, config=config, **kargs + ) + + defer.returnValue(hs) + + def get_mock_call_args(pattern_func, mock_func): """ Return the arguments the mock function was called with interpreted by the pattern functions argument list. From d4abeb835442786fb0733debbe675a8e28e6565f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 14:19:46 +0000 Subject: [PATCH 373/630] Mention new libs in CHANGES --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 922fa5b03..113838b1c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,8 @@ Changes in develop ================== * pydenticon support -- adds dep on pydenticon + * pylru + * simplejson Changes in synapse 0.6.1 (2015-01-07) ===================================== From 4ebbaf0d4382813ba896f3e8101de12e112cbed5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 14:23:10 +0000 Subject: [PATCH 374/630] Blunty replace json with simplejson --- synapse/crypto/keyclient.py | 2 +- synapse/federation/persistence.py | 2 +- synapse/federation/transport/server.py | 2 +- synapse/http/client.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/push/__init__.py | 2 +- synapse/push/pusherpool.py | 2 +- synapse/rest/client/v1/directory.py | 2 +- synapse/rest/client/v1/login.py | 2 +- synapse/rest/client/v1/presence.py | 2 +- synapse/rest/client/v1/profile.py | 2 +- synapse/rest/client/v1/push_rule.py | 2 +- synapse/rest/client/v1/pusher.py | 2 +- synapse/rest/client/v1/register.py | 2 +- synapse/rest/client/v1/room.py | 2 +- synapse/rest/client/v2_alpha/filter.py | 2 +- synapse/rest/media/v0/content_repository.py | 2 +- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 2 +- synapse/storage/filtering.py | 2 +- synapse/storage/push_rule.py | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index cd12349f6..74008347c 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -19,7 +19,7 @@ from twisted.internet.protocol import Factory from twisted.internet import defer, reactor from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.logcontext import PreserveLoggingContext -import json +import simplejson as json import logging diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 85c82a462..8a1afc0ca 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -23,7 +23,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function -import json +import simplejson as json import logging diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 9c9f8d525..2ffb37aa1 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -20,7 +20,7 @@ from synapse.api.errors import Codes, SynapseError from synapse.util.logutils import log_function import logging -import json +import simplejson as json import re diff --git a/synapse/http/client.py b/synapse/http/client.py index 198f575cf..d500e19c8 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -23,7 +23,7 @@ from twisted.web.http_headers import Headers from StringIO import StringIO -import json +import simplejson as json import logging import urllib diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 056d446e4..406203acf 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -33,7 +33,7 @@ from synapse.api.errors import ( from syutil.crypto.jsonsign import sign_json -import json +import simplejson as json import logging import urllib import urlparse diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 418a348a5..0659a1cb9 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -22,7 +22,7 @@ import synapse.util.async import baserules import logging -import json +import simplejson as json import re logger = logging.getLogger(__name__) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 5a525befd..7483d257b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -20,7 +20,7 @@ from httppusher import HttpPusher from synapse.push import PusherConfigException import logging -import json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8ed7e2d66..420aa89f3 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -20,7 +20,7 @@ from synapse.api.errors import AuthError, SynapseError, Codes from synapse.types import RoomAlias from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json import logging diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 7116ac98e..b2257b749 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError from synapse.types import UserID from base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json class LoginRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 7feb4aadb..78d4f2b12 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -21,7 +21,7 @@ from synapse.api.errors import SynapseError from synapse.types import UserID from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 15d6f3fc6..1e77eb49c 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,7 +19,7 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID -import json +import simplejson as json class ProfileDisplaynameRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index c4e7dfcf0..b012f3108 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -27,7 +27,7 @@ from synapse.push.rulekinds import ( PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP ) -import json +import simplejson as json class PushRuleRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 80e9939b7..6045e86f3 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError, Codes from synapse.push import PusherConfigException from .base import ClientV1RestServlet, client_path_pattern -import json +import simplejson as json class PusherRestServlet(ClientV1RestServlet): diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index c0423c2d4..d3399c446 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -25,7 +25,7 @@ from synapse.util.async import run_on_reactor from hashlib import sha1 import hmac -import json +import simplejson as json import logging import urllib diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 410f19ccf..0346afb1b 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -23,7 +23,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID, RoomAlias from synapse.events.utils import serialize_event -import json +import simplejson as json import logging import urllib diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py index 6ddc495d2..703250cea 100644 --- a/synapse/rest/client/v2_alpha/filter.py +++ b/synapse/rest/client/v2_alpha/filter.py @@ -21,7 +21,7 @@ from synapse.types import UserID from ._base import client_v2_pattern -import json +import simplejson as json import logging diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index 22e26e3cd..e77a20fb2 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -25,7 +25,7 @@ from twisted.web import server, resource from twisted.internet import defer import base64 -import json +import simplejson as json import logging import os import re diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a63c59a8a..924ea8903 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,7 +44,7 @@ from syutil.jsonutil import encode_canonical_json from synapse.crypto.event_signing import compute_event_reference_hash -import json +import simplejson as json import logging import os diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3e1ab0a15..b74e74ac9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -23,7 +23,7 @@ from synapse.util.logcontext import PreserveLoggingContext, LoggingContext from twisted.internet import defer import collections -import json +import simplejson as json import sys import time diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index e86eeced4..457a11fd0 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import SQLBaseStore -import json +import simplejson as json class FilteringStore(SQLBaseStore): diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 620de7139..ae46b39cc 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -20,7 +20,7 @@ from twisted.internet import defer import logging import copy -import json +import simplejson as json logger = logging.getLogger(__name__) From d8324d5a2b9ffc6f3a426ecd240f6c2460630025 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 14:52:23 +0000 Subject: [PATCH 375/630] Add a lru cache class --- synapse/util/lrucache.py | 110 ++++++++++++++++++++++++++++++++++++ tests/util/test_lrucache.py | 56 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 synapse/util/lrucache.py create mode 100644 tests/util/test_lrucache.py diff --git a/synapse/util/lrucache.py b/synapse/util/lrucache.py new file mode 100644 index 000000000..a45c673d3 --- /dev/null +++ b/synapse/util/lrucache.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class LruCache(object): + """Least-recently-used cache.""" + # TODO(mjark) Add hit/miss counters + # TODO(mjark) Add mutex for linked list for thread safety. + def __init__(self, max_size): + cache = {} + list_root = [] + list_root[:] = [list_root, list_root, None, None] + + PREV, NEXT, KEY, VALUE = 0, 1, 2, 3 + + def add_node(key, value): + prev_node = list_root + next_node = prev_node[NEXT] + node = [prev_node, next_node, key, value] + prev_node[NEXT] = node + next_node[PREV] = node + cache[key] = node + + def move_node_to_front(node): + prev_node = node[PREV] + next_node = node[NEXT] + prev_node[NEXT] = next_node + next_node[PREV] = prev_node + prev_node = list_root + next_node = prev_node[NEXT] + node[PREV] = prev_node + node[NEXT] = next_node + prev_node[NEXT] = node + next_node[PREV] = node + + def delete_node(node): + prev_node = node[PREV] + next_node = node[NEXT] + prev_node[NEXT] = next_node + next_node[PREV] = prev_node + cache.pop(node[KEY], None) + + def cache_get(key, default=None): + node = cache.get(key, None) + if node is not None: + move_node_to_front(node) + return node[VALUE] + else: + return default + + def cache_set(key, value): + node = cache.get(key, None) + if node is not None: + move_node_to_front(node) + node[VALUE] = value + else: + add_node(key, value) + if len(cache) > max_size: + delete_node(list_root[PREV]) + + def cache_set_default(key, value): + node = cache.get(key, None) + if node is not None: + return node[VALUE] + else: + add_node(key, value) + if len(cache) > max_size: + delete_node(list_root[PREV]) + return value + + def cache_pop(key, default=None): + node = cache.get(key, None) + if node: + delete_node(node) + return node[VALUE] + else: + return default + + self.sentinel = object() + self.get = cache_get + self.set = cache_set + self.setdefault = cache_set_default + self.pop = cache_pop + + def __getitem__(self, key): + result = self.get(key, self.sentinel) + if result is self.sentinel: + raise KeyError() + else: + return result + + def __setitem__(self, key, value): + self.set(key, value) + + def __delitem__(self, key, value): + result = self.pop(key, self.sentinel) + if result is self.sentinel: + raise KeyError() diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py new file mode 100644 index 000000000..ab934bf92 --- /dev/null +++ b/tests/util/test_lrucache.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .. import unittest + +from synapse.util.lrucache import LruCache + +class LruCacheTestCase(unittest.TestCase): + + def test_get_set(self): + cache = LruCache(1) + cache["key"] = "value" + self.assertEquals(cache.get("key"), "value") + self.assertEquals(cache["key"], "value") + + def test_eviction(self): + cache = LruCache(2) + cache[1] = 1 + cache[2] = 2 + + self.assertEquals(cache.get(1), 1) + self.assertEquals(cache.get(2), 2) + + cache[3] = 3 + + self.assertEquals(cache.get(1), None) + self.assertEquals(cache.get(2), 2) + self.assertEquals(cache.get(3), 3) + + def test_setdefault(self): + cache = LruCache(1) + self.assertEquals(cache.setdefault("key", 1), 1) + self.assertEquals(cache.get("key"), 1) + self.assertEquals(cache.setdefault("key", 2), 1) + self.assertEquals(cache.get("key"), 1) + + def test_pop(self): + cache = LruCache(1) + cache["key"] = 1 + self.assertEquals(cache.pop("key"), 1) + self.assertEquals(cache.pop("key"), None) + + From f5a70e0d2e890adea53b3f6565a3bbe92512a506 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 15:01:15 +0000 Subject: [PATCH 376/630] Add a cache for get_event --- synapse/config/_base.py | 10 ++++++++++ synapse/config/database.py | 5 +++++ synapse/storage/__init__.py | 3 +++ synapse/storage/_base.py | 24 +++++++++++++++++++++--- tests/storage/test_base.py | 5 +++-- tests/utils.py | 1 + 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 9b0f8c3c3..87cdbf1d3 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,6 +27,16 @@ class Config(object): def __init__(self, args): pass + @staticmethod + def parse_size(string): + sizes = {"K": 1024, "M": 1024 * 1024} + size = 1 + suffix = string[-1] + if suffix in sizes: + string = string[:-1] + size = sizes[suffix] + return int(string) * size + @staticmethod def abspath(file_path): return os.path.abspath(file_path) if file_path else file_path diff --git a/synapse/config/database.py b/synapse/config/database.py index daa161c95..87efe5464 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -24,6 +24,7 @@ class DatabaseConfig(Config): self.database_path = ":memory:" else: self.database_path = self.abspath(args.database_path) + self.event_cache_size = self.parse_size(args.event_cache_size) @classmethod def add_arguments(cls, parser): @@ -33,6 +34,10 @@ class DatabaseConfig(Config): "-d", "--database-path", default="homeserver.db", help="The database name." ) + db_group.add_argument( + "--event-cache-size", default="100K", + help="Number of events to cache in memory." + ) @classmethod def generate_config(cls, args, config_dir_path): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a63c59a8a..1170d8b6e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -164,6 +164,9 @@ class DataStore(RoomMemberStore, RoomStore, stream_ordering=None, is_new_state=True, current_state=None): + # Remove the any existing cache entries for the event_id + self._get_event_cache.pop(event.event_id) + # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table if current_state: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3e1ab0a15..f13b8f4fa 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,6 +19,7 @@ from synapse.events import FrozenEvent from synapse.events.utils import prune_event from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext, LoggingContext +from synapse.util.lrucache import LruCache from twisted.internet import defer @@ -128,6 +129,8 @@ class SQLBaseStore(object): self._txn_perf_counters = PerformanceCounters() self._get_event_counters = PerformanceCounters() + self._get_event_cache = LruCache(hs.config.event_cache_size) + def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -579,6 +582,20 @@ class SQLBaseStore(object): def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): + + start_time = time.time() * 1000 + update_counter = self._get_event_counters.update + + try: + cache = self._get_event_cache.setdefault(event_id, {}) + # Separate cache entries for each way to invoke _get_event_txn + return cache[(check_redacted, get_prev_content, allow_rejected)] + except KeyError: + pass + finally: + start_time = update_counter("event_cache", start_time) + + sql = ( "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " "FROM event_json as e " @@ -588,7 +605,6 @@ class SQLBaseStore(object): "LIMIT 1 " ) - start_time = time.time() * 1000 txn.execute(sql, (event_id,)) @@ -599,14 +615,16 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res - self._get_event_counters.update("select_event", start_time) + start_time = update_counter("select_event", start_time) if allow_rejected or not rejected_reason: - return self._get_event_from_row_txn( + result = self._get_event_from_row_txn( txn, internal_metadata, js, redacted, check_redacted=check_redacted, get_prev_content=get_prev_content, ) + cache[(check_redacted, get_prev_content, allow_rejected)] = result + return result else: return None diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index a0a24ce09..55fbffa7a 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -38,8 +38,9 @@ class SQLBaseStoreTestCase(unittest.TestCase): return defer.succeed(func(self.mock_txn, *args, **kwargs)) self.db_pool.runInteraction = runInteraction - hs = HomeServer("test", - db_pool=self.db_pool) + config = Mock() + config.event_cache_size = 1 + hs = HomeServer("test", db_pool=self.db_pool, config=config) self.datastore = SQLBaseStore(hs) diff --git a/tests/utils.py b/tests/utils.py index 25c33492a..39895c739 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -41,6 +41,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): if config is None: config = Mock() config.signing_key = [MockKey()] + config.event_cache_size = 1 if datastore is None: db_pool = SQLiteMemoryDbPool() From aff892ce79b9cd5060d268eefb5876fc7caf30f0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 15:02:35 +0000 Subject: [PATCH 377/630] Fix formatting --- synapse/storage/_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f13b8f4fa..29fc334f4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -595,7 +595,6 @@ class SQLBaseStore(object): finally: start_time = update_counter("event_cache", start_time) - sql = ( "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " "FROM event_json as e " @@ -605,7 +604,6 @@ class SQLBaseStore(object): "LIMIT 1 " ) - txn.execute(sql, (event_id,)) res = txn.fetchone() From ddc25cf4e2df10e2b175e70a2371a7e24081572f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 15:23:28 +0000 Subject: [PATCH 378/630] Invalidate the cache for an event if it is redacted --- synapse/storage/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1170d8b6e..a33e2298f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -423,6 +423,8 @@ class DataStore(RoomMemberStore, RoomStore, ) def _store_redaction(self, txn, event): + # invalidate the cache for the redacted event + self._get_event_cache.pop(event.redacts) txn.execute( "INSERT OR IGNORE INTO redactions " "(event_id, redacts) VALUES (?,?)", From ddb816cf60ca1b0c0f9bfab5df233a010ac309a3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 15:44:28 +0000 Subject: [PATCH 379/630] Don't unfreeze when using FreezeEvent.get_dict, as we are using a JSONEncoder that understands FrozenDict --- synapse/events/__init__.py | 4 ---- synapse/federation/persistence.py | 7 ++++--- synapse/handlers/federation.py | 15 ++++++++++++++- synapse/handlers/room.py | 4 +++- synapse/storage/__init__.py | 11 +++++++++-- synapse/util/frozenutils.py | 8 ++++++-- 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 8f0c6e959..f4ec8cd18 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -140,10 +140,6 @@ class FrozenEvent(EventBase): return e - def get_dict(self): - # We need to unfreeze what we return - return unfreeze(super(FrozenEvent, self).get_dict()) - def __str__(self): return self.__repr__() diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 8a1afc0ca..76a9dcd77 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -23,7 +23,8 @@ from twisted.internet import defer from synapse.util.logutils import log_function -import simplejson as json +from syutil.jsonutil import encode_canonical_json + import logging @@ -70,7 +71,7 @@ class TransactionActions(object): transaction.transaction_id, transaction.origin, code, - json.dumps(response) + encode_canonical_json(response) ) @defer.inlineCallbacks @@ -100,5 +101,5 @@ class TransactionActions(object): transaction.transaction_id, transaction.destination, response_code, - json.dumps(response_dict) + encode_canonical_json(response_dict) ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0f9c82fd0..e36f0945e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -23,6 +23,7 @@ from synapse.api.errors import ( from synapse.api.constants import EventTypes, Membership, RejectedReason from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.util.frozenutils import unfreeze from synapse.crypto.event_signing import ( compute_event_signature, add_hashes_and_signatures, ) @@ -311,9 +312,12 @@ class FederationHandler(BaseHandler): self.room_queues[room_id] = [] builder = self.event_builder_factory.new( - event.get_pdu_json() + unfreeze(event.get_pdu_json()) ) + logger.info("Builder: %s", builder.get_pdu_json()) + logger.info("Content: %s", content) + handled_events = set() try: @@ -324,14 +328,21 @@ class FederationHandler(BaseHandler): if not hasattr(event, "signatures"): builder.signatures = {} + logger.info("Content befhahs: %s", builder.content) + add_hashes_and_signatures( builder, self.hs.hostname, self.hs.config.signing_key[0], ) + logger.info("Content aftet hah: %s", builder.content) + logger.info("Content pdu json: %s", builder.get_pdu_json()) + new_event = builder.build() + logger.info("Content after build: %s", new_event.content) + # Try the host we successfully got a response to /make_join/ # request first. try: @@ -340,6 +351,7 @@ class FederationHandler(BaseHandler): except ValueError: pass + logger.info(new_event.content) ret = yield self.replication_layer.send_join( target_hosts, new_event @@ -485,6 +497,7 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False + logger.info(event.content) context = yield self._handle_new_event(origin, event) logger.debug( diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0369b907a..c685991a9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -403,7 +403,9 @@ class RoomMemberHandler(BaseHandler): "membership": Membership.JOIN, "content": content, }) + logger.info("Before build: %s", builder.get_pdu_json()) event, context = yield self._create_new_client_event(builder) + logger.info("After build: %s", event.get_dict()) yield self._do_join(event, context, room_hosts=hosts, do_auth=True) @@ -462,7 +464,7 @@ class RoomMemberHandler(BaseHandler): room_hosts, room_id, event.user_id, - event.get_dict()["content"], # FIXME To get a non-frozen dict + event.content, # FIXME To get a non-frozen dict context ) else: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 924ea8903..94603ef82 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -298,12 +298,16 @@ class DataStore(RoomMemberStore, RoomStore, or_replace=True, ) + content = encode_canonical_json( + event.content + ).decode("UTF-8") + vals = { "topological_ordering": event.depth, "event_id": event.event_id, "type": event.type, "room_id": event.room_id, - "content": json.dumps(event.get_dict()["content"]), + "content": content, "processed": True, "outlier": outlier, "depth": event.depth, @@ -323,7 +327,10 @@ class DataStore(RoomMemberStore, RoomStore, "prev_events", ] } - vals["unrecognized_keys"] = json.dumps(unrec) + + vals["unrecognized_keys"] = encode_canonical_json( + unrec + ).decode("UTF-8") try: self._simple_insert_txn( diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index a13a2015e..9e10d37ae 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -21,6 +21,9 @@ def freeze(o): if t is dict: return frozendict({k: freeze(v) for k, v in o.items()}) + if t is frozendict: + return o + if t is str or t is unicode: return o @@ -33,10 +36,11 @@ def freeze(o): def unfreeze(o): - if isinstance(o, frozendict) or isinstance(o, dict): + t = type(o) + if t is dict or t is frozendict: return dict({k: unfreeze(v) for k, v in o.items()}) - if isinstance(o, basestring): + if t is str or t is unicode: return o try: From 7be0f6594e2a6dd7c3dc745eb856025276ec7d1f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 11 Feb 2015 15:53:56 +0000 Subject: [PATCH 380/630] First step of making user_rooms_intersect() faster - implement in intersection logic in Python code terms of a DB query that is cacheable per user --- synapse/storage/roommember.py | 36 ++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index c69dd995c..d490a374e 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -240,28 +240,30 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results + @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a room. + + This is a "hot path" function that's called a lot, e.g. by presence for + generating the event stream. """ - def interaction(txn): - user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) - sql = ( - "SELECT m.room_id FROM room_memberships as m " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE m.membership = 'join' " - "AND (%(clause)s) " - # TODO(paul): We've got duplicate rows in the database somewhere - # so we have to DISTINCT m.user_id here - "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" - ) % {"clause": user_list_clause} + if len(user_id_list) < 2: + defer.returnValue(True) - args = list(user_id_list) - args.append(len(user_id_list)) + deferreds = [ + self.get_rooms_for_user_where_membership_is( + u, membership_list=[Membership.JOIN], + ) + for u in user_id_list + ] - txn.execute(sql, args) + results = yield defer.DeferredList(deferreds) - return len(txn.fetchall()) > 0 + # A list of sets of strings giving room IDs for each user + room_id_lists = [set([r.room_id for r in result[1]]) for result in results] - return self.runInteraction("user_rooms_intersect", interaction) + # There isn't a setintersection(*list_of_sets) + ret = len(room_id_lists.pop(0).intersection(*room_id_lists)) > 0 + + defer.returnValue(ret) From 45b56609ae84e7ffc3713335e7d9abc315ad1725 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 11 Feb 2015 16:04:08 +0000 Subject: [PATCH 381/630] Cache the result of a get_rooms_for_user query, to make user_rooms_intersect() much lighter in the read-common case --- synapse/storage/roommember.py | 41 +++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index d490a374e..e05465bc1 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,6 +35,11 @@ RoomsForUser = namedtuple( class RoomMemberStore(SQLBaseStore): + def __init__(self, *args, **kw): + super(RoomMemberStore, self).__init__(*args, **kw) + + self._user_rooms_cache = {} + def _store_room_member_txn(self, txn, event): """Store a room member in the database. """ @@ -98,6 +103,8 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (event.room_id, domain)) + self.invalidate_rooms_for_user(target_user_id) + @defer.inlineCallbacks def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. @@ -240,23 +247,43 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results + # TODO(paul): Create a nice @cached decorator to do this + # @cached + # def get_foo(...) + # ... + # invalidate_foo = get_foo.invalidator + + @defer.inlineCallbacks + def get_rooms_for_user(self, user_id): + # TODO(paul): put some performance counters in here so we can easily + # track what impact this cache is having + if user_id in self._user_rooms_cache: + defer.returnValue(self._user_rooms_cache[user_id]) + + rooms = yield self.get_rooms_for_user_where_membership_is( + user_id, membership_list=[Membership.JOIN], + ) + + self._user_rooms_cache[user_id] = rooms + defer.returnValue(rooms) + + def invalidate_rooms_for_user(self, user_id): + if user_id in self._user_rooms_cache: + del self._user_rooms_cache[user_id] + @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a room. This is a "hot path" function that's called a lot, e.g. by presence for - generating the event stream. + generating the event stream. As such, it is implemented locally by + wrapping logic around heavily-cached database queries. """ if len(user_id_list) < 2: defer.returnValue(True) - deferreds = [ - self.get_rooms_for_user_where_membership_is( - u, membership_list=[Membership.JOIN], - ) - for u in user_id_list - ] + deferreds = [self.get_rooms_for_user(u) for u in user_id_list] results = yield defer.DeferredList(deferreds) From 7f47ba7383302fdbdaa3a10abef00d3710c77fce Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 11 Feb 2015 16:18:21 +0000 Subject: [PATCH 382/630] Added another TODO note --- synapse/storage/roommember.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index e05465bc1..779f9ce54 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -264,6 +264,9 @@ class RoomMemberStore(SQLBaseStore): user_id, membership_list=[Membership.JOIN], ) + # TODO(paul): Consider applying a maximum size; just evict things at + # random, or consider LRU? + self._user_rooms_cache[user_id] = rooms defer.returnValue(rooms) From f51832442674a1f72c41ffce2279880109fc7ff0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 16:41:16 +0000 Subject: [PATCH 383/630] Minor tweaks based on PR feedback. --- synapse/appservice/api.py | 6 +++--- synapse/http/client.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 6192813c0..c2179f8d5 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -80,11 +80,11 @@ class ApplicationServiceApi(SimpleHttpClient): response = None try: response = yield self.put_json( - uri, - { + uri=uri, + json_body={ "events": events }, - { + args={ "access_token": service.hs_token }) if response: # just an empty json object diff --git a/synapse/http/client.py b/synapse/http/client.py index 575510637..7b2311655 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -95,7 +95,8 @@ class SimpleHttpClient(object): Deferred: Succeeds when we get *any* 2xx HTTP response, with the HTTP body as JSON. Raises: - On a non-2xx HTTP response. + On a non-2xx HTTP response. The response body will be used as the + error message. """ if len(args): query_bytes = urllib.urlencode(args, True) From cb43fbeeb47e86e2f572dd647f9e59ed181bbf79 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 11 Feb 2015 16:46:01 +0000 Subject: [PATCH 384/630] Fix tests which broke when event caching was introduced. --- tests/storage/test_appservice.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index b9ecfb338..fc733d4c7 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -19,6 +19,7 @@ from synapse.appservice import ApplicationService from synapse.server import HomeServer from synapse.storage.appservice import ApplicationServiceStore +from mock import Mock from tests.utils import SQLiteMemoryDbPool, MockClock @@ -28,7 +29,9 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): def setUp(self): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() - hs = HomeServer("test", db_pool=db_pool, clock=MockClock()) + hs = HomeServer( + "test", db_pool=db_pool, clock=MockClock(), config=Mock() + ) self.as_token = "token1" db_pool.runQuery( "INSERT INTO application_services(token) VALUES(?)", From ef276e8770e19c66d14462aa325b9cb241121bb6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:48:05 +0000 Subject: [PATCH 385/630] Fix so timing out connections to actually work. --- synapse/federation/replication.py | 2 ++ synapse/util/__init__.py | 52 +++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index e442c6c5d..54a0c7ad8 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -72,5 +72,7 @@ class ReplicationLayer(FederationClient, FederationServer): self._order = 0 + self.hs = hs + def __str__(self): return "" % self.server_name diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 29f1344c5..e77eba90a 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -18,6 +18,9 @@ from synapse.util.logcontext import LoggingContext from twisted.internet import defer, reactor, task import time +import logging + +logger = logging.getLogger(__name__) class Clock(object): @@ -55,20 +58,51 @@ class Clock(object): timer.cancel() def time_bound_deferred(self, given_deferred, time_out): + if given_deferred.called: + return given_deferred + ret_deferred = defer.Deferred() - def timed_out(): - if not given_deferred.called: - given_deferred.cancel() + def timed_out_fn(): + try: ret_deferred.errback(RuntimeError("Timed out")) + except: + pass - timer = self.call_later(time_out, timed_out) + try: + given_deferred.cancel() + except: + pass - def succeed(result): - self.cancel_call_later(timer) - ret_deferred.callback(result) + timer = None - given_deferred.addCallback(succeed) - given_deferred.addErrback(ret_deferred.errback) + def cancel(res): + try: + self.cancel_call_later(timer) + except: + pass + return res + + ret_deferred.addBoth(cancel) + + def sucess(res): + try: + ret_deferred.callback(res) + except: + pass + + return res + + def err(res): + try: + ret_deferred.errback(res) + except: + pass + + return res + + given_deferred.addCallbacks(callback=sucess, errback=err) + + timer = self.call_later(time_out, timed_out_fn) return ret_deferred From ed877d658500b736e7f7192fbeb64aac444ef4d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:50:46 +0000 Subject: [PATCH 386/630] Remove debug logging --- synapse/handlers/federation.py | 10 ---------- synapse/handlers/room.py | 2 -- 2 files changed, 12 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e36f0945e..0c35317bb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -315,9 +315,6 @@ class FederationHandler(BaseHandler): unfreeze(event.get_pdu_json()) ) - logger.info("Builder: %s", builder.get_pdu_json()) - logger.info("Content: %s", content) - handled_events = set() try: @@ -328,21 +325,14 @@ class FederationHandler(BaseHandler): if not hasattr(event, "signatures"): builder.signatures = {} - logger.info("Content befhahs: %s", builder.content) - add_hashes_and_signatures( builder, self.hs.hostname, self.hs.config.signing_key[0], ) - logger.info("Content aftet hah: %s", builder.content) - logger.info("Content pdu json: %s", builder.get_pdu_json()) - new_event = builder.build() - logger.info("Content after build: %s", new_event.content) - # Try the host we successfully got a response to /make_join/ # request first. try: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c685991a9..914742d91 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -403,9 +403,7 @@ class RoomMemberHandler(BaseHandler): "membership": Membership.JOIN, "content": content, }) - logger.info("Before build: %s", builder.get_pdu_json()) event, context = yield self._create_new_client_event(builder) - logger.info("After build: %s", event.get_dict()) yield self._do_join(event, context, room_hosts=hosts, do_auth=True) From 76935078d1983aec9966f2eb245e4186dcee0386 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:51:22 +0000 Subject: [PATCH 387/630] Remove more debug logging --- synapse/handlers/federation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0c35317bb..77c81fe2d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -341,7 +341,6 @@ class FederationHandler(BaseHandler): except ValueError: pass - logger.info(new_event.content) ret = yield self.replication_layer.send_join( target_hosts, new_event @@ -487,7 +486,6 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - logger.info(event.content) context = yield self._handle_new_event(origin, event) logger.debug( From 7eef84a95b53318e97ab74dd25fee584714a8e6a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 16:52:22 +0000 Subject: [PATCH 388/630] pyflakes --- synapse/events/__init__.py | 2 +- synapse/storage/__init__.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index f4ec8cd18..64e08223b 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.util.frozenutils import freeze, unfreeze +from synapse.util.frozenutils import freeze class _EventInternalMetadata(object): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 94603ef82..a0284d54b 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,7 +44,6 @@ from syutil.jsonutil import encode_canonical_json from synapse.crypto.event_signing import compute_event_reference_hash -import simplejson as json import logging import os From 9c24cff6ef789f277e3c6663d356abb3c915aaa8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 17:00:20 +0000 Subject: [PATCH 389/630] Allow newer versions of syutil --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index e2a9d1f6a..c7a685e71 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -4,7 +4,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil==0.0.2": ["syutil"], + "syutil>=0.0.2": ["syutil"], "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], From a793a0b810f0f5d8fa4018b52d98471cc32fdf80 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:02:52 +0000 Subject: [PATCH 390/630] Bump syutil version --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index c7a685e71..8164effee 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -4,7 +4,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.2": ["syutil"], + "syutil>=0.0.3": ["syutil"], "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], From af89456c3c6a0dbc871c2c535840c5f95c33df32 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:15:16 +0000 Subject: [PATCH 391/630] Update dependency links --- synapse/python_dependencies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 8164effee..e099a42b5 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -26,8 +26,8 @@ def github_link(project, version, egg): DEPENDENCY_LINKS = [ github_link( project="matrix-org/syutil", - version="v0.0.2", - egg="syutil-0.0.2", + version="v0.0.3", + egg="syutil-0.0.3", ), github_link( project="matrix-org/matrix-angular-sdk", From e9e54449f5ff830c36c7ec6935e723ae68dca942 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:32:42 +0000 Subject: [PATCH 392/630] Use encode_canonical_json for pushes --- synapse/push/pusherpool.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 7483d257b..90babd722 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -19,6 +19,8 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException +from syutil.jsonutil import encode_canonical_json + import logging import simplejson as json @@ -96,7 +98,7 @@ class PusherPool: pushkey=pushkey, pushkey_ts=self.hs.get_clock().time_msec(), lang=lang, - data=json.dumps(data) + data=encode_canonical_json(data).decode("UTF-8"), ) self._refresh_pusher((app_id, pushkey)) From 2bf0e85f3d43d41f450df4c475c9deb542999d82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 17:34:23 +0000 Subject: [PATCH 393/630] Use encode_canonical_json for http client --- synapse/http/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index d500e19c8..43b232978 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -15,6 +15,8 @@ from synapse.http.agent_name import AGENT_NAME +from syutil.jsonutil import encode_canonical_json + from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError @@ -64,7 +66,7 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def post_json_get_json(self, uri, post_json): - json_str = json.dumps(post_json) + json_str = encode_canonical_json(post_json) logger.info("HTTP POST %s -> %s", json_str, uri) From 8bbdf328498ef9a7e6c46e565195783b208bfeb2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 Feb 2015 18:56:13 +0000 Subject: [PATCH 394/630] Convert get_rooms to use runInteraction so the transacion has a more helpful description --- synapse/storage/room.py | 69 +++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 6542f8e4f..750b17a45 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -82,39 +82,46 @@ class RoomStore(SQLBaseStore): "topic" key if one is set, and a "name" key if one is set """ - topic_subquery = ( - "SELECT topics.event_id as event_id, " - "topics.room_id as room_id, topic " - "FROM topics " - "INNER JOIN current_state_events as c " - "ON c.event_id = topics.event_id " + def f(txn): + topic_subquery = ( + "SELECT topics.event_id as event_id, " + "topics.room_id as room_id, topic " + "FROM topics " + "INNER JOIN current_state_events as c " + "ON c.event_id = topics.event_id " + ) + + name_subquery = ( + "SELECT room_names.event_id as event_id, " + "room_names.room_id as room_id, name " + "FROM room_names " + "INNER JOIN current_state_events as c " + "ON c.event_id = room_names.event_id " + ) + + # We use non printing ascii character US () as a seperator + sql = ( + "SELECT r.room_id, n.name, t.topic, " + "group_concat(a.room_alias, '') " + "FROM rooms AS r " + "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " + "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " + "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " + "WHERE r.is_public = ? " + "GROUP BY r.room_id " + ) % { + "topic": topic_subquery, + "name": name_subquery, + } + + c = txn.execute(sql, (is_public,)) + + return c.fetchall() + + rows = yield self.runInteraction( + "get_rooms", f ) - name_subquery = ( - "SELECT room_names.event_id as event_id, " - "room_names.room_id as room_id, name " - "FROM room_names " - "INNER JOIN current_state_events as c " - "ON c.event_id = room_names.event_id " - ) - - # We use non printing ascii character US () as a seperator - sql = ( - "SELECT r.room_id, n.name, t.topic, " - "group_concat(a.room_alias, '') " - "FROM rooms AS r " - "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " - "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " - "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " - "WHERE r.is_public = ? " - "GROUP BY r.room_id " - ) % { - "topic": topic_subquery, - "name": name_subquery, - } - - rows = yield self._execute(None, sql, is_public) - ret = [ { "room_id": r[0], From ff2a2ae56ecd68c180e1d2548dffbe068d61f70f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:02:06 +0000 Subject: [PATCH 395/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 8fe8df4ed..915af3fe0 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.6.1f" +__version__ = "0.7.0" From 83d41f25d8c57a2fa3a5feed4b952690abbd57a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:05:47 +0000 Subject: [PATCH 396/630] Set database schema version in delta --- synapse/storage/schema/delta/v12.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/v12.sql index 302d958db..b87ef1fe7 100644 --- a/synapse/storage/schema/delta/v12.sql +++ b/synapse/storage/schema/delta/v12.sql @@ -63,3 +63,5 @@ CREATE TABLE IF NOT EXISTS user_filters( CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id ); + +PRAGMA user_version = 12; From b7df941589f42867a46388ec96933deae3f75958 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:30:34 +0000 Subject: [PATCH 397/630] Update UPGRADES to mention updated dependencies --- UPGRADE.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/UPGRADE.rst b/UPGRADE.rst index 0f81f3e11..b8f20c4d5 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,17 @@ +Upgrading to v0.7.0 +=================== + +New dependencies are: + + - pydenticon + - simplejson + - syutil + - matrix-angular-sdk + +To pull in these dependencies in a virtual env, run:: + + python synapse/python_dependencies.py | xargs -n 1 pip install + Upgrading to v0.6.0 =================== From 7fedc36ccc9459897e61df7b437c8555caa4aef3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:31:31 +0000 Subject: [PATCH 398/630] Fix rst --- UPGRADE.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index b8f20c4d5..6ea348acc 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -3,10 +3,10 @@ Upgrading to v0.7.0 New dependencies are: - - pydenticon - - simplejson - - syutil - - matrix-angular-sdk +- pydenticon +- simplejson +- syutil +- matrix-angular-sdk To pull in these dependencies in a virtual env, run:: From 74f49b872ee04a70e1bb53c6c0157d17ea476b31 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:42:45 +0000 Subject: [PATCH 399/630] Update CHANGES.rst --- CHANGES.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 113838b1c..adc0d4f40 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,16 @@ Changes in develop ================== - * pydenticon support -- adds dep on pydenticon - * pylru - * simplejson + * Add initial implementation of the `query auth` fedeartion API, allowing + servers to agree on whether an event should be allowed or rejected. + * Persist events we have rejected from federation, fixing the bug where + servers would keep requesting the same events. + * Various performance improvements, including: + - Caching common queries such as fetching events and current state. + - Upgraded JSON library. + * Add default avatars to new user accounts using pydenticon library. + * Correctly time out federation requests. + * Retry federation requests against different servers. Changes in synapse 0.6.1 (2015-01-07) ===================================== From 2ad2b0dccae45a40953f141d9be6dbc3d996c28a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:43:43 +0000 Subject: [PATCH 400/630] Fix rst formatting --- CHANGES.rst | 72 ++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index adc0d4f40..f2fef17c9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,68 +1,68 @@ Changes in develop ================== - * Add initial implementation of the `query auth` fedeartion API, allowing - servers to agree on whether an event should be allowed or rejected. - * Persist events we have rejected from federation, fixing the bug where - servers would keep requesting the same events. - * Various performance improvements, including: +* Add initial implementation of the `query auth` fedeartion API, allowing + servers to agree on whether an event should be allowed or rejected. +* Persist events we have rejected from federation, fixing the bug where + servers would keep requesting the same events. +* Various performance improvements, including: - Caching common queries such as fetching events and current state. - Upgraded JSON library. - * Add default avatars to new user accounts using pydenticon library. - * Correctly time out federation requests. - * Retry federation requests against different servers. +* Add default avatars to new user accounts using pydenticon library. +* Correctly time out federation requests. +* Retry federation requests against different servers. Changes in synapse 0.6.1 (2015-01-07) ===================================== - * Major optimizations to improve performance of initial sync and event sending - in large rooms (by up to 10x) - * Media repository now includes a Content-Length header on media downloads. - * Improve quality of thumbnails by changing resizing algorithm. +* Major optimizations to improve performance of initial sync and event sending + in large rooms (by up to 10x) +* Media repository now includes a Content-Length header on media downloads. +* Improve quality of thumbnails by changing resizing algorithm. Changes in synapse 0.6.0 (2014-12-16) ===================================== - * Add new API for media upload and download that supports thumbnailing. - * Replicate media uploads over multiple homeservers so media is always served - to clients from their local homeserver. This obsoletes the - --content-addr parameter and confusion over accessing content directly - from remote homeservers. - * Implement exponential backoff when retrying federation requests when - sending to remote homeservers which are offline. - * Implement typing notifications. - * Fix bugs where we sent events with invalid signatures due to bugs where - we incorrectly persisted events. - * Improve performance of database queries involving retrieving events. +* Add new API for media upload and download that supports thumbnailing. +* Replicate media uploads over multiple homeservers so media is always served + to clients from their local homeserver. This obsoletes the + --content-addr parameter and confusion over accessing content directly + from remote homeservers. +* Implement exponential backoff when retrying federation requests when + sending to remote homeservers which are offline. +* Implement typing notifications. +* Fix bugs where we sent events with invalid signatures due to bugs where + we incorrectly persisted events. +* Improve performance of database queries involving retrieving events. Changes in synapse 0.5.4a (2014-12-13) ====================================== - * Fix bug while generating the error message when a file path specified in - the config doesn't exist. +* Fix bug while generating the error message when a file path specified in + the config doesn't exist. Changes in synapse 0.5.4 (2014-12-03) ===================================== - * Fix presence bug where some rooms did not display presence updates for - remote users. - * Do not log SQL timing log lines when started with "-v" - * Fix potential memory leak. +* Fix presence bug where some rooms did not display presence updates for + remote users. +* Do not log SQL timing log lines when started with "-v" +* Fix potential memory leak. Changes in synapse 0.5.3c (2014-12-02) ====================================== - * Change the default value for the `content_addr` option to use the HTTP - listener, as by default the HTTPS listener will be using a self-signed - certificate. +* Change the default value for the `content_addr` option to use the HTTP + listener, as by default the HTTPS listener will be using a self-signed + certificate. Changes in synapse 0.5.3 (2014-11-27) ===================================== - * Fix bug that caused joining a remote room to fail if a single event was not - signed correctly. - * Fix bug which caused servers to continuously try and fetch events from other - servers. +* Fix bug that caused joining a remote room to fail if a single event was not + signed correctly. +* Fix bug which caused servers to continuously try and fetch events from other + servers. Changes in synapse 0.5.2 (2014-11-26) ===================================== From 28b468b292935ca99c93006a831ffd3618f45fbf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:46:15 +0000 Subject: [PATCH 401/630] More rst fixes. Expand on JSON library change --- CHANGES.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f2fef17c9..f384914a4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,8 +6,10 @@ Changes in develop * Persist events we have rejected from federation, fixing the bug where servers would keep requesting the same events. * Various performance improvements, including: - - Caching common queries such as fetching events and current state. - - Upgraded JSON library. + + - Caching common queries such as fetching events and current state. + - Upgraded JSON library to improve parsing and serialization speeds. + * Add default avatars to new user accounts using pydenticon library. * Correctly time out federation requests. * Retry federation requests against different servers. From e76c24d6e61035aa55a17a3a0636f64aed3440d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:46:58 +0000 Subject: [PATCH 402/630] Change develop to v0.7.0 --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f384914a4..e61b2f656 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,5 @@ -Changes in develop -================== +Changes in v0.7.0 +================= * Add initial implementation of the `query auth` fedeartion API, allowing servers to agree on whether an event should be allowed or rejected. From 596728698f62c438a644692778d7efce60eb16cc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:47:46 +0000 Subject: [PATCH 403/630] Use consistent style of heading --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index e61b2f656..810199214 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,5 @@ -Changes in v0.7.0 -================= +Changes in synapse v0.7.0 (2015-02-12) +====================================== * Add initial implementation of the `query auth` fedeartion API, allowing servers to agree on whether an event should be allowed or rejected. From a566ed2f0ea09b4a1874f2bf4b1014e05a308e8b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 10:49:07 +0000 Subject: [PATCH 404/630] Bump webclient version --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index e099a42b5..ec78fc362 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil>=0.0.3": ["syutil"], - "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], + "matrix_angular_sdk>=0.6.2": ["syweb>=0.6.2"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -31,8 +31,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.1", - egg="matrix_angular_sdk-0.6.1", + version="v0.6.2", + egg="matrix_angular_sdk-0.6.2", ), github_link( project="pyca/pynacl", From 959d77fc7a34388c30391643c83c7cb63fe039e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 11:01:53 +0000 Subject: [PATCH 405/630] Add note about push support --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 810199214..048e4b4bf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,7 @@ Changes in synapse v0.7.0 (2015-02-12) * Add default avatars to new user accounts using pydenticon library. * Correctly time out federation requests. * Retry federation requests against different servers. +* Add support for push and push rules. Changes in synapse 0.6.1 (2015-01-07) ===================================== From b656081c99e003777797b5fadaaf203ab5a380fe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 11:04:52 +0000 Subject: [PATCH 406/630] Typoes --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 048e4b4bf..b35ae4f75 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,14 +1,14 @@ Changes in synapse v0.7.0 (2015-02-12) ====================================== -* Add initial implementation of the `query auth` fedeartion API, allowing +* Add initial implementation of the `query auth` federation API, allowing servers to agree on whether an event should be allowed or rejected. * Persist events we have rejected from federation, fixing the bug where servers would keep requesting the same events. * Various performance improvements, including: - Caching common queries such as fetching events and current state. - - Upgraded JSON library to improve parsing and serialization speeds. + - Upgraded JSON library to improve parsing and serialisation speeds. * Add default avatars to new user accounts using pydenticon library. * Correctly time out federation requests. From df29666d3ca710e77adc9d04844b088709463e81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 11:14:57 +0000 Subject: [PATCH 407/630] Add note about alpha csv2 apis --- CHANGES.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b35ae4f75..160d0a128 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,11 @@ Changes in synapse v0.7.0 (2015-02-12) ====================================== -* Add initial implementation of the `query auth` federation API, allowing +* Add initial implementation of the query auth federation API, allowing servers to agree on whether an event should be allowed or rejected. * Persist events we have rejected from federation, fixing the bug where servers would keep requesting the same events. -* Various performance improvements, including: +* Various federation performance improvements, including: - Caching common queries such as fetching events and current state. - Upgraded JSON library to improve parsing and serialisation speeds. @@ -14,6 +14,7 @@ Changes in synapse v0.7.0 (2015-02-12) * Correctly time out federation requests. * Retry federation requests against different servers. * Add support for push and push rules. +* Add alpha versions of proposed new CSv2 APIs, including ``/sync`` API. Changes in synapse 0.6.1 (2015-01-07) ===================================== From 49fb1067ec1375bfe5d5fb50f62630c1a737eb63 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 11:33:29 +0000 Subject: [PATCH 408/630] Expand on caching --- CHANGES.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 160d0a128..883510759 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,7 +7,13 @@ Changes in synapse v0.7.0 (2015-02-12) servers would keep requesting the same events. * Various federation performance improvements, including: - - Caching common queries such as fetching events and current state. + - Add in memory caches on queries such as: + + * Computing the state of a room at a point in time, used for + authorization on federation requests. + * Fetching events from the database. + * User's room membership, used for authorizing presence updates. + - Upgraded JSON library to improve parsing and serialisation speeds. * Add default avatars to new user accounts using pydenticon library. From 717687e1fc61a0fa441ff0dee2d49fda518c06bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 14:39:31 +0000 Subject: [PATCH 409/630] Get an auth query one at a time --- synapse/storage/event_federation.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 0cbcdd1b5..3fbc09022 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -55,17 +55,16 @@ class EventFederationStore(SQLBaseStore): results = set() base_sql = ( - "SELECT auth_id FROM event_auth WHERE %s" + "SELECT auth_id FROM event_auth WHERE event_id = ?" ) front = set(event_ids) while front: - sql = base_sql % ( - " OR ".join(["event_id=?"] * len(front)), - ) - - txn.execute(sql, list(front)) - front = [r[0] for r in txn.fetchall()] + new_front = set() + for f in front: + txn.execute(base_sql, (f,)) + new_front.update([r[0] for r in txn.fetchall()]) + front = new_front results.update(front) return list(results) From 48fbe79f7183f0aa7e4473a573658fe2a2f9bcd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 14:52:02 +0000 Subject: [PATCH 410/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 915af3fe0..10147d88d 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0" +__version__ = "0.7.0a" From 963256638d5b3c3edee14bfbd7c00944b45d04c0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 18:17:11 +0000 Subject: [PATCH 411/630] Correctly handle all the places that can throw exceptions --- scripts/federation_client.py | 7 +++++-- synapse/federation/federation_base.py | 22 +++++++++++++--------- synapse/federation/federation_server.py | 9 ++++++--- synapse/handlers/federation.py | 12 ++++++++++++ 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/scripts/federation_client.py b/scripts/federation_client.py index 3139c6176..ea62dceb3 100644 --- a/scripts/federation_client.py +++ b/scripts/federation_client.py @@ -97,8 +97,11 @@ def lookup(destination, path): if ":" in destination: return "https://%s%s" % (destination, path) else: - srv = srvlookup.lookup("matrix", "tcp", destination)[0] - return "https://%s:%d%s" % (srv.host, srv.port, path) + try: + srv = srvlookup.lookup("matrix", "tcp", destination)[0] + return "https://%s:%d%s" % (srv.host, srv.port, path) + except: + return "https://%s:%d%s" % (destination, 8448, path) def get_json(origin_name, origin_key, destination, path): request_json = { diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index a990aec4f..30f3f5c8a 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -61,7 +61,8 @@ class FederationBase(object): # Check local db. new_pdu = yield self.store.get_event( pdu.event_id, - allow_rejected=True + allow_rejected=True, + allow_none=True, ) if new_pdu: signed_pdus.append(new_pdu) @@ -69,15 +70,18 @@ class FederationBase(object): # Check pdu.origin if pdu.origin != origin: - new_pdu = yield self.get_pdu( - destinations=[pdu.origin], - event_id=pdu.event_id, - outlier=outlier, - ) + try: + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) - if new_pdu: - signed_pdus.append(new_pdu) - continue + if new_pdu: + signed_pdus.append(new_pdu) + continue + except: + pass logger.warn("Failed to find copy of %s with valid signature") diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index b23f72c7f..9f5c98694 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -411,9 +411,12 @@ class FederationServer(FederationBase): "_handle_new_pdu getting state for %s", pdu.room_id ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) + try: + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) + except: + logger.warn("Failed to get state for event: %s", pdu.event_id) ret = yield self.handler.on_receive_pdu( origin, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 77c81fe2d..5a7593cc3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -788,6 +788,18 @@ class FederationHandler(BaseHandler): defer.returnValue(ret) + @defer.inlineCallbacks + def trigger_query_auth(self, destination, event_id): + local_auth_chain = yield self.store.get_auth_chain([event_id]) + + result = yield self.replication_layer.query_auth( + destination, + event.room_id, + event.event_id, + local_auth_chain, + ) + + @defer.inlineCallbacks @log_function def do_auth(self, origin, event, context, auth_events): From 58d848adc0ee9ab8e44a780a19be1e46e0d5cec3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 18:35:36 +0000 Subject: [PATCH 412/630] Parrellize fetching of events --- synapse/federation/federation_base.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 30f3f5c8a..966f4d2c5 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -50,8 +50,11 @@ class FederationBase(object): Returns: Deferred : A list of PDUs that have valid signatures and hashes. """ + signed_pdus = [] - for pdu in pdus: + + @defer.inlineCallbacks + def do(pdu): try: new_pdu = yield self._check_sigs_and_hash(pdu) signed_pdus.append(new_pdu) @@ -66,7 +69,7 @@ class FederationBase(object): ) if new_pdu: signed_pdus.append(new_pdu) - continue + return # Check pdu.origin if pdu.origin != origin: @@ -79,12 +82,17 @@ class FederationBase(object): if new_pdu: signed_pdus.append(new_pdu) - continue + return except: pass logger.warn("Failed to find copy of %s with valid signature") + yield defer.gatherResults( + [do(pdu) for pdu in pdus], + consumeErrors=True + ) + defer.returnValue(signed_pdus) @defer.inlineCallbacks From 860b1b4841618ec401c3652dcf19ac8b5d167f50 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:11:52 +0000 Subject: [PATCH 413/630] Only attempt to fetch presence state of JOINed members in room initialSync (SYN-202) --- synapse/handlers/message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3355adefc..c6d6188d9 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,6 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler From 29805213d1c20c89c20a4b3075c30597b4402338 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:12:57 +0000 Subject: [PATCH 414/630] Can now remove the FIXME too --- synapse/handlers/message.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6d6188d9..5f4cc1a84 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -386,16 +386,9 @@ class MessageHandler(BaseHandler): ) presence.append(member_presence) except SynapseError as e: - if e.code == 404: - # FIXME: We are doing this as a warn since this gets hit a - # lot and spams the logs. Why is this happening? - logger.warn( - "Failed to get member presence of %r", m.user_id - ) - else: - logger.exception( - "Failed to get member presence of %r", m.user_id - ) + logger.exception( + "Failed to get member presence of %r", m.user_id + ) time_now = self.clock.time_msec() From 5f6e6530d0f43c637e527cfe1797801879dbc389 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:15:23 +0000 Subject: [PATCH 415/630] Appease pyflakes --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5f4cc1a84..e028cfc2e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -385,7 +385,7 @@ class MessageHandler(BaseHandler): as_event=True, ) presence.append(member_presence) - except SynapseError as e: + except SynapseError: logger.exception( "Failed to get member presence of %r", m.user_id ) From 93d07c87dc9b837492fa0f691a77e716a2fe512b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:19:37 +0000 Subject: [PATCH 416/630] Reindent code to be less human-readable to keep pep8 from complaining --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index e028cfc2e..7b9685be7 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,7 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member - and m.content["membership"] == Membership.JOIN + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler From 38df10b99eaa7f8165c9e541142e96bf023ad744 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 19:29:32 +0000 Subject: [PATCH 417/630] Remove unused function --- synapse/handlers/federation.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5a7593cc3..77c81fe2d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -788,18 +788,6 @@ class FederationHandler(BaseHandler): defer.returnValue(ret) - @defer.inlineCallbacks - def trigger_query_auth(self, destination, event_id): - local_auth_chain = yield self.store.get_auth_chain([event_id]) - - result = yield self.replication_layer.query_auth( - destination, - event.room_id, - event.event_id, - local_auth_chain, - ) - - @defer.inlineCallbacks @log_function def do_auth(self, origin, event, context, auth_events): From 789251afa7d97a08318f92ffe96f5f7ce16d6157 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 19:29:43 +0000 Subject: [PATCH 418/630] Fix logging --- synapse/federation/federation_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 966f4d2c5..21a763214 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -86,7 +86,10 @@ class FederationBase(object): except: pass - logger.warn("Failed to find copy of %s with valid signature") + logger.warn( + "Failed to find copy of %s with valid signature", + pdu.event_id, + ) yield defer.gatherResults( [do(pdu) for pdu in pdus], From f22646efcc3d3c5c7fc0d7797de2675e1a794dd0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:11:52 +0000 Subject: [PATCH 419/630] Only attempt to fetch presence state of JOINed members in room initialSync (SYN-202) --- synapse/handlers/message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3355adefc..c6d6188d9 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,6 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler From 55e0916ffc0a4e2c7bff533c561e99889650c61d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Feb 2015 19:19:37 +0000 Subject: [PATCH 420/630] Reindent code to be less human-readable to keep pep8 from complaining --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6d6188d9..b00380387 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -372,7 +372,7 @@ class MessageHandler(BaseHandler): room_members = [ m for m in current_state.values() if m.type == EventTypes.Member - and m.content["membership"] == Membership.JOIN + and m.content["membership"] == Membership.JOIN ] presence_handler = self.hs.get_handlers().presence_handler From c09493d7aa79386645ae48f1232158f99dd6049a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 13:33:37 +0000 Subject: [PATCH 421/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 10147d88d..c0aa2a757 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0a" +__version__ = "0.7.0b" From 8c652a2b5f4ed07abd682543ad6a365c6d50b8d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 14:20:05 +0000 Subject: [PATCH 422/630] When we see a difference in current state, actually use state conflict resolution algorithm --- synapse/handlers/federation.py | 34 +++++++++++++++++++++++++++ synapse/state.py | 43 +++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 77c81fe2d..5b225ffb5 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -858,6 +858,40 @@ class FederationHandler(BaseHandler): # Do auth conflict res. logger.debug("Different auth: %s", different_auth) + different_events = yield defer.gatherResults( + [ + self.store.get_event( + d, + allow_none=True, + allow_rejected=False, + ) + for d in different_auth + if d in have_events and not have_events[d] + ], + consumeErrors=True + ) + + if different_events: + local_view = dict(auth_events) + remote_view = dict(auth_events) + remote_view.update({ + (d.type, d.state_key) for d in different_events + }) + + new_state, _ = self.state.resolve_events( + [local_view, remote_view], + event + ) + + auth_events.update(new_state) + + current_state = set(e.event_id for e in auth_events.values()) + different_auth = event_auth_events - current_state + + context.current_state.update(auth_events) + context.state_group = None + + if different_auth and not event.internal_metadata.is_outlier(): # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False diff --git a/synapse/state.py b/synapse/state.py index 98aaa2be5..fe5f3dc84 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -259,13 +259,37 @@ class StateHandler(object): defer.returnValue((name, state, prev_states)) + new_state, prev_states = self._resolve_events( + state_groups.values(), event_type, state_key + ) + + if self._state_cache is not None: + cache = _StateCacheEntry( + state=new_state, + state_group=None, + ts=self.clock.time_msec() + ) + + self._state_cache[frozenset(event_ids)] = cache + + defer.returnValue((None, new_state, prev_states)) + + def resolve_events(self, state_sets, event): + if event.is_state(): + return self._resolve_events( + state_sets, event.type, event.state_key + ) + else: + return self._resolve_events(state_sets) + + def _resolve_events(self, state_sets, event_type=None, state_key=""): state = {} - for group, g_state in state_groups.items(): - for s in g_state: + for st in state_sets: + for e in st: state.setdefault( - (s.type, s.state_key), + (e.type, e.state_key), {} - )[s.event_id] = s + )[e.event_id] = e unconflicted_state = { k: v.values()[0] for k, v in state.items() @@ -302,16 +326,7 @@ class StateHandler(object): new_state = unconflicted_state new_state.update(resolved_state) - if self._state_cache is not None: - cache = _StateCacheEntry( - state=new_state, - state_group=None, - ts=self.clock.time_msec() - ) - - self._state_cache[frozenset(event_ids)] = cache - - defer.returnValue((None, new_state, prev_states)) + return new_state, prev_states @log_function def _resolve_state_events(self, conflicted_state, auth_events): From e441c10a7302a3b0ddfa3caa7823fa6f62372f82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 14:23:39 +0000 Subject: [PATCH 423/630] pyflakes --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5b225ffb5..d667d358a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -878,7 +878,7 @@ class FederationHandler(BaseHandler): (d.type, d.state_key) for d in different_events }) - new_state, _ = self.state.resolve_events( + new_state, prev_state = self.state.resolve_events( [local_view, remote_view], event ) From 0546126cc5fb1a138c96481061084e9e5fc86e34 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 14:36:40 +0000 Subject: [PATCH 424/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index c0aa2a757..1060fcc86 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0b" +__version__ = "0.7.0c" From 183b3d4e47dd2fc0e0ca88714d0e0bd415f81736 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 Feb 2015 14:29:49 +0000 Subject: [PATCH 425/630] Prepare the database whenever a connection is opened from the db_pool so that in-memory databases will work --- synapse/app/homeserver.py | 14 ++++---------- synapse/storage/__init__.py | 3 +++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f5681fac2..6f39819d3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -90,7 +90,9 @@ class SynapseHomeServer(HomeServer): "sqlite3", self.get_db_name(), check_same_thread=False, cp_min=1, - cp_max=1 + cp_max=1, + cp_openfun=prepare_database, # Prepare the database for each conn + # so that :memory: sqlite works ) def create_resource_tree(self, web_client, redirect_root_to_web_client): @@ -252,14 +254,6 @@ def setup(): logger.info("Database prepared in %s.", db_name) - db_pool = hs.get_db_pool() - - if db_name == ":memory:": - # Memory databases will need to be setup each time they are opened. - reactor.callWhenRunning( - db_pool.runWithConnection, prepare_database - ) - if config.manhole: f = twisted.manhole.telnet.ShellFactory() f.username = "matrix" @@ -270,10 +264,10 @@ def setup(): bind_port = config.bind_port if config.no_tls: bind_port = None + hs.start_listening(bind_port, config.unsecure_port) hs.get_pusherpool().start() - hs.get_state_handler().start_caching() hs.get_datastore().start_profiling() diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 02b1f0685..1c22e19ab 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -637,10 +637,13 @@ def prepare_database(db_conn): c.executescript(sql_script) db_conn.commit() + else: + logger.info("Database is at version %r", user_version) else: sql_script = "BEGIN TRANSACTION;\n" for sql_loc in SCHEMAS: + logger.debug("Applying schema %r", sql_loc) sql_script += read_schema(sql_loc) sql_script += "\n" sql_script += "COMMIT TRANSACTION;" From b11a6e1c3ca51e81b911fa733a4169bd6cbaa5ee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 15:37:18 +0000 Subject: [PATCH 426/630] Fix wrong variable name --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d667d358a..4cd795e20 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -878,7 +878,7 @@ class FederationHandler(BaseHandler): (d.type, d.state_key) for d in different_events }) - new_state, prev_state = self.state.resolve_events( + new_state, prev_state = self.state_handler.resolve_events( [local_view, remote_view], event ) From be26697b29a44b96cb0ed371ca4aaab2654d6802 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 15:37:35 +0000 Subject: [PATCH 427/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 1060fcc86..7ba21cd24 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0c" +__version__ = "0.7.0d" From 8d5cce62ab4bbb2333f01abb9edf7e94f6d6f7d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 13 Feb 2015 16:16:16 +0000 Subject: [PATCH 428/630] Update pushers by app id and pushkey, not user id and pushkey --- synapse/push/__init__.py | 12 ++++++------ synapse/storage/pusher.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 0659a1cb9..0fb3e4f7f 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -237,7 +237,7 @@ class Pusher(object): self.user_name, config, timeout=0) self.last_token = chunk['end'] self.store.update_pusher_last_token( - self.user_name, self.pushkey, self.last_token) + self.app_id, self.pushkey, self.last_token) logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) @@ -308,7 +308,7 @@ class Pusher(object): self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token_and_success( - self.user_name, + self.app_id, self.pushkey, self.last_token, self.clock.time_msec() @@ -316,14 +316,14 @@ class Pusher(object): if self.failing_since: self.failing_since = None self.store.update_pusher_failing_since( - self.user_name, + self.app_id, self.pushkey, self.failing_since) else: if not self.failing_since: self.failing_since = self.clock.time_msec() self.store.update_pusher_failing_since( - self.user_name, + self.app_id, self.pushkey, self.failing_since ) @@ -340,14 +340,14 @@ class Pusher(object): self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] self.store.update_pusher_last_token( - self.user_name, + self.app_id, self.pushkey, self.last_token ) self.failing_since = None self.store.update_pusher_failing_since( - self.user_name, + self.app_id, self.pushkey, self.failing_since ) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index e2a662a6c..6622b4d18 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -126,27 +126,27 @@ class PusherStore(SQLBaseStore): ) @defer.inlineCallbacks - def update_pusher_last_token(self, user_name, pushkey, last_token): + def update_pusher_last_token(self, app_id, pushkey, last_token): yield self._simple_update_one( PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey}, {'last_token': last_token} ) @defer.inlineCallbacks - def update_pusher_last_token_and_success(self, user_name, pushkey, + def update_pusher_last_token_and_success(self, app_id, pushkey, last_token, last_success): yield self._simple_update_one( PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey}, {'last_token': last_token, 'last_success': last_success} ) @defer.inlineCallbacks - def update_pusher_failing_since(self, user_name, pushkey, failing_since): + def update_pusher_failing_since(self, app_id, pushkey, failing_since): yield self._simple_update_one( PushersTable.table_name, - {'user_name': user_name, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey}, {'failing_since': failing_since} ) From a78838c5ba5c9342bb3dc53e0c814b00d22be658 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Feb 2015 16:17:54 +0000 Subject: [PATCH 429/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 7ba21cd24..1a02aefe2 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0d" +__version__ = "0.7.0e" From 2b8903ce2fe56a5cc72d48de9282ee4e176f4b83 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 14 Feb 2015 00:16:33 +0000 Subject: [PATCH 430/630] we federate on port 8448 nowadays... --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5eebe5b72..d3778b9f2 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ VoIP. The basics you need to know to get up and running are: - Everything in Matrix happens in a room. Rooms are distributed and do not exist on any single server. Rooms can be located using convenience aliases - like ``#matrix:matrix.org`` or ``#test:localhost:8008``. + like ``#matrix:matrix.org`` or ``#test:localhost:8448``. - Matrix user IDs look like ``@matthew:matrix.org`` (although in the future you will normally refer to yourself and others using a 3PID: email From b99a33f283d595f5f462ab039d672ca04bdffcb8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sun, 15 Feb 2015 20:20:51 +0000 Subject: [PATCH 431/630] resolve_events expect lists, not dicts --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4cd795e20..3aecc1ca5 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -879,7 +879,7 @@ class FederationHandler(BaseHandler): }) new_state, prev_state = self.state_handler.resolve_events( - [local_view, remote_view], + [local_view.values(), remote_view.values()], event ) From 6dd5c958412ac9c589996c8c5d92eeacdbeb734b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sun, 15 Feb 2015 20:38:52 +0000 Subject: [PATCH 432/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 1a02aefe2..792b8089f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0e" +__version__ = "0.7.0f" From a5ad6f862c75a2d4c9ed5d57537c529053002daa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 13:15:41 +0000 Subject: [PATCH 433/630] Fix contrib/graph/graph2.py to handle FrozenDict --- contrib/graph/graph2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index 6b551d42e..d0d2cfe7c 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -21,6 +21,7 @@ import datetime import argparse from synapse.events import FrozenEvent +from synapse.util.frozenutils import unfreeze def make_graph(db_name, room_id, file_prefix, limit): @@ -70,7 +71,7 @@ def make_graph(db_name, room_id, file_prefix, limit): float(event.origin_server_ts) / 1000 ).strftime('%Y-%m-%d %H:%M:%S,%f') - content = json.dumps(event.get_dict()["content"]) + content = json.dumps(unfreeze(event.get_dict()["content"])) label = ( "<" From 61385846510ceed1bff571dde1233a70e02b4748 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 14:08:02 +0000 Subject: [PATCH 434/630] Don't return anything from _handle_new_pdu, since we ignore the return value anyway --- synapse/federation/federation_server.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 9f5c98694..7ca8d49c5 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -331,7 +331,6 @@ class FederationServer(FederationBase): ) if already_seen: logger.debug("Already seen pdu %s", pdu.event_id) - defer.returnValue({}) return # Check signature. @@ -418,7 +417,7 @@ class FederationServer(FederationBase): except: logger.warn("Failed to get state for event: %s", pdu.event_id) - ret = yield self.handler.on_receive_pdu( + yield self.handler.on_receive_pdu( origin, pdu, backfilled=False, @@ -426,8 +425,6 @@ class FederationServer(FederationBase): auth_chain=auth_chain, ) - defer.returnValue(ret) - def __str__(self): return "" % self.server_name From 91fc5eef1deb2f756d16f3d5ea20ede8d967f6df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 14:27:40 +0000 Subject: [PATCH 435/630] Mark old events as outliers. This is to fix the issue where if a remote server sends an event that references a really "old" event, then the local server will pull that in and send to all clients. We decide if an event is old if its depth is less than the minimum depth of the room. --- synapse/federation/federation_server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 7ca8d49c5..4391a60c0 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -366,7 +366,13 @@ class FederationServer(FederationBase): pdu.room_id, min_depth ) - if min_depth and pdu.depth > min_depth and max_recursion > 0: + if min_depth and pdu.depth < min_depth: + # This is so that we don't notify the user about this + # message, to work around the fact that some events will + # reference really really old events we really don't want to + # send to the clients. + pdu.internal_metadata.outlier = True + elif min_depth and pdu.depth > min_depth and max_recursion > 0: for event_id, hashes in pdu.prev_events: if event_id not in have_seen: logger.debug( From 2674aeb96a1b75583dc3ea514cbce580e8ae35c8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 16:16:47 +0000 Subject: [PATCH 436/630] Factor out ExpiringCache from StateHandler --- synapse/state.py | 46 +++----------- synapse/util/expiringcache.py | 115 ++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 37 deletions(-) create mode 100644 synapse/util/expiringcache.py diff --git a/synapse/state.py b/synapse/state.py index fe5f3dc84..80cced351 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.util.expiringcache import ExpiringCache from synapse.api.constants import EventTypes from synapse.api.errors import AuthError from synapse.events.snapshot import EventContext @@ -51,7 +52,6 @@ class _StateCacheEntry(object): def __init__(self, state, state_group, ts): self.state = state self.state_group = state_group - self.ts = ts class StateHandler(object): @@ -69,12 +69,15 @@ class StateHandler(object): def start_caching(self): logger.debug("start_caching") - self._state_cache = {} + self._state_cache = ExpiringCache( + cache_name="state_cache", + clock=self.clock, + max_len=SIZE_OF_CACHE, + expiry_ms=EVICTION_TIMEOUT_SECONDS*1000, + reset_expiry_on_get=True, + ) - def f(): - self._prune_cache() - - self.clock.looping_call(f, 5*1000) + self._state_cache.start() @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): @@ -409,34 +412,3 @@ class StateHandler(object): return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() return sorted(events, key=key_func) - - def _prune_cache(self): - logger.debug( - "_prune_cache. before len: %d", - len(self._state_cache.keys()) - ) - - now = self.clock.time_msec() - - if len(self._state_cache.keys()) > SIZE_OF_CACHE: - sorted_entries = sorted( - self._state_cache.items(), - key=lambda k, v: v.ts, - ) - - for k, _ in sorted_entries[SIZE_OF_CACHE:]: - self._state_cache.pop(k) - - keys_to_delete = set() - - for key, cache_entry in self._state_cache.items(): - if now - cache_entry.ts > EVICTION_TIMEOUT_SECONDS*1000: - keys_to_delete.add(key) - - for k in keys_to_delete: - self._state_cache.pop(k) - - logger.debug( - "_prune_cache. after len: %d", - len(self._state_cache.keys()) - ) diff --git a/synapse/util/expiringcache.py b/synapse/util/expiringcache.py new file mode 100644 index 000000000..fb5b27ad1 --- /dev/null +++ b/synapse/util/expiringcache.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + + +logger = logging.getLogger(__name__) + + +class ExpiringCache(object): + def __init__(self, cache_name, clock, max_len=0, expiry_ms=0, + reset_expiry_on_get=False): + """ + Args: + cache_name (str): Name of this cache, used for logging. + clock (Clock) + max_len (int): Max size of dict. If the dict grows larger than this + then the oldest items get automatically evicted. Default is 0, + which indicates there is no max limit. + expiry_ms (int): How long before an item is evicted from the cache + in milliseconds. Default is 0, indicating items never get + evicted based on time. + reset_expiry_on_get (bool): If true, will reset the expiry time for + an item on access. Defaults to False. + + """ + self._cache_name = cache_name + + self._clock = clock + + self._max_len = max_len + self._expiry_ms = expiry_ms + + self._reset_expiry_on_get = reset_expiry_on_get + + self._cache = {} + + def start(self): + if not self._expiry_ms: + # Don't bother starting the loop if things never expire + return + + def f(): + self._prune_cache() + + self._clock.looping_call(f, self._expiry_ms) + + def __setitem__(self, key, value): + now = self._clock.time_msec() + self._cache[key] = _CacheEntry(now, value) + + # Evict if there are now too many items + if self._max_len and len(self._cache.keys()) > self._max_len: + sorted_entries = sorted( + self._cache.items(), + key=lambda k, v: v.time, + ) + + for k, _ in sorted_entries[self._max_len:]: + self._cache.pop(k) + + def __getitem__(self, key): + entry = self._cache[key] + + if self._reset_expiry_on_get: + entry.time = self._clock.time_msec() + + return entry.value + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def _purge_cache(self): + if not self._expiry_ms: + # zero expiry time means don't expire. This should never get called + # since we have this check in start too. + return + begin_length = len(self._cache) + + now = self._clock.time_msec() + + keys_to_delete = set() + + for key, cache_entry in self._cache.items(): + if now - cache_entry.time > self._expiry_ms: + keys_to_delete.add(key) + + for k in keys_to_delete: + self._cache.pop(k) + + logger.debug( + "[%s] _prune_cache before: %d, after len: %d", + self._cache_name, begin_length, len(self._cache.keys()) + ) + + +class _CacheEntry(object): + def __init__(self, time, value): + self.time = time + self.value = value From baa5b9a97582d4b3c825be1225aba7863230c047 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 Feb 2015 18:02:39 +0000 Subject: [PATCH 437/630] Cache results of get_pdu. --- synapse/federation/federation_client.py | 42 +++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 70c9a6f46..83b4947b9 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -19,7 +19,8 @@ from twisted.internet import defer from .federation_base import FederationBase from .units import Edu -from synapse.api.errors import CodeMessageException +from synapse.api.errors import CodeMessageException, SynapseError +from synapse.util.expiringcache import ExpiringCache from synapse.util.logutils import log_function from synapse.events import FrozenEvent @@ -30,6 +31,20 @@ logger = logging.getLogger(__name__) class FederationClient(FederationBase): + def __init__(self): + self._fail_fetch_pdu_cache = None + + def start_pdu_fail_cache(self): + self._fail_fetch_pdu_cache = ExpiringCache( + cache_name="get_pdu_cache", + clock=self._clock, + max_len=1000, + expiry_ms=120*1000, + reset_expiry_on_get=False, + ) + + self._fail_fetch_pdu_cache.start() + @log_function def send_pdu(self, pdu, destinations): """Informs the replication layer about a new PDU generated within the @@ -160,6 +175,11 @@ class FederationClient(FederationBase): # TODO: Rate limit the number of times we try and get the same event. + if self._fail_fetch_pdu_cache: + e = self._fail_fetch_pdu_cache.get(event_id) + if e: + defer.returnValue(e) + pdu = None for destination in destinations: try: @@ -181,8 +201,21 @@ class FederationClient(FederationBase): pdu = yield self._check_sigs_and_hash(pdu) break - except CodeMessageException: - raise + except SynapseError: + logger.info( + "Failed to get PDU %s from %s because %s", + event_id, destination, e, + ) + continue + except CodeMessageException as e: + if 400 <= e.code < 500: + raise + + logger.info( + "Failed to get PDU %s from %s because %s", + event_id, destination, e, + ) + continue except Exception as e: logger.info( "Failed to get PDU %s from %s because %s", @@ -190,6 +223,9 @@ class FederationClient(FederationBase): ) continue + if self._fail_fetch_pdu_cache is not None: + self._fail_fetch_pdu_cache[event_id] = pdu + defer.returnValue(pdu) @defer.inlineCallbacks From 72a4de2ce627528a13bda480403344ffde6275d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 10:03:23 +0000 Subject: [PATCH 438/630] Use consumeErrors=True on all DeferredLists. This is so that the DeferredLists actually consume the error instead of propogating down the non-existent errback chain. This should reduce the number of unhandled errors we are seeing. --- synapse/federation/federation_server.py | 2 +- synapse/federation/transaction_queue.py | 2 +- synapse/handlers/presence.py | 8 ++++---- synapse/notifier.py | 6 ++++-- synapse/storage/roommember.py | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 9f5c98694..e94d0411b 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -124,7 +124,7 @@ class FederationServer(FederationBase): edu.content ) - results = yield defer.DeferredList(dl) + results = yield defer.DeferredList(dl, consumeErrors=True) ret = [] for r in results: diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 731019ad9..bb20f2eba 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -98,7 +98,7 @@ class TransactionQueue(object): deferreds.append(deferred) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) # NO inlineCallbacks def enqueue_edu(self, edu): diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 59287010e..8ef248ecf 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -492,7 +492,7 @@ class PresenceHandler(BaseHandler): user, domain, remoteusers )) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) def _start_polling_local(self, user, target_user): target_localpart = target_user.localpart @@ -548,7 +548,7 @@ class PresenceHandler(BaseHandler): self._stop_polling_remote(user, domain, remoteusers) ) - return defer.DeferredList(deferreds) + return defer.DeferredList(deferreds, consumeErrors=True) def _stop_polling_local(self, user, target_user): for localpart in self._local_pushmap.keys(): @@ -729,7 +729,7 @@ class PresenceHandler(BaseHandler): del self._remote_sendmap[user] with PreserveLoggingContext(): - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) @defer.inlineCallbacks def push_update_to_local_and_remote(self, observed_user, statuscache, @@ -768,7 +768,7 @@ class PresenceHandler(BaseHandler): ) ) - yield defer.DeferredList(deferreds) + yield defer.DeferredList(deferreds, consumeErrors=True) defer.returnValue((localusers, remote_domains)) diff --git a/synapse/notifier.py b/synapse/notifier.py index e3b6ead62..f5a394596 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -135,7 +135,8 @@ class Notifier(object): with PreserveLoggingContext(): yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners] + [notify(l).addErrback(eb) for l in listeners], + consumeErrors=True, ) @defer.inlineCallbacks @@ -203,7 +204,8 @@ class Notifier(object): with PreserveLoggingContext(): yield defer.DeferredList( - [notify(l).addErrback(eb) for l in listeners] + [notify(l).addErrback(eb) for l in listeners], + consumeErrors=True, ) @defer.inlineCallbacks diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 779f9ce54..9bf608bc9 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -288,7 +288,7 @@ class RoomMemberStore(SQLBaseStore): deferreds = [self.get_rooms_for_user(u) for u in user_id_list] - results = yield defer.DeferredList(deferreds) + results = yield defer.DeferredList(deferreds, consumeErrors=True) # A list of sets of strings giving room IDs for each user room_id_lists = [set([r.room_id for r in result[1]]) for result in results] From ea1d6c16cd2b91ec2980eef31446990cf305b796 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 10:54:06 +0000 Subject: [PATCH 439/630] Don't write bytecode --- synapse/app/homeserver.py | 4 +++- synapse/app/synctl.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f5681fac2..18585e87f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys +sys.dont_write_bytecode = True + from synapse.storage import prepare_database, UpgradeDatabaseException from synapse.server import HomeServer @@ -48,7 +51,6 @@ import synapse import logging import os import re -import sys import sqlite3 import syweb diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 363c20f99..3a70a248d 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -19,7 +19,7 @@ import os import subprocess import signal -SYNAPSE = ["python", "-m", "synapse.app.homeserver"] +SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] CONFIGFILE = "homeserver.yaml" PIDFILE = "homeserver.pid" From c2b1dbd84c27be8759b14502b124504cba4c9295 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 11:11:11 +0000 Subject: [PATCH 440/630] We do want to consumeError --- synapse/handlers/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index c69787005..c2762f92c 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -181,7 +181,7 @@ class TypingNotificationHandler(BaseHandler): }, )) - yield defer.DeferredList(deferreds, consumeErrors=False) + yield defer.DeferredList(deferreds, consumeErrors=True) @defer.inlineCallbacks def _recv_edu(self, origin, content): From 02bfa889de1990eb8dc47770834bef777252dc82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 13:08:27 +0000 Subject: [PATCH 441/630] Handle recieving failures in transactions --- synapse/federation/federation_server.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index e94d0411b..34f5b1744 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -114,7 +114,15 @@ class FederationServer(FederationBase): with PreserveLoggingContext(): dl = [] for pdu in pdu_list: - dl.append(self._handle_new_pdu(transaction.origin, pdu)) + d = self._handle_new_pdu(transaction.origin, pdu) + + def handle_failure(failure): + failure.trap(FederationError) + self.enqueue_failure(failure.value, transaction.origin) + + d.addErrback(handle_failure) + + dl.append(d) if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: @@ -124,6 +132,9 @@ class FederationServer(FederationBase): edu.content ) + for failure in getattr(transaction, "failures", []): + logger.info("Got failure %r", failure) + results = yield defer.DeferredList(dl, consumeErrors=True) ret = [] From 47281f8fa4b3f529f84409367736483da66416ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 11:51:22 +0000 Subject: [PATCH 442/630] Change some debug logging to info --- synapse/handlers/federation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3aecc1ca5..0eb2ff95c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -802,7 +802,7 @@ class FederationHandler(BaseHandler): missing_auth = event_auth_events - seen_events if missing_auth: - logger.debug("Missing auth: %s", missing_auth) + logger.info("Missing auth: %s", missing_auth) # If we don't have all the auth events, we need to get them. try: remote_auth_chain = yield self.replication_layer.get_event_auth( @@ -856,7 +856,7 @@ class FederationHandler(BaseHandler): if different_auth and not event.internal_metadata.is_outlier(): # Do auth conflict res. - logger.debug("Different auth: %s", different_auth) + logger.info("Different auth: %s", different_auth) different_events = yield defer.gatherResults( [ @@ -892,6 +892,8 @@ class FederationHandler(BaseHandler): context.state_group = None if different_auth and not event.internal_metadata.is_outlier(): + logger.info("Different auth after resolution: %s", different_auth) + # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False From c82e26ad4ba80742adc68a7e8c52441d760e4746 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 13:24:13 +0000 Subject: [PATCH 443/630] Actually respond with JSON to incoming transaction --- synapse/federation/federation_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 34f5b1744..c48a41b5d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -147,6 +147,8 @@ class FederationServer(FederationBase): logger.debug("Returning: %s", str(ret)) + response = ret + yield self.transaction_actions.set_response( transaction, 200, response From 659ead082ff11842fea803e44d75667e0ca38d71 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 13:58:52 +0000 Subject: [PATCH 444/630] Format the response of transaction request in a nicer way --- synapse/federation/federation_server.py | 19 +++++++++++++++---- synapse/federation/transaction_queue.py | 22 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index c48a41b5d..d1ec0b9ea 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -118,7 +118,7 @@ class FederationServer(FederationBase): def handle_failure(failure): failure.trap(FederationError) - self.enqueue_failure(failure.value, transaction.origin) + self.send_failure(failure.value, transaction.origin) d.addErrback(handle_failure) @@ -132,7 +132,7 @@ class FederationServer(FederationBase): edu.content ) - for failure in getattr(transaction, "failures", []): + for failure in getattr(transaction, "pdu_failures", []): logger.info("Got failure %r", failure) results = yield defer.DeferredList(dl, consumeErrors=True) @@ -143,11 +143,15 @@ class FederationServer(FederationBase): ret.append({}) else: logger.exception(r[1]) - ret.append({"error": str(r[1])}) + ret.append({"error": str(r[1].value)}) logger.debug("Returning: %s", str(ret)) - response = ret + response = { + "pdus": dict(zip( + (p.event_id for p in pdu_list), ret + )), + } yield self.transaction_actions.set_response( transaction, @@ -358,6 +362,13 @@ class FederationServer(FederationBase): affected=pdu.event_id, ) + raise FederationError( + "ERROR", + 403, + "Forbidden", + affected=pdu.event_id, + ) + state = None auth_chain = [] diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index bb20f2eba..6faaa066f 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -91,7 +91,7 @@ class TransactionQueue(object): if not deferred.called: deferred.errback(failure) else: - logger.warn("Failed to send pdu", failure) + logger.warn("Failed to send pdu", failure.value) with PreserveLoggingContext(): self._attempt_new_transaction(destination).addErrback(eb) @@ -116,7 +116,7 @@ class TransactionQueue(object): if not deferred.called: deferred.errback(failure) else: - logger.warn("Failed to send edu", failure) + logger.warn("Failed to send edu", failure.value) with PreserveLoggingContext(): self._attempt_new_transaction(destination).addErrback(eb) @@ -133,6 +133,15 @@ class TransactionQueue(object): (failure, deferred) ) + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send failure", failure.value) + + with PreserveLoggingContext(): + self._attempt_new_transaction(destination).addErrback(eb) + yield deferred @defer.inlineCallbacks @@ -249,6 +258,15 @@ class TransactionQueue(object): transaction, json_data_cb ) code = 200 + + if response: + for e_id, r in getattr(response, "pdus", {}).items(): + if "error" in r: + logger.warn( + "Transaction returned error for %s: %s", + e_id, r, + ) + except HttpResponseException as e: code = e.code response = e.response From 4de93001bf6a7d8e770b990ea3546237b2569609 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:12:06 +0000 Subject: [PATCH 445/630] Make matrixfederationclient log more nicely --- synapse/http/matrixfederationclient.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 192794800..764b151d9 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -16,6 +16,7 @@ from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError +from twisted.python.failure import Failure from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone @@ -146,14 +147,22 @@ class MatrixFederationHttpClient(object): ) raise SynapseError(400, "Domain specified not found.") + if hasattr(e, "reasons"): + reasons = ", ".join( + f.value.message + for f in e.reasons + ) + else: + reasons = e.message + logger.warn( - "Sending request failed to %s: %s %s : %s", + "Sending request failed to %s: %s %s: %s - %s", destination, method, url_bytes, - e + type(e). __name__, + reasons, ) - _print_ex(e) if retries_left: yield sleep(2 ** (5 - retries_left)) @@ -447,14 +456,6 @@ def _readBodyToFile(response, stream, max_size): return d -def _print_ex(e): - if hasattr(e, "reasons") and e.reasons: - for ex in e.reasons: - _print_ex(ex) - else: - logger.warn(e) - - class _JsonProducer(object): """ Used by the twisted http client to create the HTTP body from json """ From 472734a8cc19d4e4b9f5e311b59f58a37142f8a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:13:50 +0000 Subject: [PATCH 446/630] Consume errors in time_bound_deferred --- synapse/util/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index e77eba90a..79109d0b1 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -99,8 +99,6 @@ class Clock(object): except: pass - return res - given_deferred.addCallbacks(callback=sucess, errback=err) timer = self.call_later(time_out, timed_out_fn) From 0647e27a414e5a86cab57bba65931376e855c289 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:19:54 +0000 Subject: [PATCH 447/630] Remove unused import --- synapse/http/matrixfederationclient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 764b151d9..454c3d4ab 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -16,7 +16,6 @@ from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError -from twisted.python.failure import Failure from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone From 676e8ee78ae3f51cbc0113c8d810d4bc1e81cdab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:22:45 +0000 Subject: [PATCH 448/630] Remove debug raise --- synapse/federation/federation_server.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 679fb141e..22b966383 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -361,13 +361,6 @@ class FederationServer(FederationBase): affected=pdu.event_id, ) - raise FederationError( - "ERROR", - 403, - "Forbidden", - affected=pdu.event_id, - ) - state = None auth_chain = [] From 8b919c00f30c03ddae257f0129f58f2d0285723c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:44:01 +0000 Subject: [PATCH 449/630] Start the get_pdu cache --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 27b478a1c..7565d9444 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -275,6 +275,7 @@ def setup(): hs.get_pusherpool().start() hs.get_state_handler().start_caching() hs.get_datastore().start_profiling() + hs.get_replication_layer().start_pdu_fail_cache() if config.daemonize: print config.pid_file From e7e20417ca468c1afe2e27162b9790f860d2da51 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:44:26 +0000 Subject: [PATCH 450/630] ExpiringCache: purge every 1/2 interval --- synapse/util/expiringcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/expiringcache.py b/synapse/util/expiringcache.py index fb5b27ad1..3c8409b16 100644 --- a/synapse/util/expiringcache.py +++ b/synapse/util/expiringcache.py @@ -55,7 +55,7 @@ class ExpiringCache(object): def f(): self._prune_cache() - self._clock.looping_call(f, self._expiry_ms) + self._clock.looping_call(f, self._expiry_ms/2) def __setitem__(self, key, value): now = self._clock.time_msec() From 964bb43fbe15105c58f9550f376fa76709734cd8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:44:41 +0000 Subject: [PATCH 451/630] Fix typo in function name --- synapse/util/expiringcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/expiringcache.py b/synapse/util/expiringcache.py index 3c8409b16..1c7859297 100644 --- a/synapse/util/expiringcache.py +++ b/synapse/util/expiringcache.py @@ -85,7 +85,7 @@ class ExpiringCache(object): except KeyError: return default - def _purge_cache(self): + def _prune_cache(self): if not self._expiry_ms: # zero expiry time means don't expire. This should never get called # since we have this check in start too. From 1a989c436cda4926f11b04b4f26d83e7d3ce9ef5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:45:55 +0000 Subject: [PATCH 452/630] Bump schema version --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/{v14.sql => v13.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/delta/{v14.sql => v13.sql} (100%) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c6e96b842..ec701014a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -74,7 +74,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 12 +SCHEMA_VERSION = 13 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v14.sql b/synapse/storage/schema/delta/v13.sql similarity index 100% rename from synapse/storage/schema/delta/v14.sql rename to synapse/storage/schema/delta/v13.sql From 5025305fb250c358236584cae06bf0855f02b0a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 15:57:42 +0000 Subject: [PATCH 453/630] Rate limit retries when fetching server keys. --- synapse/crypto/keyring.py | 136 +++++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 60 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 3fb99f712..3250cff59 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -87,69 +87,85 @@ class Keyring(object): return # Try to fetch the key from the remote server. - # TODO(markjh): Ratelimit requests to a given server. - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + retry_last_ts, retry_interval = (0, 0) + retry_timings = yield self.store.get_destination_retry_timings( + server_name ) + if retry_timings: + retry_last_ts, retry_interval = ( + retry_timings.retry_last_ts, retry_timings.retry_interval + ) + if retry_last_ts + retry_interval > int(self.clock.time_msec()): + logger.info("%s not ready for retry", server_name) + raise ValueError("No verification key found for given key ids") - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] - ) - - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key + try: + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory ) - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return + # Check the response. - raise ValueError("No verification key found for given key ids") + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) + + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") + + except: + self.set_retrying(server_name, retry_interval) + raise From 2b8f1a956c6a1d767a3b60e84e7d0afe5857fb0d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 17:20:56 +0000 Subject: [PATCH 454/630] Add per server retry limiting. Factor out the pre destination retry logic from TransactionQueue so it can be reused in both get_pdu and crypto.keyring --- synapse/crypto/keyring.py | 22 ++-- synapse/federation/federation_client.py | 36 ++++-- synapse/federation/transaction_queue.py | 161 ++++++++++-------------- synapse/util/retryutils.py | 108 ++++++++++++++++ 4 files changed, 205 insertions(+), 122 deletions(-) create mode 100644 synapse/util/retryutils.py diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 3250cff59..ea00c830c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -22,6 +22,8 @@ from syutil.crypto.signing_key import ( from syutil.base64util import decode_base64, encode_base64 from synapse.api.errors import SynapseError, Codes +from synapse.util.retryutils import get_retry_limiter + from OpenSSL import crypto import logging @@ -88,19 +90,13 @@ class Keyring(object): # Try to fetch the key from the remote server. - retry_last_ts, retry_interval = (0, 0) - retry_timings = yield self.store.get_destination_retry_timings( - server_name + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, ) - if retry_timings: - retry_last_ts, retry_interval = ( - retry_timings.retry_last_ts, retry_timings.retry_interval - ) - if retry_last_ts + retry_interval > int(self.clock.time_msec()): - logger.info("%s not ready for retry", server_name) - raise ValueError("No verification key found for given key ids") - try: + with limiter: (response, tls_certificate) = yield fetch_server_key( server_name, self.hs.tls_context_factory ) @@ -165,7 +161,3 @@ class Keyring(object): return raise ValueError("No verification key found for given key ids") - - except: - self.set_retrying(server_name, retry_interval) - raise diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 70c9a6f46..c5b0274c2 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -23,6 +23,8 @@ from synapse.api.errors import CodeMessageException from synapse.util.logutils import log_function from synapse.events import FrozenEvent +from synapse.util.retryutils import get_retry_limiter, NotRetryingDestination + import logging @@ -163,24 +165,34 @@ class FederationClient(FederationBase): pdu = None for destination in destinations: try: - transaction_data = yield self.transport_layer.get_event( - destination, event_id + limiter = yield get_retry_limiter( + destination, + self._clock, + self.store, ) - logger.debug("transaction_data %r", transaction_data) + with limiter: + transaction_data = yield self.transport_layer.get_event( + destination, event_id + ) - pdu_list = [ - self.event_from_pdu_json(p, outlier=outlier) - for p in transaction_data["pdus"] - ] + logger.debug("transaction_data %r", transaction_data) - if pdu_list: - pdu = pdu_list[0] + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction_data["pdus"] + ] - # Check signatures are correct. - pdu = yield self._check_sigs_and_hash(pdu) + if pdu_list: + pdu = pdu_list[0] - break + # Check signatures are correct. + pdu = yield self._check_sigs_and_hash(pdu) + + break + except NotRetryingDestination as e: + logger.info(e.message) + continue except CodeMessageException: raise except Exception as e: diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index bb20f2eba..7d02afe16 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -22,6 +22,9 @@ from .units import Transaction from synapse.api.errors import HttpResponseException from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext +from synapse.util.retryutils import ( + get_retry_limiter, NotRetryingDestination, +) import logging @@ -138,25 +141,6 @@ class TransactionQueue(object): @defer.inlineCallbacks @log_function def _attempt_new_transaction(self, destination): - - (retry_last_ts, retry_interval) = (0, 0) - retry_timings = yield self.store.get_destination_retry_timings( - destination - ) - if retry_timings: - (retry_last_ts, retry_interval) = ( - retry_timings.retry_last_ts, retry_timings.retry_interval - ) - if retry_last_ts + retry_interval > int(self._clock.time_msec()): - logger.info( - "TX [%s] not ready for retry yet - " - "dropping transaction for now", - destination, - ) - return - else: - logger.info("TX [%s] is ready for retry", destination) - if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. @@ -204,77 +188,79 @@ class TransactionQueue(object): ] try: - self.pending_transactions[destination] = 1 - - logger.debug("TX [%s] Persisting transaction...", destination) - - transaction = Transaction.create_new( - origin_server_ts=int(self._clock.time_msec()), - transaction_id=str(self._next_txn_id), - origin=self.server_name, - destination=destination, - pdus=pdus, - edus=edus, - pdu_failures=failures, - ) - - self._next_txn_id += 1 - - yield self.transaction_actions.prepare_to_send(transaction) - - logger.debug("TX [%s] Persisted transaction", destination) - logger.info( - "TX [%s] Sending transaction [%s]", + limiter = yield get_retry_limiter( destination, - transaction.transaction_id, + self._clock, + self.store, ) - # Actually send the transaction + with limiter: + self.pending_transactions[destination] = 1 - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def json_data_cb(): - data = transaction.get_dict() - now = int(self._clock.time_msec()) - if "pdus" in data: - for p in data["pdus"]: - if "age_ts" in p: - unsigned = p.setdefault("unsigned", {}) - unsigned["age"] = now - int(p["age_ts"]) - del p["age_ts"] - return data + logger.debug("TX [%s] Persisting transaction...", destination) - try: - response = yield self.transport_layer.send_transaction( - transaction, json_data_cb + transaction = Transaction.create_new( + origin_server_ts=int(self._clock.time_msec()), + transaction_id=str(self._next_txn_id), + origin=self.server_name, + destination=destination, + pdus=pdus, + edus=edus, + pdu_failures=failures, ) - code = 200 - except HttpResponseException as e: - code = e.code - response = e.response - logger.info("TX [%s] got %d response", destination, code) + self._next_txn_id += 1 - logger.debug("TX [%s] Sent transaction", destination) - logger.debug("TX [%s] Marking as delivered...", destination) + yield self.transaction_actions.prepare_to_send(transaction) - yield self.transaction_actions.delivered( - transaction, code, response - ) + logger.debug("TX [%s] Persisted transaction", destination) + logger.info( + "TX [%s] Sending transaction [%s]", + destination, + transaction.transaction_id, + ) + + # Actually send the transaction + + # FIXME (erikj): This is a bit of a hack to make the Pdu age + # keys work + def json_data_cb(): + data = transaction.get_dict() + now = int(self._clock.time_msec()) + if "pdus" in data: + for p in data["pdus"]: + if "age_ts" in p: + unsigned = p.setdefault("unsigned", {}) + unsigned["age"] = now - int(p["age_ts"]) + del p["age_ts"] + return data + + try: + response = yield self.transport_layer.send_transaction( + transaction, json_data_cb + ) + code = 200 + except HttpResponseException as e: + code = e.code + response = e.response + + logger.info("TX [%s] got %d response", destination, code) + + logger.debug("TX [%s] Sent transaction", destination) + logger.debug("TX [%s] Marking as delivered...", destination) + + yield self.transaction_actions.delivered( + transaction, code, response + ) + + logger.debug("TX [%s] Marked as delivered", destination) - logger.debug("TX [%s] Marked as delivered", destination) logger.debug("TX [%s] Yielding to callbacks...", destination) for deferred in deferreds: if code == 200: - if retry_last_ts: - # this host is alive! reset retry schedule - yield self.store.set_destination_retry_timings( - destination, 0, 0 - ) deferred.callback(None) else: - self.set_retrying(destination, retry_interval) deferred.errback(RuntimeError("Got status %d" % code)) # Ensures we don't continue until all callbacks on that @@ -285,6 +271,12 @@ class TransactionQueue(object): pass logger.debug("TX [%s] Yielded to callbacks", destination) + except NotRetryingDestination: + logger.info( + "TX [%s] not ready for retry yet - " + "dropping transaction for now", + destination, + ) except RuntimeError as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. @@ -302,8 +294,6 @@ class TransactionQueue(object): e, ) - self.set_retrying(destination, retry_interval) - for deferred in deferreds: if not deferred.called: deferred.errback(e) @@ -314,22 +304,3 @@ class TransactionQueue(object): # Check to see if there is anything else to send. self._attempt_new_transaction(destination) - - @defer.inlineCallbacks - def set_retrying(self, destination, retry_interval): - # track that this destination is having problems and we should - # give it a chance to recover before trying it again - - if retry_interval: - retry_interval *= 2 - # plateau at hourly retries for now - if retry_interval >= 60 * 60 * 1000: - retry_interval = 60 * 60 * 1000 - else: - retry_interval = 2000 # try again at first after 2 seconds - - yield self.store.set_destination_retry_timings( - destination, - int(self._clock.time_msec()), - retry_interval - ) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py new file mode 100644 index 000000000..bba524ebd --- /dev/null +++ b/synapse/util/retryutils.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +import logging + + +logger = logging.getLogger(__name__) + + +class NotRetryingDestination(Exception): + def __init__(self, retry_last_ts, retry_interval, destination): + msg = "Not retrying server %s." % (destination,) + super(NotRetryingDestination, self).__init__(msg) + + self.retry_last_ts = retry_last_ts + self.retry_interval = retry_interval + self.destination = destination + + +@defer.inlineCallbacks +def get_retry_limiter(destination, clock, store, **kwargs): + retry_last_ts, retry_interval = (0, 0) + + retry_timings = yield store.get_destination_retry_timings( + destination + ) + + if retry_timings: + retry_last_ts, retry_interval = ( + retry_timings.retry_last_ts, retry_timings.retry_interval + ) + + now = int(clock.time_msec()) + + if retry_last_ts + retry_interval > now: + raise NotRetryingDestination( + retry_last_ts=retry_last_ts, + retry_interval=retry_interval, + destination=destination, + ) + + defer.returnValue( + RetryDestinationLimiter( + destination, + clock, + store, + retry_interval, + **kwargs + ) + ) + + +class RetryDestinationLimiter(object): + def __init__(self, destination, clock, store, retry_interval, + min_retry_interval=20000, max_retry_interval=60 * 60 * 1000, + multiplier_retry_interval=2): + self.clock = clock + self.store = store + self.destination = destination + + self.retry_interval = retry_interval + self.min_retry_interval = min_retry_interval + self.max_retry_interval = max_retry_interval + self.multiplier_retry_interval = multiplier_retry_interval + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + def err(self, failure): + logger.exception( + "Failed to store set_destination_retry_timings", + failure.value + ) + + if exc_type is None and exc_val is None and exc_tb is None: + # We connected successfully. + retry_last_ts = 0 + self.retry_interval = 0 + else: + # We couldn't connect. + if self.retry_interval: + self.retry_interval *= self.multiplier_retry_interval + + if self.retry_interval >= self.max_retry_interval: + self.retry_interval = self.max_retry_interval + else: + self.retry_interval = self.min_retry_interval + + retry_last_ts = int(self._clock.time_msec()), + + self.store.set_destination_retry_timings( + self.destination, retry_last_ts, self.retry_interval + ).addErrback(err) From 2c29ed3e847285973ab552b9617750e1ba6693e0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 17:22:24 +0000 Subject: [PATCH 455/630] Use absolute path when loading delta sql files --- synapse/storage/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index ec701014a..d16e7b8fa 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -76,6 +76,8 @@ SCHEMAS = [ # database schema files, so the users will be informed on server restarts. SCHEMA_VERSION = 13 +dir_path = os.path.abspath(os.path.dirname(__file__)) + class _RollbackButIsFineException(Exception): """ This exception is used to rollback a transaction without implying @@ -583,7 +585,6 @@ def schema_path(schema): A filesystem path pointing at a ".sql" file. """ - dir_path = os.path.dirname(__file__) schemaPath = os.path.join(dir_path, "schema", schema + ".sql") return schemaPath From f91263b1e03a6e30480af1ece357c27439a8c6b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 17:37:51 +0000 Subject: [PATCH 456/630] Remove spurious self --- synapse/util/retryutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index bba524ebd..9ab119f29 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -81,7 +81,7 @@ class RetryDestinationLimiter(object): pass def __exit__(self, exc_type, exc_val, exc_tb): - def err(self, failure): + def err(failure): logger.exception( "Failed to store set_destination_retry_timings", failure.value From c8436b38a08c29fd7027e040564678ec8a635a14 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 17:38:38 +0000 Subject: [PATCH 457/630] Only update destination_retry_timings if we have succeeded when retrying --- synapse/util/retryutils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 9ab119f29..888b7ef2e 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -89,6 +89,9 @@ class RetryDestinationLimiter(object): if exc_type is None and exc_val is None and exc_tb is None: # We connected successfully. + if not self.retry_interval: + return + retry_last_ts = 0 self.retry_interval = 0 else: From 9371019133bf16cec163d58fe69aa701c8ca5305 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Feb 2015 18:13:34 +0000 Subject: [PATCH 458/630] Try to only back off if we think we failed to connect to the remote --- synapse/crypto/keyring.py | 100 ++++++++++++------------ synapse/federation/transaction_queue.py | 76 +++++++++--------- synapse/util/retryutils.py | 10 ++- 3 files changed, 96 insertions(+), 90 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index ea00c830c..828aced44 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -101,63 +101,63 @@ class Keyring(object): server_name, self.hs.tls_context_factory ) - # Check the response. + # Check the response. - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") - tls_certificate_b64 = response["tls_certificate"] + tls_certificate_b64 = response["tls_certificate"] - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] - ) - - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] ) - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return + # Cache the result in the datastore. - raise ValueError("No verification key found for given key ids") + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 7d02afe16..dc0f30cb6 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -167,15 +167,6 @@ class TransactionQueue(object): logger.info("TX [%s] Nothing to send", destination) return - logger.debug( - "TX [%s] Attempting new transaction" - " (pdus: %d, edus: %d, failures: %d)", - destination, - len(pending_pdus), - len(pending_edus), - len(pending_failures) - ) - # Sort based on the order field pending_pdus.sort(key=lambda t: t[2]) @@ -194,32 +185,41 @@ class TransactionQueue(object): self.store, ) + logger.debug( + "TX [%s] Attempting new transaction" + " (pdus: %d, edus: %d, failures: %d)", + destination, + len(pending_pdus), + len(pending_edus), + len(pending_failures) + ) + + self.pending_transactions[destination] = 1 + + logger.debug("TX [%s] Persisting transaction...", destination) + + transaction = Transaction.create_new( + origin_server_ts=int(self._clock.time_msec()), + transaction_id=str(self._next_txn_id), + origin=self.server_name, + destination=destination, + pdus=pdus, + edus=edus, + pdu_failures=failures, + ) + + self._next_txn_id += 1 + + yield self.transaction_actions.prepare_to_send(transaction) + + logger.debug("TX [%s] Persisted transaction", destination) + logger.info( + "TX [%s] Sending transaction [%s]", + destination, + transaction.transaction_id, + ) + with limiter: - self.pending_transactions[destination] = 1 - - logger.debug("TX [%s] Persisting transaction...", destination) - - transaction = Transaction.create_new( - origin_server_ts=int(self._clock.time_msec()), - transaction_id=str(self._next_txn_id), - origin=self.server_name, - destination=destination, - pdus=pdus, - edus=edus, - pdu_failures=failures, - ) - - self._next_txn_id += 1 - - yield self.transaction_actions.prepare_to_send(transaction) - - logger.debug("TX [%s] Persisted transaction", destination) - logger.info( - "TX [%s] Sending transaction [%s]", - destination, - transaction.transaction_id, - ) - # Actually send the transaction # FIXME (erikj): This is a bit of a hack to make the Pdu age @@ -249,11 +249,11 @@ class TransactionQueue(object): logger.debug("TX [%s] Sent transaction", destination) logger.debug("TX [%s] Marking as delivered...", destination) - yield self.transaction_actions.delivered( - transaction, code, response - ) + yield self.transaction_actions.delivered( + transaction, code, response + ) - logger.debug("TX [%s] Marked as delivered", destination) + logger.debug("TX [%s] Marked as delivered", destination) logger.debug("TX [%s] Yielding to callbacks...", destination) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 888b7ef2e..d190100c8 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from synapse.api.errors import CodeMessageException + import logging @@ -67,7 +69,7 @@ def get_retry_limiter(destination, clock, store, **kwargs): class RetryDestinationLimiter(object): def __init__(self, destination, clock, store, retry_interval, min_retry_interval=20000, max_retry_interval=60 * 60 * 1000, - multiplier_retry_interval=2): + multiplier_retry_interval=2,): self.clock = clock self.store = store self.destination = destination @@ -87,7 +89,11 @@ class RetryDestinationLimiter(object): failure.value ) - if exc_type is None and exc_val is None and exc_tb is None: + valid_err_code = False + if exc_type is CodeMessageException: + valid_err_code = 0 <= exc_val.code < 500 + + if exc_type is None or valid_err_code: # We connected successfully. if not self.retry_interval: return From d77912ff4483fa63308eeefb70da93b8c327b740 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:09:54 +0000 Subject: [PATCH 459/630] Docs. --- synapse/util/retryutils.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index d190100c8..08285e811 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -35,6 +35,23 @@ class NotRetryingDestination(Exception): @defer.inlineCallbacks def get_retry_limiter(destination, clock, store, **kwargs): + """For a given destination check if we have previously failed to + send a request there and are waiting before retrying the destination. + If we are not ready to retry the destination, this will raise a + NotRetryingDestination exception. Otherwise, will return a Context Manager + that will mark the destination as down if an exception is thrown (excluding + CodeMessageException with code < 500) + + Example usage: + + try: + limiter = yield get_retry_limiter(destination, clock, store) + with limiter: + response = yield do_request() + except NotRetryingDestination: + # We aren't ready to retry that destination. + raise + """ retry_last_ts, retry_interval = (0, 0) retry_timings = yield store.get_destination_retry_timings( @@ -68,8 +85,23 @@ def get_retry_limiter(destination, clock, store, **kwargs): class RetryDestinationLimiter(object): def __init__(self, destination, clock, store, retry_interval, - min_retry_interval=20000, max_retry_interval=60 * 60 * 1000, + min_retry_interval=5000, max_retry_interval=60 * 60 * 1000, multiplier_retry_interval=2,): + """ + Args: + destination (str) + clock (Clock) + store (DataStore) + retry_interval (int): The next retry interval taken from the + database in milliseconds, or zero if the last request was + successful. + min_retry_interval (int): The minimum retry interval to use after + a failed request, in milliseconds. + max_retry_interval (int): The maximum retry interval to use after + a failed request, in milliseconds. + multiplier_retry_interval (int): The multiplier to use to increase + the retry interval after a failed request. + """ self.clock = clock self.store = store self.destination = destination From 4fd176a41db9f3b9073db172e20c8ede40b5f5f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:11:24 +0000 Subject: [PATCH 460/630] More docs --- synapse/util/retryutils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 08285e811..366f7c48a 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -87,7 +87,11 @@ class RetryDestinationLimiter(object): def __init__(self, destination, clock, store, retry_interval, min_retry_interval=5000, max_retry_interval=60 * 60 * 1000, multiplier_retry_interval=2,): - """ + """Marks the destination as "down" if an exception is thrown in the + context, except for CodeMessageException with code < 500. + + If no exception is raised, marks the destination as "up". + Args: destination (str) clock (Clock) From ec847059f3e9b9b5de62aa2f7ad2366c4e883fac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:14:10 +0000 Subject: [PATCH 461/630] Rename _fail_fetch_pdu_cache to _get_pdu_cache --- synapse/app/homeserver.py | 2 +- synapse/federation/federation_client.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7565d9444..7be82d057 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -275,7 +275,7 @@ def setup(): hs.get_pusherpool().start() hs.get_state_handler().start_caching() hs.get_datastore().start_profiling() - hs.get_replication_layer().start_pdu_fail_cache() + hs.get_replication_layer().start_get_pdu_cache() if config.daemonize: print config.pid_file diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 83b4947b9..6042e366b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -32,10 +32,10 @@ logger = logging.getLogger(__name__) class FederationClient(FederationBase): def __init__(self): - self._fail_fetch_pdu_cache = None + self._get_pdu_cache = None - def start_pdu_fail_cache(self): - self._fail_fetch_pdu_cache = ExpiringCache( + def start_get_pdu_cache(self): + self._get_pdu_cache = ExpiringCache( cache_name="get_pdu_cache", clock=self._clock, max_len=1000, @@ -43,7 +43,7 @@ class FederationClient(FederationBase): reset_expiry_on_get=False, ) - self._fail_fetch_pdu_cache.start() + self._get_pdu_cache.start() @log_function def send_pdu(self, pdu, destinations): @@ -175,8 +175,8 @@ class FederationClient(FederationBase): # TODO: Rate limit the number of times we try and get the same event. - if self._fail_fetch_pdu_cache: - e = self._fail_fetch_pdu_cache.get(event_id) + if self._get_pdu_cache: + e = self._get_pdu_cache.get(event_id) if e: defer.returnValue(e) @@ -223,8 +223,8 @@ class FederationClient(FederationBase): ) continue - if self._fail_fetch_pdu_cache is not None: - self._fail_fetch_pdu_cache[event_id] = pdu + if self._get_pdu_cache is not None: + self._get_pdu_cache[event_id] = pdu defer.returnValue(pdu) From 859fbd4423c57a0d64dde5322dde46120692f64b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:39:14 +0000 Subject: [PATCH 462/630] s/self._clock/self.clock/ --- synapse/util/retryutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 366f7c48a..ac3054507 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -146,7 +146,7 @@ class RetryDestinationLimiter(object): else: self.retry_interval = self.min_retry_interval - retry_last_ts = int(self._clock.time_msec()), + retry_last_ts = int(self.clock.time_msec()), self.store.set_destination_retry_timings( self.destination, retry_last_ts, self.retry_interval From 0db52d43fa41526b83d0c70141c43bfd89388820 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:41:46 +0000 Subject: [PATCH 463/630] strings.join() expects iterable of strings --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 454c3d4ab..b5e201b5f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -148,7 +148,7 @@ class MatrixFederationHttpClient(object): if hasattr(e, "reasons"): reasons = ", ".join( - f.value.message + str(f.value.message) for f in e.reasons ) else: From e482541e1d77cb179dafa44037ee1e04003501ab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:44:22 +0000 Subject: [PATCH 464/630] Fix pyflakes --- synapse/federation/transaction_queue.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index a77201ad4..ae04774c7 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -266,7 +266,6 @@ class TransactionQueue(object): logger.debug("TX [%s] Sent transaction", destination) logger.debug("TX [%s] Marking as delivered...", destination) - yield self.transaction_actions.delivered( transaction, code, response ) From 5e2447146982a30d0d4ea1334297cc0bfb32c3c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:50:10 +0000 Subject: [PATCH 465/630] Fix up ResponseNeverReceived to str --- synapse/http/matrixfederationclient.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b5e201b5f..304efcdd5 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -146,21 +146,13 @@ class MatrixFederationHttpClient(object): ) raise SynapseError(400, "Domain specified not found.") - if hasattr(e, "reasons"): - reasons = ", ".join( - str(f.value.message) - for f in e.reasons - ) - else: - reasons = e.message - logger.warn( "Sending request failed to %s: %s %s: %s - %s", destination, method, url_bytes, type(e). __name__, - reasons, + _flatten_response_never_received(e), ) if retries_left: @@ -474,3 +466,13 @@ class _JsonProducer(object): def stopProducing(self): pass + + +def _flatten_response_never_received(e): + if hasattr(e, "reasons"): + return ", ".join( + _flatten_response_never_received(f.value) + for f in e.reasons + ) + else: + return "%s: %s" % (type(e), e.message,) From 65ca713ff53794bd517a85de7df063e631fb255a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:51:32 +0000 Subject: [PATCH 466/630] Add .__name__ after type(e) --- synapse/http/matrixfederationclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 304efcdd5..49a4c7d9d 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -151,7 +151,7 @@ class MatrixFederationHttpClient(object): destination, method, url_bytes, - type(e). __name__, + type(e).__name__, _flatten_response_never_received(e), ) @@ -475,4 +475,4 @@ def _flatten_response_never_received(e): for f in e.reasons ) else: - return "%s: %s" % (type(e), e.message,) + return "%s: %s" % (type(e).__name__, e.message,) From cedad8fbd688eef70da4d758987bb39f46fe84bf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:54:34 +0000 Subject: [PATCH 467/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 792b8089f..a804a2574 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.0f" +__version__ = "0.7.1" From 7e9d59f3b4628981d5e53ac0cc71325ad896287c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 10:58:13 +0000 Subject: [PATCH 468/630] Don't convert DNSLookupError to a 4xx SynapseError --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 49a4c7d9d..fe5736338 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -144,7 +144,7 @@ class MatrixFederationHttpClient(object): destination, e ) - raise SynapseError(400, "Domain specified not found.") + raise logger.warn( "Sending request failed to %s: %s %s: %s - %s", From 87e9aeb9143a3dab2dcac24797c822dedde845f1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Feb 2015 10:59:30 +0000 Subject: [PATCH 469/630] Move pynacl to the top of the depedency link list so that it is installed before syutil --- synapse/python_dependencies.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ec78fc362..a30a57055 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -24,6 +24,11 @@ def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) DEPENDENCY_LINKS = [ + github_link( + project="pyca/pynacl", + version="d4d3175589b892f6ea7c22f466e0e223853516fa", + egg="pynacl-0.3.0", + ) github_link( project="matrix-org/syutil", version="v0.0.3", @@ -34,11 +39,6 @@ DEPENDENCY_LINKS = [ version="v0.6.2", egg="matrix_angular_sdk-0.6.2", ), - github_link( - project="pyca/pynacl", - version="d4d3175589b892f6ea7c22f466e0e223853516fa", - egg="pynacl-0.3.0", - ) ] From 5806d52423a97e219b710ca638866e0763fa416e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Feb 2015 11:01:37 +0000 Subject: [PATCH 470/630] Fix syntax --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a30a57055..9464b4fb6 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -28,7 +28,7 @@ DEPENDENCY_LINKS = [ project="pyca/pynacl", version="d4d3175589b892f6ea7c22f466e0e223853516fa", egg="pynacl-0.3.0", - ) + ), github_link( project="matrix-org/syutil", version="v0.0.3", From b17bd31da0574cf335f90763dee2708af40ac87f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 11:17:26 +0000 Subject: [PATCH 471/630] Temporarily add a run_on_reactor() call --- synapse/util/retryutils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index ac3054507..ee1194f1d 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -17,6 +17,8 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException +from synapse.util.async import run_on_reactor + import logging @@ -52,6 +54,7 @@ def get_retry_limiter(destination, clock, store, **kwargs): # We aren't ready to retry that destination. raise """ + yield run_on_reactor() retry_last_ts, retry_interval = (0, 0) retry_timings = yield store.get_destination_retry_timings( From 36e144091b3be995754f7100a9c5322b224f2692 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 11:25:20 +0000 Subject: [PATCH 472/630] Remove spurious comma. Remove temp run_on_reactor --- synapse/util/retryutils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index ee1194f1d..ab58eb528 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -54,7 +54,6 @@ def get_retry_limiter(destination, clock, store, **kwargs): # We aren't ready to retry that destination. raise """ - yield run_on_reactor() retry_last_ts, retry_interval = (0, 0) retry_timings = yield store.get_destination_retry_timings( @@ -149,7 +148,7 @@ class RetryDestinationLimiter(object): else: self.retry_interval = self.min_retry_interval - retry_last_ts = int(self.clock.time_msec()), + retry_last_ts = int(self.clock.time_msec()) self.store.set_destination_retry_timings( self.destination, retry_last_ts, self.retry_interval From 47d3ff4cf8ebceacc680c9e5b414150c6edbf85b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 11:30:37 +0000 Subject: [PATCH 473/630] Don't send failure to self --- synapse/federation/transaction_queue.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index ae04774c7..f0041d4d1 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -128,6 +128,9 @@ class TransactionQueue(object): @defer.inlineCallbacks def enqueue_failure(self, failure, destination): + if destination == self.server_name: + return + deferred = defer.Deferred() self.pending_failures_by_dest.setdefault( From b68e4a729ff91372a3a7269acc08c1f77584b0f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 11:32:39 +0000 Subject: [PATCH 474/630] Discard destination 'localhost' --- synapse/federation/transaction_queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index f0041d4d1..06944eefb 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -107,7 +107,7 @@ class TransactionQueue(object): def enqueue_edu(self, edu): destination = edu.destination - if destination == self.server_name: + if destination == self.server_name or destination == "localhost": return deferred = defer.Deferred() @@ -128,7 +128,7 @@ class TransactionQueue(object): @defer.inlineCallbacks def enqueue_failure(self, failure, destination): - if destination == self.server_name: + if destination == self.server_name or destination == "localhost": return deferred = defer.Deferred() From 2462aacd77963431507bb97769acee3ed9d65ceb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Feb 2015 11:51:00 +0000 Subject: [PATCH 475/630] Restrict the destinations that synapse can talk to --- synapse/federation/transaction_queue.py | 30 ++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index ae04774c7..4b5460c79 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -66,6 +66,26 @@ class TransactionQueue(object): # HACK to get unique tx id self._next_txn_id = int(self._clock.time_msec()) + def can_send_to(self, destination): + """Can we send messages to the given server? + + We can't send messages to ourselves. If we are running on localhost + then we can only federation with other servers running on localhost. + Otherwise we only federate with servers on a public domain. + + Args: + destination(str): The server we are possibly trying to send to. + Returns: + bool: True if we can send to the server. + """ + + if destination == self.server_name: + return False + if self.server_name.startswith("localhost"): + return destination.startswith("localhost") + else: + return not destination.startswith("localhost") + @defer.inlineCallbacks @log_function def enqueue_pdu(self, pdu, destinations, order): @@ -74,8 +94,9 @@ class TransactionQueue(object): # table and we'll get back to it later. destinations = set(destinations) - destinations.discard(self.server_name) - destinations.discard("localhost") + destinations = set( + dest for dest in destinations if self.can_send_to(dest) + ) logger.debug("Sending to: %s", str(destinations)) @@ -107,7 +128,7 @@ class TransactionQueue(object): def enqueue_edu(self, edu): destination = edu.destination - if destination == self.server_name: + if not self.can_send_to(destination): return deferred = defer.Deferred() @@ -130,6 +151,9 @@ class TransactionQueue(object): def enqueue_failure(self, failure, destination): deferred = defer.Deferred() + if not self.can_send_to(destination): + return + self.pending_failures_by_dest.setdefault( destination, [] ).append( From 6375bd3e33f687499e6d15298e74416c77ab499e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 18 Feb 2015 12:01:37 +0000 Subject: [PATCH 476/630] SYN-282: Don't log tracebacks for client errors --- synapse/rest/media/v1/base_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index d44d5f129..b10cbddb8 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -54,7 +54,7 @@ class BaseMediaResource(Resource): try: yield request_handler(self, request) except CodeMessageException as e: - logger.exception(e) + logger.info("Responding with error: %r", e) respond_with_json( request, e.code, cs_exception(e), send_cors=True ) From cc3d3babb0c04b21501f59c25739b9e48fa5b490 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 12:01:41 +0000 Subject: [PATCH 477/630] Remove unused import --- synapse/util/retryutils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index ab58eb528..4e8223279 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -17,8 +17,6 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException -from synapse.util.async import run_on_reactor - import logging From 446ef589920a9c3c4738873023494bed79d1f6c3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 12:03:12 +0000 Subject: [PATCH 478/630] Add errback to all deferreds in transaction_queue --- synapse/federation/transaction_queue.py | 37 +++++++++++++++---------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 06944eefb..1e5505a4b 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -90,14 +90,17 @@ class TransactionQueue(object): (pdu, deferred, order) ) - def eb(failure): + def chain(failure): if not deferred.called: deferred.errback(failure) - else: - logger.warn("Failed to send pdu", failure.value) + + def log_failure(failure): + logger.warn("Failed to send pdu", failure.value) + + deferred.addErrback(log_failure) with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) + self._attempt_new_transaction(destination).addErrback(chain) deferreds.append(deferred) @@ -115,14 +118,17 @@ class TransactionQueue(object): (edu, deferred) ) - def eb(failure): + def chain(failure): if not deferred.called: deferred.errback(failure) - else: - logger.warn("Failed to send edu", failure.value) + + def log_failure(failure): + logger.warn("Failed to send pdu", failure.value) + + deferred.addErrback(log_failure) with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) + self._attempt_new_transaction(destination).addErrback(chain) return deferred @@ -139,14 +145,17 @@ class TransactionQueue(object): (failure, deferred) ) - def eb(failure): + def chain(f): if not deferred.called: - deferred.errback(failure) - else: - logger.warn("Failed to send failure", failure.value) + deferred.errback(f) + + def log_failure(f): + logger.warn("Failed to send pdu", f.value) + + deferred.addErrback(log_failure) with PreserveLoggingContext(): - self._attempt_new_transaction(destination).addErrback(eb) + self._attempt_new_transaction(destination).addErrback(chain) yield deferred @@ -308,7 +317,7 @@ class TransactionQueue(object): except Exception as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. - logger.exception( + logger.warn( "TX [%s] Problem in _attempt_transaction: %s", destination, e, From d122e215ff459648d0352627c91adb00c6a34213 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:21:35 +0000 Subject: [PATCH 479/630] Generate a version string that includes git details if run from git checkout --- synapse/app/homeserver.py | 62 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 351fc6914..166a0169a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -52,6 +52,7 @@ import synapse import logging import os import re +import subprocess import sqlite3 import syweb @@ -208,6 +209,61 @@ class SynapseHomeServer(HomeServer): logger.info("Synapse now listening on port %d", unsecure_port) +def get_version_string(): + null = open(os.devnull, 'w') + try: + git_branch = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + stderr=null, + ).strip() + git_branch = "b=" + git_branch + except subprocess.CalledProcessError: + git_branch = "" + + try: + git_tag = subprocess.check_output( + ['git', 'describe', '--exact-match'], + stderr=null, + ).strip() + git_tag = "t=" + git_tag + except subprocess.CalledProcessError: + git_tag = "" + + try: + git_commit = subprocess.check_output( + ['git', 'rev-parse', '--short', 'HEAD'], + stderr=null, + ).strip() + except subprocess.CalledProcessError: + git_commit = "" + + try: + dirty_string = "-this_is_a_dirty_checkout" + is_dirty = subprocess.check_output( + ['git', 'describe', '--dirty=' + dirty_string], + stderr=null, + ).strip().endswith(dirty_string) + + git_dirty = "dirty" if is_dirty else "" + except subprocess.CalledProcessError: + git_dirty = "" + + if git_branch or git_tag or git_commit or git_dirty: + git_version = ",".join( + s for s in + (git_branch, git_tag, git_commit, git_dirty,) + if s + ) + + return ( + "Synapse/v=%s,%s" % ( + synapse.__version__, git_version, + ) + ).encode("ascii") + + return ("Synapse/%s" % (synapse.__version__,)).encode("ascii") + + def setup(): config = HomeServerConfig.load_config( "Synapse Homeserver", @@ -219,8 +275,10 @@ def setup(): check_requirements() + version_string = get_version_string() + logger.info("Server hostname: %s", config.server_name) - logger.info("Server version: %s", synapse.__version__) + logger.info("Server version: %s", version_string) if re.search(":[0-9]+$", config.server_name): domain_with_port = config.server_name @@ -229,6 +287,7 @@ def setup(): tls_context_factory = context_factory.ServerContextFactory(config) + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, @@ -237,6 +296,7 @@ def setup(): tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, + version_string=version_string, ) hs.create_resource_tree( From aa577df06405c1574084389bcd189baedc3a7397 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:26:30 +0000 Subject: [PATCH 480/630] When computing git version run git commands in same dir as source files --- synapse/app/homeserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 166a0169a..6acb3b2f4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -211,10 +211,12 @@ class SynapseHomeServer(HomeServer): def get_version_string(): null = open(os.devnull, 'w') + cwd = os.path.dirname(os.path.abspath(__file__)) try: git_branch = subprocess.check_output( ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stderr=null, + cwd=cwd, ).strip() git_branch = "b=" + git_branch except subprocess.CalledProcessError: @@ -224,6 +226,7 @@ def get_version_string(): git_tag = subprocess.check_output( ['git', 'describe', '--exact-match'], stderr=null, + cwd=cwd, ).strip() git_tag = "t=" + git_tag except subprocess.CalledProcessError: @@ -233,6 +236,7 @@ def get_version_string(): git_commit = subprocess.check_output( ['git', 'rev-parse', '--short', 'HEAD'], stderr=null, + cwd=cwd, ).strip() except subprocess.CalledProcessError: git_commit = "" @@ -242,6 +246,7 @@ def get_version_string(): is_dirty = subprocess.check_output( ['git', 'describe', '--dirty=' + dirty_string], stderr=null, + cwd=cwd, ).strip().endswith(dirty_string) git_dirty = "dirty" if is_dirty else "" From 5358966a878aa543659c10be09f4bf58b1568f2f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:51:33 +0000 Subject: [PATCH 481/630] Use git aware version string in User-Agent and Server headers --- synapse/http/agent_name.py | 18 ----------------- synapse/http/client.py | 10 +++++----- synapse/http/matrixfederationclient.py | 4 ++-- synapse/http/server.py | 27 +++++++++++++++++--------- tests/utils.py | 10 ++++++++-- 5 files changed, 33 insertions(+), 36 deletions(-) delete mode 100644 synapse/http/agent_name.py diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py deleted file mode 100644 index d76189086..000000000 --- a/synapse/http/agent_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse import __version__ - -AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii") diff --git a/synapse/http/client.py b/synapse/http/client.py index e46e7db14..22b7145ac 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -14,7 +14,6 @@ # limitations under the License. from synapse.api.errors import CodeMessageException -from synapse.http.agent_name import AGENT_NAME from syutil.jsonutil import encode_canonical_json from twisted.internet import defer, reactor @@ -44,6 +43,7 @@ class SimpleHttpClient(object): # BrowserLikePolicyForHTTPS which will do regular cert validation # 'like a browser' self.agent = Agent(reactor) + self.version_string = hs.version_string @defer.inlineCallbacks def post_urlencoded_get_json(self, uri, args={}): @@ -55,7 +55,7 @@ class SimpleHttpClient(object): uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -108,7 +108,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }) ) @@ -149,7 +149,7 @@ class SimpleHttpClient(object): "PUT", uri.encode("ascii"), headers=Headers({ - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], "Content-Type": ["application/json"] }), bodyProducer=FileBodyProducer(StringIO(json_str)) @@ -182,7 +182,7 @@ class CaptchaServerHttpClient(SimpleHttpClient): bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": [AGENT_NAME], + b"User-Agent": [self.version_string], }) ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 49a4c7d9d..9d483032a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -20,7 +20,6 @@ from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone -from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext @@ -80,6 +79,7 @@ class MatrixFederationHttpClient(object): self.server_name = hs.hostname self.agent = MatrixFederationHttpAgent(reactor) self.clock = hs.get_clock() + self.version_string = hs.version_string @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, @@ -87,7 +87,7 @@ class MatrixFederationHttpClient(object): query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ - headers_dict[b"User-Agent"] = [AGENT_NAME] + headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( diff --git a/synapse/http/server.py b/synapse/http/server.py index 589c30c19..74a101a5d 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -14,7 +14,6 @@ # limitations under the License. -from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError ) @@ -74,6 +73,7 @@ class JsonResource(HttpServer, resource.Resource): self.clock = hs.get_clock() self.path_regexs = {} + self.version_string = hs.version_string def register_path(self, method, path_pattern, callback): self.path_regexs.setdefault(method, []).append( @@ -189,9 +189,13 @@ class JsonResource(HttpServer, resource.Resource): return # TODO: Only enable CORS for the requests that need it. - respond_with_json(request, code, response_json_object, send_cors=True, - response_code_message=response_code_message, - pretty_print=self._request_user_agent_is_curl) + respond_with_json( + request, code, response_json_object, + send_cors=True, + response_code_message=response_code_message, + pretty_print=self._request_user_agent_is_curl, + version_string=self.version_string, + ) @staticmethod def _request_user_agent_is_curl(request): @@ -221,18 +225,23 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, - response_code_message=None, pretty_print=False): + response_code_message=None, pretty_print=False, + version_string=""): if not pretty_print: json_bytes = encode_pretty_printed_json(json_object) else: json_bytes = encode_canonical_json(json_object) - return respond_with_json_bytes(request, code, json_bytes, send_cors, - response_code_message=response_code_message) + return respond_with_json_bytes( + request, code, json_bytes, + send_cors=send_cors, + response_code_message=response_code_message, + version_string=version_string + ) def respond_with_json_bytes(request, code, json_bytes, send_cors=False, - response_code_message=None): + version_string="", response_code_message=None): """Sends encoded JSON in response to the given request. Args: @@ -246,7 +255,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") - request.setHeader(b"Server", AGENT_NAME) + request.setHeader(b"Server", version_string) request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) if send_cors: diff --git a/tests/utils.py b/tests/utils.py index 39895c739..110b9f86b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -46,10 +46,16 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): if datastore is None: db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() - hs = HomeServer(name, db_pool=db_pool, config=config, **kargs) + hs = HomeServer( + name, db_pool=db_pool, config=config, + version_string="Synapse/tests", + **kargs + ) else: hs = HomeServer( - name, db_pool=None, datastore=datastore, config=config, **kargs + name, db_pool=None, datastore=datastore, config=config, + version_string="Synapse/tests", + **kargs ) defer.returnValue(hs) From 1df8bad63e7f55dca79f20919a6873157c76d8b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 16:54:25 +0000 Subject: [PATCH 482/630] pyflakes --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6acb3b2f4..3368ec675 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -292,7 +292,6 @@ def setup(): tls_context_factory = context_factory.ServerContextFactory(config) - hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, From 5b753d472bd4ffba124103753f83b55c567889fb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:02:40 +0000 Subject: [PATCH 483/630] Bump matrix-angular-sdk version --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 9464b4fb6..c71df7540 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil>=0.0.3": ["syutil"], - "matrix_angular_sdk>=0.6.2": ["syweb>=0.6.2"], + "matrix_angular_sdk>=0.6.3": ["syweb>=0.6.3"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -36,8 +36,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.2", - egg="matrix_angular_sdk-0.6.2", + version="v0.6.3", + egg="matrix_angular_sdk-0.6.3", ), ] From b6771037a6d726803f76d486bf82075553d4a7b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:31:50 +0000 Subject: [PATCH 484/630] Make version_string conform to User-Agent and Server spec --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3368ec675..bf01483b8 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -261,7 +261,7 @@ def get_version_string(): ) return ( - "Synapse/v=%s,%s" % ( + "Synapse/%s/(%s)" % ( synapse.__version__, git_version, ) ).encode("ascii") From 3f1871021e4efd1bbc513790e5c82f09b4c04503 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:32:12 +0000 Subject: [PATCH 485/630] Make /keys/ return correct Server version --- synapse/http/server_key_resource.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/http/server_key_resource.py b/synapse/http/server_key_resource.py index 4fc491dc8..71e9a51f5 100644 --- a/synapse/http/server_key_resource.py +++ b/synapse/http/server_key_resource.py @@ -50,6 +50,7 @@ class LocalKey(Resource): def __init__(self, hs): self.hs = hs + self.version_string = hs.version_string self.response_body = encode_canonical_json( self.response_json_object(hs.config) ) @@ -82,7 +83,10 @@ class LocalKey(Resource): return json_object def render_GET(self, request): - return respond_with_json_bytes(request, 200, self.response_body) + return respond_with_json_bytes( + request, 200, self.response_body, + version_string=self.version_string + ) def getChild(self, name, request): if name == '': From 6311ae8968d5f2e3b867789376933de83678e036 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 17:34:26 +0000 Subject: [PATCH 486/630] Conform to header spec take two --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bf01483b8..ea20de143 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -261,7 +261,7 @@ def get_version_string(): ) return ( - "Synapse/%s/(%s)" % ( + "Synapse/%s (%s)" % ( synapse.__version__, git_version, ) ).encode("ascii") From faf4f678474feb5d9964df6aa621990e73d1792d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Feb 2015 18:02:54 +0000 Subject: [PATCH 487/630] Update CHANGES --- CHANGES.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 883510759..04f9d9440 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,14 @@ +Changes in synapse v0.7.1 (2015-02-18) +====================================== + +* Add cache when fetching events from remote servers to stop repeatedly + fetching events with bad signatures. +* Respect the per remote server retry scheme when fetching both events and + server keys to reduce the number of times we send requests to dead servers. +* Inform remote servers when the local server fails to handle a received event. +* Turn off python bytecode generation due to problems experienced when + upgrading from previous versions. + Changes in synapse v0.7.0 (2015-02-12) ====================================== From b457f1677c0798894140eb6b7e091800c7c7bf2d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 19 Feb 2015 10:06:17 +0000 Subject: [PATCH 488/630] Send room ID in http notifications so clients know which room to go to if the user responds to the notification. --- synapse/push/httppusher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 5788db4eb..202fcb42f 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -66,6 +66,7 @@ class HttpPusher(Pusher): d = { 'notification': { 'id': event['event_id'], + 'room_id': event['room_id'], 'type': event['type'], 'sender': event['user_id'], 'counts': { # -- we don't mark messages as read yet so From 63c1f4fa987874e2bde76bd2c2d8cb35beef7d99 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 10:33:31 +0000 Subject: [PATCH 489/630] Update release date --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 04f9d9440..d4474ca5e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -Changes in synapse v0.7.1 (2015-02-18) +Changes in synapse v0.7.1 (2015-02-19) ====================================== * Add cache when fetching events from remote servers to stop repeatedly From c3eb7dd9c572cfcea831e4d5f41b5a34cb3c57e8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 11:50:49 +0000 Subject: [PATCH 490/630] Add config option to set the soft fd limit on start --- synapse/app/homeserver.py | 20 +++++++++++++++++--- synapse/config/server.py | 7 +++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ea20de143..07386f6f2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -52,6 +52,7 @@ import synapse import logging import os import re +import resource import subprocess import sqlite3 import syweb @@ -269,6 +270,15 @@ def get_version_string(): return ("Synapse/%s" % (synapse.__version__,)).encode("ascii") +def change_resource_limit(soft_file_no): + try: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) + logger.info("Set file limit to: %d", soft_file_no) + except (ValueError, resource.error) as e: + logger.warn("Failed to set file limit: %s", e) + + def setup(): config = HomeServerConfig.load_config( "Synapse Homeserver", @@ -345,10 +355,11 @@ def setup(): if config.daemonize: print config.pid_file + daemon = Daemonize( app="synapse-homeserver", pid=config.pid_file, - action=run, + action=lambda: run(config), auto_close_fds=False, verbose=True, logger=logger, @@ -356,11 +367,14 @@ def setup(): daemon.start() else: - reactor.run() + run(config) -def run(): +def run(config): with LoggingContext("run"): + if config.soft_file_limit: + change_resource_limit(config.soft_file_limit) + reactor.run() diff --git a/synapse/config/server.py b/synapse/config/server.py index 31e44cc85..a3edf208e 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -31,6 +31,7 @@ class ServerConfig(Config): self.webclient = True self.manhole = args.manhole self.no_tls = args.no_tls + self.soft_file_limit = args.soft_file_limit if not args.content_addr: host = args.server_name @@ -77,6 +78,12 @@ class ServerConfig(Config): "content repository") server_group.add_argument("--no-tls", action='store_true', help="Don't bind to the https port.") + server_group.add_argument("--soft-file-limit", type=int, default=0, + help="Set the limit on the number of file " + "descriptors synapse can use. Zero " + "is used to indicate synapse should " + "not change the limit from system " + "default.") def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") From 939273c4b0936b95eb14763d8bf1027159fa4cb3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 11:53:13 +0000 Subject: [PATCH 491/630] Rename resource variable so as to not shadow module import --- synapse/app/homeserver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 07386f6f2..5f671dfd2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -141,8 +141,8 @@ class SynapseHomeServer(HomeServer): # instead, we'll store a copy of this mapping so we can actually add # extra resources to existing nodes. See self._resource_id for the key. resource_mappings = {} - for (full_path, resource) in desired_tree: - logger.info("Attaching %s to path %s", resource, full_path) + for full_path, res in desired_tree: + logger.info("Attaching %s to path %s", res, full_path) last_resource = self.root_resource for path_seg in full_path.split('/')[1:-1]: if path_seg not in last_resource.listNames(): @@ -173,12 +173,12 @@ class SynapseHomeServer(HomeServer): child_name) child_resource = resource_mappings[child_res_id] # steal the children - resource.putChild(child_name, child_resource) + res.putChild(child_name, child_resource) # finally, insert the desired resource in the right place - last_resource.putChild(last_path_seg, resource) + last_resource.putChild(last_path_seg, res) res_id = self._resource_id(last_resource, last_path_seg) - resource_mappings[res_id] = resource + resource_mappings[res_id] = res return self.root_resource From 894a89d99b939a36c4547bcda2af59fdfe380e3d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 13:46:39 +0000 Subject: [PATCH 492/630] Update CHANGES.rst with missing changes --- CHANGES.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index d4474ca5e..0ccf4c004 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,15 @@ Changes in synapse v0.7.1 (2015-02-19) ====================================== +* Initial alpha implementation of parts of the Application Services API. + Including: + + - AS Registration / Unregistration + - User Query API + - Room Alias Query API + - Push transport for receiving events. + - User/Alias namespace admin control + * Add cache when fetching events from remote servers to stop repeatedly fetching events with bad signatures. * Respect the per remote server retry scheme when fetching both events and From 81163f822e5133ae6e724f18e70e9b84b2081a4e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 14:16:53 +0000 Subject: [PATCH 493/630] Add config option to disable registration. --- synapse/config/homeserver.py | 3 ++- synapse/config/registration.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 synapse/config/registration.py diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index b0fe21745..c024535f5 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -22,11 +22,12 @@ from .repository import ContentRepositoryConfig from .captcha import CaptchaConfig from .email import EmailConfig from .voip import VoipConfig +from .registration import RegistrationConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, - EmailConfig, VoipConfig): + EmailConfig, VoipConfig, RegistrationConfig,): pass diff --git a/synapse/config/registration.py b/synapse/config/registration.py new file mode 100644 index 000000000..cca8ab567 --- /dev/null +++ b/synapse/config/registration.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class RegistrationConfig(Config): + + def __init__(self, args): + super(RegistrationConfig, self).__init__(args) + self.disable_registration = args.disable_registration + + @classmethod + def add_arguments(cls, parser): + super(RegistrationConfig, cls).add_arguments(parser) + reg_group = parser.add_argument_group("registration") + reg_group.add_argument( + "--disable-registration", + action='store_true', + help="Disable registration of new users." + ) From 0722f982d3b664e6e1c20c3853bb3742683aee88 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 14:22:20 +0000 Subject: [PATCH 494/630] Disable registration if config option was set. --- synapse/rest/client/v1/register.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index 8d2115082..dba49129a 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -59,6 +59,7 @@ class RegisterRestServlet(ClientV1RestServlet): # } # TODO: persistent storage self.sessions = {} + self.disable_registration = hs.config.disable_registration def on_GET(self, request): if self.hs.config.enable_registration_captcha: @@ -97,6 +98,9 @@ class RegisterRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): + if self.disable_registration: + raise SynapseError(403, "Registration has been disabled") + register_json = _parse_json(request) session = (register_json["session"] From 5f4c28d313ce854e0ac943b100412878c4ef8dd9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 14:34:32 +0000 Subject: [PATCH 495/630] Update tests --- tests/rest/client/v1/test_events.py | 1 + tests/utils.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index f36340121..36b0f2ff6 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -128,6 +128,7 @@ class EventStreamPermissionsTestCase(RestTestCase): self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) hs.config.enable_registration_captcha = False + hs.config.disable_registration = False hs.get_handlers().federation_handler = Mock() diff --git a/tests/utils.py b/tests/utils.py index 110b9f86b..81e82a80d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -42,6 +42,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config = Mock() config.signing_key = [MockKey()] config.event_cache_size = 1 + config.disable_registration = False if datastore is None: db_pool = SQLiteMemoryDbPool() From 61959928bb4b6b0191d301f1b267af7290a61bd2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 14:58:07 +0000 Subject: [PATCH 496/630] =?UTF-8?q?Pull=20out=20the=20'get=5Frooms=5Ffor?= =?UTF-8?q?=5Fuser'=20cache=20logic=20into=20a=20re=C3=BCsable=20@cached?= =?UTF-8?q?=20decorator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- synapse/storage/roommember.py | 53 +++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 9bf608bc9..569bd55d0 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -33,6 +33,32 @@ RoomsForUser = namedtuple( ) +# TODO(paul): +# * Move this somewhere higher-level, shared; +# * more generic key management +# * export monitoring stats +# * maximum size; just evict things at random, or consider LRU? +def cached(orig): + cache = {} + + @defer.inlineCallbacks + def wrapped(self, key): + if key in cache: + defer.returnValue(cache[key]) + + ret = yield orig(self, key) + + cache[key] = ret; + defer.returnValue(ret) + + def invalidate(key): + if key in cache: + del cache[key] + + wrapped.invalidate = invalidate + return wrapped + + class RoomMemberStore(SQLBaseStore): def __init__(self, *args, **kw): @@ -103,7 +129,7 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (event.room_id, domain)) - self.invalidate_rooms_for_user(target_user_id) + self.get_rooms_for_user.invalidate(target_user_id) @defer.inlineCallbacks def get_room_member(self, user_id, room_id): @@ -247,33 +273,12 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results - # TODO(paul): Create a nice @cached decorator to do this - # @cached - # def get_foo(...) - # ... - # invalidate_foo = get_foo.invalidator - - @defer.inlineCallbacks + @cached def get_rooms_for_user(self, user_id): - # TODO(paul): put some performance counters in here so we can easily - # track what impact this cache is having - if user_id in self._user_rooms_cache: - defer.returnValue(self._user_rooms_cache[user_id]) - - rooms = yield self.get_rooms_for_user_where_membership_is( + return self.get_rooms_for_user_where_membership_is( user_id, membership_list=[Membership.JOIN], ) - # TODO(paul): Consider applying a maximum size; just evict things at - # random, or consider LRU? - - self._user_rooms_cache[user_id] = rooms - defer.returnValue(rooms) - - def invalidate_rooms_for_user(self, user_id): - if user_id in self._user_rooms_cache: - del self._user_rooms_cache[user_id] - @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a From 0ac2a79faa918280767c18e4db7ec29d7d3a3afb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 17:24:14 +0000 Subject: [PATCH 497/630] Initial stab at implementing a batched get_missing_pdus request --- synapse/federation/federation_server.py | 72 +++++++++++++++++++++++++ synapse/handlers/federation.py | 9 ++-- synapse/storage/event_federation.py | 63 ++++++++++++++++++++-- 3 files changed, 135 insertions(+), 9 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 22b966383..34bc397e8 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -305,6 +305,78 @@ class FederationServer(FederationBase): (200, send_content) ) + @defer.inlineCallbacks + def get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + limit = max(limit, 50) + min_depth = max(min_depth, 0) + + missing_events = yield self.store.get_missing_events( + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + limit=limit, + min_depth=min_depth, + ) + + known_ids = {e.event_id for e in missing_events} | {earliest_events} + + back_edges = { + e for e in missing_events + if {i for i, h in e.prev_events.items()} <= known_ids + } + + decoded_auth_events = set() + state = {} + auth_events = set() + auth_and_state = {} + for event in back_edges: + state_pdus = yield self.handler.get_state_for_pdu( + origin, room_id, event.event_id, + do_auth=False, + ) + + state[event.event_id] = [s.event_id for s in state_pdus] + + auth_and_state.update({ + s.event_id: s for s in state_pdus + }) + + state_ids = {pdu.event_id for pdu in state_pdus} + prev_ids = {i for i, h in event.prev_events.items()} + partial_auth_chain = yield self.store.get_auth_chain( + state_ids | prev_ids, have_ids=decoded_auth_events.keys() + ) + + for p in partial_auth_chain: + p.signatures.update( + compute_event_signature( + p, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + + auth_events.update( + a.event_id for a in partial_auth_chain + ) + + auth_and_state.update({ + a.event_id: a for a in partial_auth_chain + }) + + time_now = self._clock.time_msec() + + defer.returnValue({ + "events": [ev.get_pdu_json(time_now) for ev in missing_events], + "state_for_events": state, + "auth_events": auth_events, + "event_map": { + k: ev.get_pdu_json(time_now) + for k, ev in auth_and_state.items() + }, + }) + @log_function def _get_persisted_pdu(self, origin, event_id, do_auth=True): """ Get a PDU from the database with given origin and id. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0eb2ff95c..26bdc6d1a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -581,12 +581,13 @@ class FederationHandler(BaseHandler): defer.returnValue(event) @defer.inlineCallbacks - def get_state_for_pdu(self, origin, room_id, event_id): + def get_state_for_pdu(self, origin, room_id, event_id, do_auth=True): yield run_on_reactor() - in_room = yield self.auth.check_host_in_room(room_id, origin) - if not in_room: - raise AuthError(403, "Host not in room.") + if do_auth: + in_room = yield self.auth.check_host_in_room(room_id, origin) + if not in_room: + raise AuthError(403, "Host not in room.") state_groups = yield self.store.get_state_groups( [event_id] diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 3fbc09022..22bf7ad83 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,15 +32,15 @@ class EventFederationStore(SQLBaseStore): and backfilling from another server respectively. """ - def get_auth_chain(self, event_ids): + def get_auth_chain(self, event_ids, have_ids=set()): return self.runInteraction( "get_auth_chain", self._get_auth_chain_txn, - event_ids + event_ids, have_ids ) - def _get_auth_chain_txn(self, txn, event_ids): - results = self._get_auth_chain_ids_txn(txn, event_ids) + def _get_auth_chain_txn(self, txn, event_ids, have_ids): + results = self._get_auth_chain_ids_txn(txn, event_ids, have_ids) return self._get_events_txn(txn, results) @@ -51,8 +51,9 @@ class EventFederationStore(SQLBaseStore): event_ids ) - def _get_auth_chain_ids_txn(self, txn, event_ids): + def _get_auth_chain_ids_txn(self, txn, event_ids, have_ids): results = set() + have_ids = set(have_ids) base_sql = ( "SELECT auth_id FROM event_auth WHERE event_id = ?" @@ -64,6 +65,10 @@ class EventFederationStore(SQLBaseStore): for f in front: txn.execute(base_sql, (f,)) new_front.update([r[0] for r in txn.fetchall()]) + + new_front -= results + new_front -= have_ids + front = new_front results.update(front) @@ -378,3 +383,51 @@ class EventFederationStore(SQLBaseStore): event_results += new_front return self._get_events_txn(txn, event_results) + + def get_missing_events(self, room_id, earliest_events, latest_events, + limit, min_depth): + return self.runInteraction( + "get_missing_events", + self._get_missing_events, + room_id, earliest_events, latest_events, limit, min_depth + ) + + def _get_missing_events(self, txn, room_id, earliest_events, latest_events, + limit, min_depth): + + earliest_events = set(earliest_events) + front = set(latest_events) - earliest_events + + event_results = set() + + query = ( + "SELECT prev_event_id FROM event_edges " + "WHERE room_id = ? AND event_id = ? AND is_state = 0 " + "LIMIT ?" + ) + + while front and len(event_results) < limit: + new_front = set() + for event_id in front: + txn.execute( + query, + (room_id, event_id, limit - len(event_results)) + ) + + for e_id, in txn.fetchall(): + new_front.add(e_id) + + new_front -= earliest_events + new_front -= event_results + + front = new_front + event_results |= new_front + + events = self._get_events_txn(txn, event_results) + + events = sorted( + [ev for ev in events if ev.depth >= min_depth], + key=lambda e: e.depth, + ) + + return events[:limit] From 077d20034278ea57c57d501de11bfb1f0c7f9603 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 17:29:39 +0000 Subject: [PATCH 498/630] Move @cached decorator out into synapse.storage._base; add minimal docs --- synapse/storage/_base.py | 35 +++++++++++++++++++++++++++++++++++ synapse/storage/roommember.py | 28 +--------------------------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index be9934c66..fd275039b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -35,6 +35,41 @@ sql_logger = logging.getLogger("synapse.storage.SQL") transaction_logger = logging.getLogger("synapse.storage.txn") +# TODO(paul): +# * Move this somewhere higher-level, shared; +# * more generic key management +# * export monitoring stats +# * maximum size; just evict things at random, or consider LRU? +def cached(orig): + """ A method decorator that applies a memoizing cache around the function. + + The function is presumed to take one additional argument, which is used as + the key for the cache. Cache hits are served directly from the cache; + misses use the function body to generate the value. + + The wrapped function has an additional member, a callable called + "invalidate". This can be used to remove individual entries from the cache. + """ + cache = {} + + @defer.inlineCallbacks + def wrapped(self, key): + if key in cache: + defer.returnValue(cache[key]) + + ret = yield orig(self, key) + + cache[key] = ret; + defer.returnValue(ret) + + def invalidate(key): + if key in cache: + del cache[key] + + wrapped.invalidate = invalidate + return wrapped + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging to the .execute() method.""" diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 569bd55d0..b8fcc1927 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -17,7 +17,7 @@ from twisted.internet import defer from collections import namedtuple -from ._base import SQLBaseStore +from ._base import SQLBaseStore, cached from synapse.api.constants import Membership from synapse.types import UserID @@ -33,32 +33,6 @@ RoomsForUser = namedtuple( ) -# TODO(paul): -# * Move this somewhere higher-level, shared; -# * more generic key management -# * export monitoring stats -# * maximum size; just evict things at random, or consider LRU? -def cached(orig): - cache = {} - - @defer.inlineCallbacks - def wrapped(self, key): - if key in cache: - defer.returnValue(cache[key]) - - ret = yield orig(self, key) - - cache[key] = ret; - defer.returnValue(ret) - - def invalidate(key): - if key in cache: - del cache[key] - - wrapped.invalidate = invalidate - return wrapped - - class RoomMemberStore(SQLBaseStore): def __init__(self, *args, **kw): From ebc3db295bfe6d0c43bf45b8fcd7fa6bbc429375 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 18:36:02 +0000 Subject: [PATCH 499/630] Take named arguments to @cached() decorator, add a 'max_entries' limit --- synapse/storage/_base.py | 39 +++++++++------ synapse/storage/roommember.py | 2 +- tests/storage/test__base.py | 89 +++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 16 deletions(-) create mode 100644 tests/storage/test__base.py diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index fd275039b..61657d36e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,8 +39,8 @@ transaction_logger = logging.getLogger("synapse.storage.txn") # * Move this somewhere higher-level, shared; # * more generic key management # * export monitoring stats -# * maximum size; just evict things at random, or consider LRU? -def cached(orig): +# * consider other eviction strategies - LRU? +def cached(max_entries=1000): """ A method decorator that applies a memoizing cache around the function. The function is presumed to take one additional argument, which is used as @@ -50,24 +50,33 @@ def cached(orig): The wrapped function has an additional member, a callable called "invalidate". This can be used to remove individual entries from the cache. """ - cache = {} + def wrap(orig): + cache = {} - @defer.inlineCallbacks - def wrapped(self, key): - if key in cache: - defer.returnValue(cache[key]) + @defer.inlineCallbacks + def wrapped(self, key): + if key in cache: + defer.returnValue(cache[key]) - ret = yield orig(self, key) + ret = yield orig(self, key) - cache[key] = ret; - defer.returnValue(ret) + while len(cache) > max_entries: + # TODO(paul): This feels too biased. However, a random index + # would be a bit inefficient, walking the list of keys just + # to ignore most of them? + del cache[cache.keys()[0]] - def invalidate(key): - if key in cache: - del cache[key] + cache[key] = ret; + defer.returnValue(ret) - wrapped.invalidate = invalidate - return wrapped + def invalidate(key): + if key in cache: + del cache[key] + + wrapped.invalidate = invalidate + return wrapped + + return wrap class LoggingTransaction(object): diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index b8fcc1927..33a832483 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -247,7 +247,7 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results - @cached + @cached() def get_rooms_for_user(self, user_id): return self.get_rooms_for_user_where_membership_is( user_id, membership_list=[Membership.JOIN], diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py new file mode 100644 index 000000000..057f79864 --- /dev/null +++ b/tests/storage/test__base.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from synapse.storage._base import cached + + +class CacheDecoratorTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def test_passthrough(self): + @cached() + def func(self, key): + return key + + self.assertEquals((yield func(self, "foo")), "foo") + self.assertEquals((yield func(self, "bar")), "bar") + + @defer.inlineCallbacks + def test_hit(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + yield func(self, "foo") + + self.assertEquals(callcount[0], 1) + + self.assertEquals((yield func(self, "foo")), "foo") + self.assertEquals(callcount[0], 1) + + @defer.inlineCallbacks + def test_invalidate(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + yield func(self, "foo") + + self.assertEquals(callcount[0], 1) + + func.invalidate("foo") + + yield func(self, "foo") + + self.assertEquals(callcount[0], 2) + + @defer.inlineCallbacks + def test_max_entries(self): + callcount = [0] + + @cached(max_entries=10) + def func(self, key): + callcount[0] += 1 + return key + + for k in range(0,12): + yield func(self, k) + + self.assertEquals(callcount[0], 12) + + # There must have been at least 2 evictions, meaning if we calculate + # all 12 values again, we must get called at least 2 more times + for k in range(0,12): + yield func(self, k) + + self.assertTrue(callcount[0] >= 14, + msg="Expected callcount >= 14, got %d" % (callcount[0])) From 55022d6ca5bfe3e99fd3144e291906063885ce12 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 18:38:09 +0000 Subject: [PATCH 500/630] Remove a TODO note --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 61657d36e..78ba5f25e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -36,7 +36,6 @@ transaction_logger = logging.getLogger("synapse.storage.txn") # TODO(paul): -# * Move this somewhere higher-level, shared; # * more generic key management # * export monitoring stats # * consider other eviction strategies - LRU? From 15e2d7e38762efbc9e9c706d4d2a63bb6fcfd541 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 11:39:53 +0000 Subject: [PATCH 501/630] Always allow AS to register --- synapse/rest/client/v1/register.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index dba49129a..f5acfb945 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -98,9 +98,6 @@ class RegisterRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - if self.disable_registration: - raise SynapseError(403, "Registration has been disabled") - register_json = _parse_json(request) session = (register_json["session"] @@ -111,6 +108,11 @@ class RegisterRestServlet(ClientV1RestServlet): try: login_type = register_json["type"] + + is_application_server = login_type == LoginType.APPLICATION_SERVICE + if self.disable_registration and not is_application_server: + raise SynapseError(403, "Registration has been disabled") + stages = { LoginType.RECAPTCHA: self._do_recaptcha, LoginType.PASSWORD: self._do_password, From 2bafeca2703635ea1fee2a7617c30cfad1b00315 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 14:08:42 +0000 Subject: [PATCH 502/630] Add missing comma so that it generates a dict and not a set --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0eb2ff95c..7deed16f9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -875,7 +875,7 @@ class FederationHandler(BaseHandler): local_view = dict(auth_events) remote_view = dict(auth_events) remote_view.update({ - (d.type, d.state_key) for d in different_events + (d.type, d.state_key): d for d in different_events }) new_state, prev_state = self.state_handler.resolve_events( From 0edfecc9043feb294ef09a2d382d7c52d258a809 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 14:14:28 +0000 Subject: [PATCH 503/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index a804a2574..b39b01ac3 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1" +__version__ = "0.7.1.1" From dc0ee55110c1bcb0e3be39a07403e43763c0af25 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 15:00:14 +0000 Subject: [PATCH 504/630] Change version scheme --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index b39b01ac3..f8ec4b045 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1.1" +__version__ = "0.7.1-r1" From 7c56210f204ef735444129c4f7b2a393eb7539ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 16:09:44 +0000 Subject: [PATCH 505/630] By default set soft limit to hard limit --- synapse/app/homeserver.py | 8 ++++++-- synapse/config/server.py | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5f671dfd2..89af091f1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -273,7 +273,12 @@ def get_version_string(): def change_resource_limit(soft_file_no): try: soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + + if not soft_file_no: + soft_file_no = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) + logger.info("Set file limit to: %d", soft_file_no) except (ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) @@ -372,8 +377,7 @@ def setup(): def run(config): with LoggingContext("run"): - if config.soft_file_limit: - change_resource_limit(config.soft_file_limit) + change_resource_limit(config.soft_file_limit) reactor.run() diff --git a/synapse/config/server.py b/synapse/config/server.py index a3edf208e..4e4892d40 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -79,11 +79,11 @@ class ServerConfig(Config): server_group.add_argument("--no-tls", action='store_true', help="Don't bind to the https port.") server_group.add_argument("--soft-file-limit", type=int, default=0, - help="Set the limit on the number of file " - "descriptors synapse can use. Zero " - "is used to indicate synapse should " - "not change the limit from system " - "default.") + help="Set the soft limit on the number of " + "file descriptors synapse can use. " + "Zero is used to indicate synapse " + "should set the soft limit to the hard" + "limit.") def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") From 1f29fafc95af65c259c19c8ef90758328ac9e810 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 21 Feb 2015 13:44:46 +0000 Subject: [PATCH 506/630] Don't exit if we can't work out if we're running in a git repo --- synapse/app/homeserver.py | 99 ++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ea20de143..2c463936c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -210,61 +210,64 @@ class SynapseHomeServer(HomeServer): def get_version_string(): - null = open(os.devnull, 'w') - cwd = os.path.dirname(os.path.abspath(__file__)) try: - git_branch = subprocess.check_output( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], - stderr=null, - cwd=cwd, - ).strip() - git_branch = "b=" + git_branch - except subprocess.CalledProcessError: - git_branch = "" + null = open(os.devnull, 'w') + cwd = os.path.dirname(os.path.abspath(__file__)) + try: + git_branch = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + stderr=null, + cwd=cwd, + ).strip() + git_branch = "b=" + git_branch + except subprocess.CalledProcessError: + git_branch = "" - try: - git_tag = subprocess.check_output( - ['git', 'describe', '--exact-match'], - stderr=null, - cwd=cwd, - ).strip() - git_tag = "t=" + git_tag - except subprocess.CalledProcessError: - git_tag = "" + try: + git_tag = subprocess.check_output( + ['git', 'describe', '--exact-match'], + stderr=null, + cwd=cwd, + ).strip() + git_tag = "t=" + git_tag + except subprocess.CalledProcessError: + git_tag = "" - try: - git_commit = subprocess.check_output( - ['git', 'rev-parse', '--short', 'HEAD'], - stderr=null, - cwd=cwd, - ).strip() - except subprocess.CalledProcessError: - git_commit = "" + try: + git_commit = subprocess.check_output( + ['git', 'rev-parse', '--short', 'HEAD'], + stderr=null, + cwd=cwd, + ).strip() + except subprocess.CalledProcessError: + git_commit = "" - try: - dirty_string = "-this_is_a_dirty_checkout" - is_dirty = subprocess.check_output( - ['git', 'describe', '--dirty=' + dirty_string], - stderr=null, - cwd=cwd, - ).strip().endswith(dirty_string) + try: + dirty_string = "-this_is_a_dirty_checkout" + is_dirty = subprocess.check_output( + ['git', 'describe', '--dirty=' + dirty_string], + stderr=null, + cwd=cwd, + ).strip().endswith(dirty_string) - git_dirty = "dirty" if is_dirty else "" - except subprocess.CalledProcessError: - git_dirty = "" + git_dirty = "dirty" if is_dirty else "" + except subprocess.CalledProcessError: + git_dirty = "" - if git_branch or git_tag or git_commit or git_dirty: - git_version = ",".join( - s for s in - (git_branch, git_tag, git_commit, git_dirty,) - if s - ) - - return ( - "Synapse/%s (%s)" % ( - synapse.__version__, git_version, + if git_branch or git_tag or git_commit or git_dirty: + git_version = ",".join( + s for s in + (git_branch, git_tag, git_commit, git_dirty,) + if s ) - ).encode("ascii") + + return ( + "Synapse/%s (%s)" % ( + synapse.__version__, git_version, + ) + ).encode("ascii") + except Exception as e: + logger.warn("Failed to check for git repository: %s", e) return ("Synapse/%s" % (synapse.__version__,)).encode("ascii") From a87c56c67327fcacc008c169a8f7fc244c6d8016 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 21 Feb 2015 13:45:07 +0000 Subject: [PATCH 507/630] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index f8ec4b045..7d548b0a8 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.7.1-r1" +__version__ = "0.7.1-r2" From 596daf6e68eefcce6f868157be6dfb22a51b82d6 Mon Sep 17 00:00:00 2001 From: brabo Date: Sun, 22 Feb 2015 18:52:59 +0100 Subject: [PATCH 508/630] added "cd ~/.synapse" before setup of the homeserver to generate our files in there instead of ~ --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index d3778b9f2..c2af7c933 100644 --- a/README.rst +++ b/README.rst @@ -118,6 +118,7 @@ environment under ``~/.synapse``. To set up your homeserver, run (in your virtualenv, as before):: + $ cd ~/.synapse $ python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ --config-path homeserver.yaml \ @@ -179,6 +180,7 @@ installing under virtualenv):: During setup of homeserver you need to call python2.7 directly again:: + $ cd ~/.synapse $ python2.7 -m synapse.app.homeserver \ --server-name machine.my.domain.name \ --config-path homeserver.yaml \ From db215b7e0007a207b8775d78c6693153e16f2731 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Feb 2015 13:58:02 +0000 Subject: [PATCH 509/630] Implement and use new batched get missing pdu --- synapse/federation/federation_client.py | 19 +++ synapse/federation/federation_server.py | 148 ++++++++---------------- synapse/federation/transaction_queue.py | 2 +- synapse/federation/transport/client.py | 19 +++ synapse/federation/transport/server.py | 31 +++++ synapse/handlers/federation.py | 23 ++++ 6 files changed, 143 insertions(+), 99 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index cd3c962d5..ca89a0787 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -439,6 +439,25 @@ class FederationClient(FederationBase): defer.returnValue(ret) + @defer.inlineCallbacks + def get_missing_events(self, destination, room_id, earliest_events, + latest_events, limit, min_depth): + content = yield self.transport_layer.get_missing_events( + destination, room_id, earliest_events, latest_events, limit, + min_depth, + ) + + events = [ + self.event_from_pdu_json(e) + for e in content.get("events", []) + ] + + signed_events = yield self._check_sigs_and_hash_and_fetch( + destination, events, outlier=True + ) + + defer.returnValue(signed_events) + def event_from_pdu_json(self, pdu_json, outlier=False): event = FrozenEvent( pdu_json diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 34bc397e8..f74e16abd 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -142,7 +142,15 @@ class FederationServer(FederationBase): if r[0]: ret.append({}) else: - logger.exception(r[1]) + failure = r[1] + logger.error( + "Failed to handle PDU", + exc_info=( + failure.type, + failure.value, + failure.getTracebackObject() + ) + ) ret.append({"error": str(r[1].value)}) logger.debug("Returning: %s", str(ret)) @@ -306,75 +314,17 @@ class FederationServer(FederationBase): ) @defer.inlineCallbacks - def get_missing_events(self, origin, room_id, earliest_events, - latest_events, limit, min_depth): - limit = max(limit, 50) - min_depth = max(min_depth, 0) - - missing_events = yield self.store.get_missing_events( - room_id=room_id, - earliest_events=earliest_events, - latest_events=latest_events, - limit=limit, - min_depth=min_depth, + @log_function + def on_get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + missing_events = yield self.handler.on_get_missing_events( + origin, room_id, earliest_events, latest_events, limit, min_depth ) - known_ids = {e.event_id for e in missing_events} | {earliest_events} - - back_edges = { - e for e in missing_events - if {i for i, h in e.prev_events.items()} <= known_ids - } - - decoded_auth_events = set() - state = {} - auth_events = set() - auth_and_state = {} - for event in back_edges: - state_pdus = yield self.handler.get_state_for_pdu( - origin, room_id, event.event_id, - do_auth=False, - ) - - state[event.event_id] = [s.event_id for s in state_pdus] - - auth_and_state.update({ - s.event_id: s for s in state_pdus - }) - - state_ids = {pdu.event_id for pdu in state_pdus} - prev_ids = {i for i, h in event.prev_events.items()} - partial_auth_chain = yield self.store.get_auth_chain( - state_ids | prev_ids, have_ids=decoded_auth_events.keys() - ) - - for p in partial_auth_chain: - p.signatures.update( - compute_event_signature( - p, - self.hs.hostname, - self.hs.config.signing_key[0] - ) - ) - - auth_events.update( - a.event_id for a in partial_auth_chain - ) - - auth_and_state.update({ - a.event_id: a for a in partial_auth_chain - }) - time_now = self._clock.time_msec() defer.returnValue({ "events": [ev.get_pdu_json(time_now) for ev in missing_events], - "state_for_events": state, - "auth_events": auth_events, - "event_map": { - k: ev.get_pdu_json(time_now) - for k, ev in auth_and_state.items() - }, }) @log_function @@ -403,7 +353,7 @@ class FederationServer(FederationBase): @defer.inlineCallbacks @log_function - def _handle_new_pdu(self, origin, pdu, max_recursion=10): + def _handle_new_pdu(self, origin, pdu, get_missing=True): # We reprocess pdus when we have seen them only as outliers existing = yield self._get_persisted_pdu( origin, pdu.event_id, do_auth=False @@ -455,48 +405,50 @@ class FederationServer(FederationBase): pdu.room_id, min_depth ) + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if min_depth and pdu.depth < min_depth: # This is so that we don't notify the user about this # message, to work around the fact that some events will # reference really really old events we really don't want to # send to the clients. pdu.internal_metadata.outlier = True - elif min_depth and pdu.depth > min_depth and max_recursion > 0: - for event_id, hashes in pdu.prev_events: - if event_id not in have_seen: - logger.debug( - "_handle_new_pdu requesting pdu %s", - event_id + elif min_depth and pdu.depth > min_depth: + if get_missing and prevs - seen: + latest_tuples = yield self.store.get_latest_events_in_room( + pdu.room_id + ) + + # We add the prev events that we have seen to the latest + # list to ensure the remote server doesn't give them to us + latest = set(e_id for e_id, _, _ in latest_tuples) + latest |= seen + + missing_events = yield self.get_missing_events( + origin, + pdu.room_id, + earliest_events=list(latest), + latest_events=[pdu.event_id], + limit=10, + min_depth=min_depth, + ) + + for e in missing_events: + yield self._handle_new_pdu( + origin, + e, + get_missing=False ) - try: - new_pdu = yield self.federation_client.get_pdu( - [origin, pdu.origin], - event_id=event_id, - ) + have_seen = yield self.store.have_events( + [ev for ev, _ in pdu.prev_events] + ) - if new_pdu: - yield self._handle_new_pdu( - origin, - new_pdu, - max_recursion=max_recursion-1 - ) - - logger.debug("Processed pdu %s", event_id) - else: - logger.warn("Failed to get PDU %s", event_id) - fetch_state = True - except: - # TODO(erikj): Do some more intelligent retries. - logger.exception("Failed to get PDU") - fetch_state = True - else: - prevs = {e_id for e_id, _ in pdu.prev_events} - seen = set(have_seen.keys()) - if prevs - seen: - fetch_state = True - else: - fetch_state = True + prevs = {e_id for e_id, _ in pdu.prev_events} + seen = set(have_seen.keys()) + if prevs - seen: + fetch_state = True if fetch_state: # We need to get the state at this event, since we haven't diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 7d30c924d..8f1acbe59 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -287,7 +287,7 @@ class TransactionQueue(object): code = 200 if response: - for e_id, r in getattr(response, "pdus", {}).items(): + for e_id, r in response.get("pdus", {}).items(): if "error" in r: logger.warn( "Transaction returned error for %s: %s", diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 8b137e712..80d03012b 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -219,3 +219,22 @@ class TransportLayerClient(object): ) defer.returnValue(content) + + @defer.inlineCallbacks + @log_function + def get_missing_events(self, destination, room_id, earliest_events, + latest_events, limit, min_depth): + path = PREFIX + "/get_missing_events/%s" % (room_id,) + + content = yield self.client.post_json( + destination=destination, + path=path, + data={ + "limit": int(limit), + "min_depth": int(min_depth), + "earliest_events": earliest_events, + "latest_events": latest_events, + } + ) + + defer.returnValue(content) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 2ffb37aa1..ad75c8ddb 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -234,6 +234,7 @@ class TransportLayerServer(object): ) ) ) + self.server.register_path( "POST", re.compile("^" + PREFIX + "/query_auth/([^/]*)/([^/]*)$"), @@ -245,6 +246,17 @@ class TransportLayerServer(object): ) ) + self.server.register_path( + "POST", + re.compile("^" + PREFIX + "/get_missing_events/([^/]*)/?$"), + self._with_authentication( + lambda origin, content, query, room_id: + self._get_missing_events( + origin, content, room_id, + ) + ) + ) + @defer.inlineCallbacks @log_function def _on_send_request(self, origin, content, query, transaction_id): @@ -344,3 +356,22 @@ class TransportLayerServer(object): ) defer.returnValue((200, new_content)) + + @defer.inlineCallbacks + @log_function + def _get_missing_events(self, origin, content, room_id): + limit = int(content.get("limit", 10)) + min_depth = int(content.get("min_depth", 0)) + earliest_events = content.get("earliest_events", []) + latest_events = content.get("latest_events", []) + + content = yield self.request_handler.on_get_missing_events( + origin, + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + min_depth=min_depth, + limit=limit, + ) + + defer.returnValue((200, content)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 26bdc6d1a..628e62f8b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -789,6 +789,29 @@ class FederationHandler(BaseHandler): defer.returnValue(ret) + @defer.inlineCallbacks + def on_get_missing_events(self, origin, room_id, earliest_events, + latest_events, limit, min_depth): + in_room = yield self.auth.check_host_in_room( + room_id, + origin + ) + if not in_room: + raise AuthError(403, "Host not in room.") + + limit = min(limit, 20) + min_depth = max(min_depth, 0) + + missing_events = yield self.store.get_missing_events( + room_id=room_id, + earliest_events=earliest_events, + latest_events=latest_events, + limit=limit, + min_depth=min_depth, + ) + + defer.returnValue(missing_events) + @defer.inlineCallbacks @log_function def do_auth(self, origin, event, context, auth_events): From 42b972bccd0cf7d903befb498f9c1bbd5c4e6583 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Feb 2015 14:35:23 +0000 Subject: [PATCH 510/630] Revert get_auth_chain changes --- synapse/storage/event_federation.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 22bf7ad83..2deda8ac5 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,15 +32,15 @@ class EventFederationStore(SQLBaseStore): and backfilling from another server respectively. """ - def get_auth_chain(self, event_ids, have_ids=set()): + def get_auth_chain(self, event_ids): return self.runInteraction( "get_auth_chain", self._get_auth_chain_txn, - event_ids, have_ids + event_ids ) - def _get_auth_chain_txn(self, txn, event_ids, have_ids): - results = self._get_auth_chain_ids_txn(txn, event_ids, have_ids) + def _get_auth_chain_txn(self, txn, event_ids): + results = self._get_auth_chain_ids_txn(txn, event_ids) return self._get_events_txn(txn, results) @@ -51,9 +51,8 @@ class EventFederationStore(SQLBaseStore): event_ids ) - def _get_auth_chain_ids_txn(self, txn, event_ids, have_ids): + def _get_auth_chain_ids_txn(self, txn, event_ids): results = set() - have_ids = set(have_ids) base_sql = ( "SELECT auth_id FROM event_auth WHERE event_id = ?" @@ -67,7 +66,6 @@ class EventFederationStore(SQLBaseStore): new_front.update([r[0] for r in txn.fetchall()]) new_front -= results - new_front -= have_ids front = new_front results.update(front) From 4631b737fdb08185d514e69f0e6860c0860768b5 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 14:38:44 +0000 Subject: [PATCH 511/630] Squash out the now-redundant ApplicationServicesCache object class --- synapse/storage/appservice.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index d941b1f38..dc3666efd 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -23,23 +23,11 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) -class ApplicationServiceCache(object): - """Caches ApplicationServices and provides utility functions on top. - - This class is designed to be invoked on incoming events in order to avoid - hammering the database every time to extract a list of application service - regexes. - """ - - def __init__(self): - self.services = [] - - class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) - self.cache = ApplicationServiceCache() + self.services_cache = [] self.cache_defer = self._populate_cache() @defer.inlineCallbacks @@ -56,7 +44,7 @@ class ApplicationServiceStore(SQLBaseStore): token, ) # update cache TODO: Should this be in the txn? - for service in self.cache.services: + for service in self.services_cache: if service.token == token: service.url = None service.namespaces = None @@ -110,13 +98,13 @@ class ApplicationServiceStore(SQLBaseStore): ) # update cache TODO: Should this be in the txn? - for (index, cache_service) in enumerate(self.cache.services): + for (index, cache_service) in enumerate(self.services_cache): if service.token == cache_service.token: - self.cache.services[index] = service + self.services_cache[index] = service logger.info("Updated: %s", service) return # new entry - self.cache.services.append(service) + self.services_cache.append(service) logger.info("Updated(new): %s", service) def _update_app_service_txn(self, txn, service): @@ -160,7 +148,7 @@ class ApplicationServiceStore(SQLBaseStore): @defer.inlineCallbacks def get_app_services(self): yield self.cache_defer # make sure the cache is ready - defer.returnValue(self.cache.services) + defer.returnValue(self.services_cache) @defer.inlineCallbacks def get_app_service_by_token(self, token, from_cache=True): @@ -176,7 +164,7 @@ class ApplicationServiceStore(SQLBaseStore): yield self.cache_defer # make sure the cache is ready if from_cache: - for service in self.cache.services: + for service in self.services_cache: if service.token == token: defer.returnValue(service) return @@ -235,7 +223,7 @@ class ApplicationServiceStore(SQLBaseStore): # TODO get last successful txn id f.e. service for service in services.values(): logger.info("Found application service: %s", service) - self.cache.services.append(ApplicationService( + self.services_cache.append(ApplicationService( token=service["token"], url=service["url"], namespaces=service["namespaces"], From 852816befe678f0061dc153fab20e1d4c70a9094 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Feb 2015 15:14:09 +0000 Subject: [PATCH 512/630] Fix presence tests --- tests/handlers/test_presence.py | 43 +++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index d88a977be..6ffc3c99c 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -389,14 +389,18 @@ class PresenceInvitesTestCase(PresenceTestCase): @defer.inlineCallbacks def test_invite_remote(self): + # Use a different destination, otherwise retry logic might fail the + # request + u_rocket = UserID.from_string("@rocket:there") + put_json = self.mock_http_client.put_json put_json.expect_call_and_return( - call("elsewhere", + call("there", path="/_matrix/federation/v1/send/1000000/", - data=_expect_edu("elsewhere", "m.presence_invite", + data=_expect_edu("there", "m.presence_invite", content={ "observer_user": "@apple:test", - "observed_user": "@cabbage:elsewhere", + "observed_user": "@rocket:there", } ), json_data_callback=ANY, @@ -405,10 +409,10 @@ class PresenceInvitesTestCase(PresenceTestCase): ) yield self.handler.send_invite( - observer_user=self.u_apple, observed_user=self.u_cabbage) + observer_user=self.u_apple, observed_user=u_rocket) self.assertEquals( - [{"observed_user_id": "@cabbage:elsewhere", "accepted": 0}], + [{"observed_user_id": "@rocket:there", "accepted": 0}], (yield self.datastore.get_presence_list(self.u_apple.localpart)) ) @@ -418,13 +422,18 @@ class PresenceInvitesTestCase(PresenceTestCase): def test_accept_remote(self): # TODO(paul): This test will likely break if/when real auth permissions # are added; for now the HS will always accept any invite + + # Use a different destination, otherwise retry logic might fail the + # request + u_rocket = UserID.from_string("@rocket:moon") + put_json = self.mock_http_client.put_json put_json.expect_call_and_return( - call("elsewhere", + call("moon", path="/_matrix/federation/v1/send/1000000/", - data=_expect_edu("elsewhere", "m.presence_accept", + data=_expect_edu("moon", "m.presence_accept", content={ - "observer_user": "@cabbage:elsewhere", + "observer_user": "@rocket:moon", "observed_user": "@apple:test", } ), @@ -437,7 +446,7 @@ class PresenceInvitesTestCase(PresenceTestCase): "/_matrix/federation/v1/send/1000000/", _make_edu_json("elsewhere", "m.presence_invite", content={ - "observer_user": "@cabbage:elsewhere", + "observer_user": "@rocket:moon", "observed_user": "@apple:test", } ) @@ -446,7 +455,7 @@ class PresenceInvitesTestCase(PresenceTestCase): self.assertTrue( (yield self.datastore.is_presence_visible( observed_localpart=self.u_apple.localpart, - observer_userid=self.u_cabbage.to_string(), + observer_userid=u_rocket.to_string(), )) ) @@ -454,13 +463,17 @@ class PresenceInvitesTestCase(PresenceTestCase): @defer.inlineCallbacks def test_invited_remote_nonexistant(self): + # Use a different destination, otherwise retry logic might fail the + # request + u_rocket = UserID.from_string("@rocket:sun") + put_json = self.mock_http_client.put_json put_json.expect_call_and_return( - call("elsewhere", + call("sun", path="/_matrix/federation/v1/send/1000000/", - data=_expect_edu("elsewhere", "m.presence_deny", + data=_expect_edu("sun", "m.presence_deny", content={ - "observer_user": "@cabbage:elsewhere", + "observer_user": "@rocket:sun", "observed_user": "@durian:test", } ), @@ -471,9 +484,9 @@ class PresenceInvitesTestCase(PresenceTestCase): yield self.mock_federation_resource.trigger("PUT", "/_matrix/federation/v1/send/1000000/", - _make_edu_json("elsewhere", "m.presence_invite", + _make_edu_json("sun", "m.presence_invite", content={ - "observer_user": "@cabbage:elsewhere", + "observer_user": "@rocket:sun", "observed_user": "@durian:test", } ) From 22399d3d8f5931c4302a9df37ebde94f5d1fc277 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 23 Feb 2015 15:14:56 +0000 Subject: [PATCH 513/630] Add RegisterFallbackResource to /_matrix/static/client/register Try to keep both forms of registration logic (native/fallback) close together for sanity. --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 2 ++ synapse/rest/client/v1/__init__.py | 8 ++++++++ synapse/rest/client/v1/register.py | 21 +++++++++++++++++++++ 4 files changed, 32 insertions(+) diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 948571933..3d4367462 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -18,6 +18,7 @@ CLIENT_PREFIX = "/_matrix/client/api/v1" CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" FEDERATION_PREFIX = "/_matrix/federation/v1" +STATIC_PREFIX = "/_matrix/static" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ea20de143..137293bd6 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -125,6 +125,8 @@ class SynapseHomeServer(HomeServer): (MEDIA_PREFIX, self.get_resource_for_media_repository()), (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), ] + desired_tree += self.get_resource_for_client().get_extra_resources(self) + if web_client: logger.info("Adding the web client.") desired_tree.append((WEB_CLIENT_PREFIX, diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 21876b348..1ebdeadca 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -28,6 +28,14 @@ class ClientV1RestResource(JsonResource): JsonResource.__init__(self, hs) self.register_servlets(self, hs) + def get_extra_resources(self, hs): + # some parts of client v1 need to produce HTML as the output (e.g. + # fallback pages) but we can only produce JSON output. In an effort to + # keep similar logic close together, we'll call through to any servlet + # which requires HTML output. + register_resources = register.get_prefixes_and_resources(hs) + return register_resources + @staticmethod def register_servlets(client_resource, hs): room.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index f5acfb945..eee567a58 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -18,10 +18,12 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.api.constants import LoginType +from synapse.api.urls import STATIC_PREFIX from base import ClientV1RestServlet, client_path_pattern import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor +from twisted.web.resource import Resource from hashlib import sha1 import hmac @@ -305,6 +307,16 @@ class RegisterRestServlet(ClientV1RestServlet): }) +class RegisterFallbackResource(Resource): + + def __init__(self, hs): + Resource.__init__(self) # Resource is an old-style class :( + self.hs = hs + + def render_GET(self, request): + return "NOT_YET_IMPLEMENTED" + + def _parse_json(request): try: content = json.loads(request.content.read()) @@ -315,5 +327,14 @@ def _parse_json(request): raise SynapseError(400, "Content not JSON.") +def get_prefixes_and_resources(hs): + return [ + ( + STATIC_PREFIX + "/client/register", + RegisterFallbackResource(hs) + ) + ] + + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) From 0696dfd94b5d32679009368508d26ae1a6630994 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 23 Feb 2015 15:35:09 +0000 Subject: [PATCH 514/630] Actually treat this as static content, not random Resources. --- synapse/app/homeserver.py | 8 ++++++-- synapse/rest/client/v1/__init__.py | 8 -------- synapse/rest/client/v1/register.py | 21 --------------------- synapse/server.py | 1 + 4 files changed, 7 insertions(+), 31 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 137293bd6..688bf8d96 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -36,7 +36,8 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, APP_SERVICE_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, APP_SERVICE_PREFIX, + STATIC_PREFIX ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -81,6 +82,9 @@ class SynapseHomeServer(HomeServer): webclient_path = os.path.join(syweb_path, "webclient") return File(webclient_path) # TODO configurable? + def build_resource_for_static_content(self): + return File("static") + def build_resource_for_content_repo(self): return ContentRepoResource( self, self.upload_dir, self.auth, self.content_addr @@ -124,8 +128,8 @@ class SynapseHomeServer(HomeServer): (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), + (STATIC_PREFIX, self.get_resource_for_static_content()) ] - desired_tree += self.get_resource_for_client().get_extra_resources(self) if web_client: logger.info("Adding the web client.") diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 1ebdeadca..21876b348 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -28,14 +28,6 @@ class ClientV1RestResource(JsonResource): JsonResource.__init__(self, hs) self.register_servlets(self, hs) - def get_extra_resources(self, hs): - # some parts of client v1 need to produce HTML as the output (e.g. - # fallback pages) but we can only produce JSON output. In an effort to - # keep similar logic close together, we'll call through to any servlet - # which requires HTML output. - register_resources = register.get_prefixes_and_resources(hs) - return register_resources - @staticmethod def register_servlets(client_resource, hs): room.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index eee567a58..f5acfb945 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -18,12 +18,10 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, Codes from synapse.api.constants import LoginType -from synapse.api.urls import STATIC_PREFIX from base import ClientV1RestServlet, client_path_pattern import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor -from twisted.web.resource import Resource from hashlib import sha1 import hmac @@ -307,16 +305,6 @@ class RegisterRestServlet(ClientV1RestServlet): }) -class RegisterFallbackResource(Resource): - - def __init__(self, hs): - Resource.__init__(self) # Resource is an old-style class :( - self.hs = hs - - def render_GET(self, request): - return "NOT_YET_IMPLEMENTED" - - def _parse_json(request): try: content = json.loads(request.content.read()) @@ -327,14 +315,5 @@ def _parse_json(request): raise SynapseError(400, "Content not JSON.") -def get_prefixes_and_resources(hs): - return [ - ( - STATIC_PREFIX + "/client/register", - RegisterFallbackResource(hs) - ) - ] - - def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index ba2b2593f..cb8610a1b 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -73,6 +73,7 @@ class BaseHomeServer(object): 'resource_for_client', 'resource_for_client_v2_alpha', 'resource_for_federation', + 'resource_for_static_content', 'resource_for_web_client', 'resource_for_content_repo', 'resource_for_server_key', From e76d485e29498fce7412423e7a5b6ac6bc287ec3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 15:41:54 +0000 Subject: [PATCH 515/630] Allow @cached-wrapped functions to have a prefill method for setting entries --- synapse/storage/_base.py | 23 +++++++++++++++-------- tests/storage/test__base.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 78ba5f25e..4b1ec687c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -48,24 +48,30 @@ def cached(max_entries=1000): The wrapped function has an additional member, a callable called "invalidate". This can be used to remove individual entries from the cache. + + The wrapped function has another additional callable, called "prefill", + which can be used to insert values into the cache specifically, without + calling the calculation function. """ def wrap(orig): cache = {} + def prefill(key, value): + while len(cache) > max_entries: + # TODO(paul): This feels too biased. However, a random index + # would be a bit inefficient, walking the list of keys just + # to ignore most of them? + del cache[cache.keys()[0]] + + cache[key] = value + @defer.inlineCallbacks def wrapped(self, key): if key in cache: defer.returnValue(cache[key]) ret = yield orig(self, key) - - while len(cache) > max_entries: - # TODO(paul): This feels too biased. However, a random index - # would be a bit inefficient, walking the list of keys just - # to ignore most of them? - del cache[cache.keys()[0]] - - cache[key] = ret; + prefill(key, ret) defer.returnValue(ret) def invalidate(key): @@ -73,6 +79,7 @@ def cached(max_entries=1000): del cache[key] wrapped.invalidate = invalidate + wrapped.prefill = prefill return wrapped return wrap diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 057f79864..fb306cb78 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -87,3 +87,17 @@ class CacheDecoratorTestCase(unittest.TestCase): self.assertTrue(callcount[0] >= 14, msg="Expected callcount >= 14, got %d" % (callcount[0])) + + @defer.inlineCallbacks + def test_prefill(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + func.prefill("foo", 123) + + self.assertEquals((yield func(self, "foo")), 123) + self.assertEquals(callcount[0], 0) From 357fba2c24067796ce89f25636a2541bc9a10752 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 15:57:41 +0000 Subject: [PATCH 516/630] RoomMemberStore no longer needs a _user_rooms_cache member --- synapse/storage/roommember.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 33a832483..58aa376c2 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,11 +35,6 @@ RoomsForUser = namedtuple( class RoomMemberStore(SQLBaseStore): - def __init__(self, *args, **kw): - super(RoomMemberStore, self).__init__(*args, **kw) - - self._user_rooms_cache = {} - def _store_room_member_txn(self, txn, event): """Store a room member in the database. """ From 044d813ef7bb67e7b680087d82f931e4c780218f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 16:04:40 +0000 Subject: [PATCH 517/630] Use the @cached decorator to implement the destination_retry_timings cache --- synapse/storage/transactions.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index e06ef3569..6cac8d01a 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, Table +from ._base import SQLBaseStore, Table, cached from collections import namedtuple @@ -28,10 +28,6 @@ class TransactionStore(SQLBaseStore): """A collection of queries for handling PDUs. """ - # a write-through cache of DestinationsTable.EntryType indexed by - # destination string - destination_retry_cache = {} - def get_received_txn_response(self, transaction_id, origin): """For an incoming transaction from a given origin, check if we have already responded to it. If so, return the response code and response @@ -211,6 +207,7 @@ class TransactionStore(SQLBaseStore): return ReceivedTransactionsTable.decode_results(txn.fetchall()) + @cached() def get_destination_retry_timings(self, destination): """Gets the current retry timings (if any) for a given destination. @@ -221,9 +218,6 @@ class TransactionStore(SQLBaseStore): None if not retrying Otherwise a DestinationsTable.EntryType for the retry scheme """ - if destination in self.destination_retry_cache: - return defer.succeed(self.destination_retry_cache[destination]) - return self.runInteraction( "get_destination_retry_timings", self._get_destination_retry_timings, destination) @@ -250,7 +244,9 @@ class TransactionStore(SQLBaseStore): retry_interval (int) - how long until next retry in ms """ - self.destination_retry_cache[destination] = ( + # As this is the new value, we might as well prefill the cache + self.get_destination_retry_timings.prefill( + destination, DestinationsTable.EntryType( destination, retry_last_ts, From e6363857d04b5ae4594d5e3358763a6f3706cfc1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 23 Feb 2015 16:08:43 +0000 Subject: [PATCH 518/630] Add core registration html/js --- .gitignore | 1 + static/client/register/index.html | 35 ++++ static/client/register/js/recaptcha_ajax.js | 195 ++++++++++++++++++ static/client/register/js/register.js | 23 +++ .../client/register/register_config.sample.js | 3 + static/client/register/style.css | 52 +++++ 6 files changed, 309 insertions(+) create mode 100644 static/client/register/index.html create mode 100644 static/client/register/js/recaptcha_ajax.js create mode 100644 static/client/register/js/register.js create mode 100644 static/client/register/register_config.sample.js create mode 100644 static/client/register/style.css diff --git a/.gitignore b/.gitignore index 3899c5a25..960183a79 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ media_store/ build/ localhost-800*/ +static/client/register/register_config.js diff --git a/static/client/register/index.html b/static/client/register/index.html new file mode 100644 index 000000000..2248d6e5a --- /dev/null +++ b/static/client/register/index.html @@ -0,0 +1,35 @@ + + + Registration + + + + + + +
+
+ Create account:
+ +
+ +
+ +
+ +
+ +
+ + +
+ +
+ + +
Your home server stores all your conversation and account data.
+
+
+ + + diff --git a/static/client/register/js/recaptcha_ajax.js b/static/client/register/js/recaptcha_ajax.js new file mode 100644 index 000000000..d0e71e5b8 --- /dev/null +++ b/static/client/register/js/recaptcha_ajax.js @@ -0,0 +1,195 @@ +(function(){var h,k=this,l=function(a){return void 0!==a},ba=function(){},n=function(a){var b=typeof a;if("object"==b)if(a){if(a instanceof Array)return"array";if(a instanceof Object)return b;var c=Object.prototype.toString.call(a);if("[object Window]"==c)return"object";if("[object Array]"==c||"number"==typeof a.length&&"undefined"!=typeof a.splice&&"undefined"!=typeof a.propertyIsEnumerable&&!a.propertyIsEnumerable("splice"))return"array";if("[object Function]"==c||"undefined"!=typeof a.call&&"undefined"!=typeof a.propertyIsEnumerable&& +!a.propertyIsEnumerable("call"))return"function"}else return"null";else if("function"==b&&"undefined"==typeof a.call)return"object";return b},p=function(a){return"array"==n(a)},ca=function(a){var b=n(a);return"array"==b||"object"==b&&"number"==typeof a.length},q=function(a){return"string"==typeof a},r=function(a){return"function"==n(a)},da=function(a){var b=typeof a;return"object"==b&&null!=a||"function"==b},ea=function(a,b,c){return a.call.apply(a.bind,arguments)},fa=function(a,b,c){if(!a)throw Error(); +if(2
';v.CleanCss=".recaptchatable td img{display:block}.recaptchatable .recaptcha_image_cell center img{height:57px}.recaptchatable .recaptcha_image_cell center{height:57px}.recaptchatable .recaptcha_image_cell{background-color:white;height:57px;padding:7px!important}.recaptchatable,#recaptcha_area tr,#recaptcha_area td,#recaptcha_area th{margin:0!important;border:0!important;border-collapse:collapse!important;vertical-align:middle!important}.recaptchatable *{margin:0;padding:0;border:0;color:black;position:static;top:auto;left:auto;right:auto;bottom:auto}.recaptchatable #recaptcha_image{position:relative;margin:auto;border:1px solid #dfdfdf!important}.recaptchatable #recaptcha_image #recaptcha_challenge_image{display:block}.recaptchatable #recaptcha_image #recaptcha_ad_image{display:block;position:absolute;top:0}.recaptchatable a img{border:0}.recaptchatable a,.recaptchatable a:hover{cursor:pointer;outline:none;border:0!important;padding:0!important;text-decoration:none;color:blue;background:none!important;font-weight:normal}.recaptcha_input_area{position:relative!important;background:none!important}.recaptchatable label.recaptcha_input_area_text{border:1px solid #dfdfdf!important;margin:0!important;padding:0!important;position:static!important;top:auto!important;left:auto!important;right:auto!important;bottom:auto!important}.recaptcha_theme_red label.recaptcha_input_area_text,.recaptcha_theme_white label.recaptcha_input_area_text{color:black!important}.recaptcha_theme_blackglass label.recaptcha_input_area_text{color:white!important}.recaptchatable #recaptcha_response_field{font-size:11pt}.recaptcha_theme_blackglass #recaptcha_response_field,.recaptcha_theme_white #recaptcha_response_field{border:1px solid gray}.recaptcha_theme_red #recaptcha_response_field{border:1px solid #cca940}.recaptcha_audio_cant_hear_link{font-size:7pt;color:black}.recaptchatable{line-height:1em;border:1px solid #dfdfdf!important}.recaptcha_error_text{color:red}.recaptcha_only_if_privacy{float:right;text-align:right;margin-right:7px}#recaptcha-ad-choices{position:absolute;height:15px;top:0;right:0}#recaptcha-ad-choices img{height:15px}.recaptcha-ad-choices-collapsed{width:30px;height:15px;display:block}.recaptcha-ad-choices-expanded{width:75px;height:15px;display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-collapsed{display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-expanded{display:block}";v.CleanHtml='
';v.VertCss=".recaptchatable td img{display:block}.recaptchatable .recaptcha_r1_c1{background:url('IMGROOT/sprite.png') 0 -63px no-repeat;width:318px;height:9px}.recaptchatable .recaptcha_r2_c1{background:url('IMGROOT/sprite.png') -18px 0 no-repeat;width:9px;height:57px}.recaptchatable .recaptcha_r2_c2{background:url('IMGROOT/sprite.png') -27px 0 no-repeat;width:9px;height:57px}.recaptchatable .recaptcha_r3_c1{background:url('IMGROOT/sprite.png') 0 0 no-repeat;width:9px;height:63px}.recaptchatable .recaptcha_r3_c2{background:url('IMGROOT/sprite.png') -18px -57px no-repeat;width:300px;height:6px}.recaptchatable .recaptcha_r3_c3{background:url('IMGROOT/sprite.png') -9px 0 no-repeat;width:9px;height:63px}.recaptchatable .recaptcha_r4_c1{background:url('IMGROOT/sprite.png') -43px 0 no-repeat;width:171px;height:49px}.recaptchatable .recaptcha_r4_c2{background:url('IMGROOT/sprite.png') -36px 0 no-repeat;width:7px;height:57px}.recaptchatable .recaptcha_r4_c4{background:url('IMGROOT/sprite.png') -214px 0 no-repeat;width:97px;height:57px}.recaptchatable .recaptcha_r7_c1{background:url('IMGROOT/sprite.png') -43px -49px no-repeat;width:171px;height:8px}.recaptchatable .recaptcha_r8_c1{background:url('IMGROOT/sprite.png') -43px -49px no-repeat;width:25px;height:8px}.recaptchatable .recaptcha_image_cell center img{height:57px}.recaptchatable .recaptcha_image_cell center{height:57px}.recaptchatable .recaptcha_image_cell{background-color:white;height:57px}#recaptcha_area,#recaptcha_table{width:318px!important}.recaptchatable,#recaptcha_area tr,#recaptcha_area td,#recaptcha_area th{margin:0!important;border:0!important;padding:0!important;border-collapse:collapse!important;vertical-align:middle!important}.recaptchatable *{margin:0;padding:0;border:0;font-family:helvetica,sans-serif;font-size:8pt;color:black;position:static;top:auto;left:auto;right:auto;bottom:auto}.recaptchatable #recaptcha_image{position:relative;margin:auto}.recaptchatable #recaptcha_image #recaptcha_challenge_image{display:block}.recaptchatable #recaptcha_image #recaptcha_ad_image{display:block;position:absolute;top:0}.recaptchatable img{border:0!important;margin:0!important;padding:0!important}.recaptchatable a,.recaptchatable a:hover{cursor:pointer;outline:none;border:0!important;padding:0!important;text-decoration:none;color:blue;background:none!important;font-weight:normal}.recaptcha_input_area{position:relative!important;width:153px!important;height:45px!important;margin-left:7px!important;margin-right:7px!important;background:none!important}.recaptchatable label.recaptcha_input_area_text{margin:0!important;padding:0!important;position:static!important;top:auto!important;left:auto!important;right:auto!important;bottom:auto!important;background:none!important;height:auto!important;width:auto!important}.recaptcha_theme_red label.recaptcha_input_area_text,.recaptcha_theme_white label.recaptcha_input_area_text{color:black!important}.recaptcha_theme_blackglass label.recaptcha_input_area_text{color:white!important}.recaptchatable #recaptcha_response_field{width:153px!important;position:relative!important;bottom:7px!important;padding:0!important;margin:15px 0 0 0!important;font-size:10pt}.recaptcha_theme_blackglass #recaptcha_response_field,.recaptcha_theme_white #recaptcha_response_field{border:1px solid gray}.recaptcha_theme_red #recaptcha_response_field{border:1px solid #cca940}.recaptcha_audio_cant_hear_link{font-size:7pt;color:black}.recaptchatable{line-height:1!important}#recaptcha_instructions_error{color:red!important}.recaptcha_only_if_privacy{float:right;text-align:right}#recaptcha-ad-choices{position:absolute;height:15px;top:0;right:0}#recaptcha-ad-choices img{height:15px}.recaptcha-ad-choices-collapsed{width:30px;height:15px;display:block}.recaptcha-ad-choices-expanded{width:75px;height:15px;display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-collapsed{display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-expanded{display:block}";var w={visual_challenge:"Get a visual challenge",audio_challenge:"Get an audio challenge",refresh_btn:"Get a new challenge",instructions_visual:"Type the text:",instructions_audio:"Type what you hear:",help_btn:"Help",play_again:"Play sound again",cant_hear_this:"Download sound as MP3",incorrect_try_again:"Incorrect. Try again.",image_alt_text:"reCAPTCHA challenge image",privacy_and_terms:"Privacy & Terms"},ja={visual_challenge:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u0645\u0631\u0626\u064a", +audio_challenge:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u0635\u0648\u062a\u064a",refresh_btn:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u062c\u062f\u064a\u062f",instructions_visual:"\u064a\u0631\u062c\u0649 \u0643\u062a\u0627\u0628\u0629 \u0627\u0644\u0646\u0635:",instructions_audio:"\u0627\u0643\u062a\u0628 \u0645\u0627 \u062a\u0633\u0645\u0639\u0647:",help_btn:"\u0645\u0633\u0627\u0639\u062f\u0629",play_again:"\u062a\u0634\u063a\u064a\u0644 \u0627\u0644\u0635\u0648\u062a \u0645\u0631\u0629 \u0623\u062e\u0631\u0649", +cant_hear_this:"\u062a\u0646\u0632\u064a\u0644 \u0627\u0644\u0635\u0648\u062a \u0628\u062a\u0646\u0633\u064a\u0642 MP3",incorrect_try_again:"\u063a\u064a\u0631 \u0635\u062d\u064a\u062d. \u0623\u0639\u062f \u0627\u0644\u0645\u062d\u0627\u0648\u0644\u0629.",image_alt_text:"\u0635\u0648\u0631\u0629 \u0627\u0644\u062a\u062d\u062f\u064a \u0645\u0646 reCAPTCHA",privacy_and_terms:"\u0627\u0644\u062e\u0635\u0648\u0635\u064a\u0629 \u0648\u0627\u0644\u0628\u0646\u0648\u062f"},ka={visual_challenge:"Obtener una pista visual", +audio_challenge:"Obtener una pista sonora",refresh_btn:"Obtener una pista nueva",instructions_visual:"Introduzca el texto:",instructions_audio:"Escribe lo que oigas:",help_btn:"Ayuda",play_again:"Volver a reproducir el sonido",cant_hear_this:"Descargar el sonido en MP3",incorrect_try_again:"Incorrecto. Vu\u00e9lvelo a intentar.",image_alt_text:"Pista de imagen reCAPTCHA",privacy_and_terms:"Privacidad y condiciones"},la={visual_challenge:"Kumuha ng pagsubok na visual",audio_challenge:"Kumuha ng pagsubok na audio", +refresh_btn:"Kumuha ng bagong pagsubok",instructions_visual:"I-type ang teksto:",instructions_audio:"I-type ang iyong narinig",help_btn:"Tulong",play_again:"I-play muli ang tunog",cant_hear_this:"I-download ang tunog bilang MP3",incorrect_try_again:"Hindi wasto. Muling subukan.",image_alt_text:"larawang panghamon ng reCAPTCHA",privacy_and_terms:"Privacy at Mga Tuntunin"},ma={visual_challenge:"Test visuel",audio_challenge:"Test audio",refresh_btn:"Nouveau test",instructions_visual:"Saisissez le texte\u00a0:", +instructions_audio:"Qu'entendez-vous ?",help_btn:"Aide",play_again:"R\u00e9\u00e9couter",cant_hear_this:"T\u00e9l\u00e9charger l'audio au format MP3",incorrect_try_again:"Incorrect. Veuillez r\u00e9essayer.",image_alt_text:"Image reCAPTCHA",privacy_and_terms:"Confidentialit\u00e9 et conditions d'utilisation"},na={visual_challenge:"Dapatkan kata pengujian berbentuk visual",audio_challenge:"Dapatkan kata pengujian berbentuk audio",refresh_btn:"Dapatkan kata pengujian baru",instructions_visual:"Ketik teks:", +instructions_audio:"Ketik yang Anda dengar:",help_btn:"Bantuan",play_again:"Putar suara sekali lagi",cant_hear_this:"Unduh suara sebagai MP3",incorrect_try_again:"Salah. Coba lagi.",image_alt_text:"Gambar tantangan reCAPTCHA",privacy_and_terms:"Privasi & Persyaratan"},oa={visual_challenge:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05d7\u05d6\u05d5\u05ea\u05d9",audio_challenge:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05e9\u05de\u05e2",refresh_btn:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05d7\u05d3\u05e9", +instructions_visual:"\u05d4\u05e7\u05dc\u05d3 \u05d0\u05ea \u05d4\u05d8\u05e7\u05e1\u05d8:",instructions_audio:"\u05d4\u05e7\u05dc\u05d3 \u05d0\u05ea \u05de\u05d4 \u05e9\u05d0\u05ea\u05d4 \u05e9\u05d5\u05de\u05e2:",help_btn:"\u05e2\u05d6\u05e8\u05d4",play_again:"\u05d4\u05e4\u05e2\u05dc \u05e9\u05d5\u05d1 \u05d0\u05ea \u05d4\u05e9\u05de\u05e2",cant_hear_this:"\u05d4\u05d5\u05e8\u05d3 \u05e9\u05de\u05e2 \u05db-3MP",incorrect_try_again:"\u05e9\u05d2\u05d5\u05d9. \u05e0\u05e1\u05d4 \u05e9\u05d5\u05d1.", +image_alt_text:"\u05ea\u05de\u05d5\u05e0\u05ea \u05d0\u05ea\u05d2\u05e8 \u05e9\u05dc reCAPTCHA",privacy_and_terms:"\u05e4\u05e8\u05d8\u05d9\u05d5\u05ea \u05d5\u05ea\u05e0\u05d0\u05d9\u05dd"},pa={visual_challenge:"Obter um desafio visual",audio_challenge:"Obter um desafio de \u00e1udio",refresh_btn:"Obter um novo desafio",instructions_visual:"Digite o texto:",instructions_audio:"Digite o que voc\u00ea ouve:",help_btn:"Ajuda",play_again:"Reproduzir som novamente",cant_hear_this:"Fazer download do som no formato MP3", +incorrect_try_again:"Incorreto. Tente novamente.",image_alt_text:"Imagem de desafio reCAPTCHA",privacy_and_terms:"Privacidade e Termos"},qa={visual_challenge:"Ob\u0163ine\u0163i un cod captcha vizual",audio_challenge:"Ob\u0163ine\u0163i un cod captcha audio",refresh_btn:"Ob\u0163ine\u0163i un nou cod captcha",instructions_visual:"Introduce\u021bi textul:",instructions_audio:"Introduce\u0163i ceea ce auzi\u0163i:",help_btn:"Ajutor",play_again:"Reda\u0163i sunetul din nou",cant_hear_this:"Desc\u0103rca\u0163i fi\u015fierul audio ca MP3", +incorrect_try_again:"Incorect. \u00cencerca\u0163i din nou.",image_alt_text:"Imagine de verificare reCAPTCHA",privacy_and_terms:"Confiden\u0163ialitate \u015fi termeni"},ra={visual_challenge:"\u6536\u5230\u4e00\u4e2a\u89c6\u9891\u9080\u8bf7",audio_challenge:"\u6362\u4e00\u7ec4\u97f3\u9891\u9a8c\u8bc1\u7801",refresh_btn:"\u6362\u4e00\u7ec4\u9a8c\u8bc1\u7801",instructions_visual:"\u8f93\u5165\u6587\u5b57\uff1a",instructions_audio:"\u8bf7\u952e\u5165\u60a8\u542c\u5230\u7684\u5185\u5bb9\uff1a",help_btn:"\u5e2e\u52a9", +play_again:"\u91cd\u65b0\u64ad\u653e",cant_hear_this:"\u4ee5 MP3 \u683c\u5f0f\u4e0b\u8f7d\u58f0\u97f3",incorrect_try_again:"\u4e0d\u6b63\u786e\uff0c\u8bf7\u91cd\u8bd5\u3002",image_alt_text:"reCAPTCHA \u9a8c\u8bc1\u56fe\u7247",privacy_and_terms:"\u9690\u79c1\u6743\u548c\u4f7f\u7528\u6761\u6b3e"},sa={en:w,af:{visual_challenge:"Kry 'n visuele verifi\u00ebring",audio_challenge:"Kry 'n klankverifi\u00ebring",refresh_btn:"Kry 'n nuwe verifi\u00ebring",instructions_visual:"",instructions_audio:"Tik wat jy hoor:", +help_btn:"Hulp",play_again:"Speel geluid weer",cant_hear_this:"Laai die klank af as MP3",incorrect_try_again:"Verkeerd. Probeer weer.",image_alt_text:"reCAPTCHA-uitdagingprent",privacy_and_terms:"Privaatheid en bepalings"},am:{visual_challenge:"\u12e8\u12a5\u12ed\u1273 \u1270\u130b\u1323\u121a \u12a0\u130d\u129d",audio_challenge:"\u120c\u120b \u12a0\u12f2\u1235 \u12e8\u12f5\u121d\u133d \u1325\u12eb\u1244 \u12ed\u1245\u1228\u1265",refresh_btn:"\u120c\u120b \u12a0\u12f2\u1235 \u1325\u12eb\u1244 \u12ed\u1245\u1228\u1265", +instructions_visual:"",instructions_audio:"\u12e8\u121d\u1275\u1230\u121b\u12cd\u1295 \u1270\u12ed\u1265\u1361-",help_btn:"\u12a5\u1308\u12db",play_again:"\u12f5\u121d\u1339\u1295 \u12a5\u1295\u12f0\u1308\u1293 \u12a0\u132b\u12cd\u1275",cant_hear_this:"\u12f5\u121d\u1339\u1295 \u1260MP3 \u1245\u122d\u133d \u12a0\u12cd\u122d\u12f5",incorrect_try_again:"\u1275\u12ad\u12ad\u120d \u12a0\u12ed\u12f0\u1208\u121d\u1362 \u12a5\u1295\u12f0\u1308\u1293 \u121e\u12ad\u122d\u1362",image_alt_text:"reCAPTCHA \u121d\u1235\u120d \u130d\u1320\u121d", +privacy_and_terms:"\u130d\u120b\u12ca\u1290\u1275 \u12a5\u1293 \u12cd\u120d"},ar:ja,"ar-EG":ja,bg:{visual_challenge:"\u041f\u043e\u043b\u0443\u0447\u0430\u0432\u0430\u043d\u0435 \u043d\u0430 \u0432\u0438\u0437\u0443\u0430\u043b\u043d\u0430 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",audio_challenge:"\u0417\u0430\u0440\u0435\u0436\u0434\u0430\u043d\u0435 \u043d\u0430 \u0430\u0443\u0434\u0438\u043e\u0442\u0435\u0441\u0442",refresh_btn:"\u0417\u0430\u0440\u0435\u0436\u0434\u0430\u043d\u0435 \u043d\u0430 \u043d\u043e\u0432 \u0442\u0435\u0441\u0442", +instructions_visual:"\u0412\u044a\u0432\u0435\u0434\u0435\u0442\u0435 \u0442\u0435\u043a\u0441\u0442\u0430:",instructions_audio:"\u0412\u044a\u0432\u0435\u0434\u0435\u0442\u0435 \u0447\u0443\u0442\u043e\u0442\u043e:",help_btn:"\u041f\u043e\u043c\u043e\u0449",play_again:"\u041f\u043e\u0432\u0442\u043e\u0440\u043d\u043e \u043f\u0443\u0441\u043a\u0430\u043d\u0435 \u043d\u0430 \u0437\u0432\u0443\u043a\u0430",cant_hear_this:"\u0418\u0437\u0442\u0435\u0433\u043b\u044f\u043d\u0435 \u043d\u0430 \u0437\u0432\u0443\u043a\u0430 \u0432\u044a\u0432 \u0444\u043e\u0440\u043c\u0430\u0442 MP3", +incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u043d\u043e. \u041e\u043f\u0438\u0442\u0430\u0439\u0442\u0435 \u043e\u0442\u043d\u043e\u0432\u043e.",image_alt_text:"\u0418\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u043d\u0430 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430\u0442\u0430 \u0441 reCAPTCHA",privacy_and_terms:"\u041f\u043e\u0432\u0435\u0440\u0438\u0442\u0435\u043b\u043d\u043e\u0441\u0442 \u0438 \u041e\u0431\u0449\u0438 \u0443\u0441\u043b\u043e\u0432\u0438\u044f"}, +bn:{visual_challenge:"\u098f\u0995\u099f\u09bf \u09a6\u09c3\u09b6\u09cd\u09af\u09ae\u09be\u09a8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be \u09aa\u09be\u09a8",audio_challenge:"\u098f\u0995\u099f\u09bf \u0985\u09a1\u09bf\u0993 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be \u09aa\u09be\u09a8",refresh_btn:"\u098f\u0995\u099f\u09bf \u09a8\u09a4\u09c1\u09a8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be \u09aa\u09be\u09a8", +instructions_visual:"",instructions_audio:"\u0986\u09aa\u09a8\u09bf \u09af\u09be \u09b6\u09c1\u09a8\u099b\u09c7\u09a8 \u09a4\u09be \u09b2\u09bf\u0996\u09c1\u09a8:",help_btn:"\u09b8\u09b9\u09be\u09df\u09a4\u09be",play_again:"\u0986\u09ac\u09be\u09b0 \u09b8\u09be\u0989\u09a8\u09cd\u09a1 \u09aa\u09cd\u09b2\u09c7 \u0995\u09b0\u09c1\u09a8",cant_hear_this:"MP3 \u09b0\u09c2\u09aa\u09c7 \u09b6\u09ac\u09cd\u09a6 \u09a1\u09be\u0989\u09a8\u09b2\u09cb\u09a1 \u0995\u09b0\u09c1\u09a8",incorrect_try_again:"\u09ac\u09c7\u09a0\u09bf\u0995\u09f7 \u0986\u09ac\u09be\u09b0 \u099a\u09c7\u09b7\u09cd\u099f\u09be \u0995\u09b0\u09c1\u09a8\u09f7", +image_alt_text:"reCAPTCHA \u099a\u09cd\u09af\u09be\u09b2\u09c7\u099e\u09cd\u099c \u099a\u09bf\u09a4\u09cd\u09b0",privacy_and_terms:"\u0997\u09cb\u09aa\u09a8\u09c0\u09af\u09bc\u09a4\u09be \u0993 \u09b6\u09b0\u09cd\u09a4\u09be\u09ac\u09b2\u09c0"},ca:{visual_challenge:"Obt\u00e9n un repte visual",audio_challenge:"Obteniu una pista sonora",refresh_btn:"Obteniu una pista nova",instructions_visual:"Escriviu el text:",instructions_audio:"Escriviu el que escolteu:",help_btn:"Ajuda",play_again:"Torna a reproduir el so", +cant_hear_this:"Baixa el so com a MP3",incorrect_try_again:"No \u00e9s correcte. Torna-ho a provar.",image_alt_text:"Imatge del repte de reCAPTCHA",privacy_and_terms:"Privadesa i condicions"},cs:{visual_challenge:"Zobrazit vizu\u00e1ln\u00ed podobu v\u00fdrazu",audio_challenge:"P\u0159ehr\u00e1t zvukovou podobu v\u00fdrazu",refresh_btn:"Zobrazit nov\u00fd v\u00fdraz",instructions_visual:"Zadejte text:",instructions_audio:"Napi\u0161te, co jste sly\u0161eli:",help_btn:"N\u00e1pov\u011bda",play_again:"Znovu p\u0159ehr\u00e1t zvuk", +cant_hear_this:"St\u00e1hnout zvuk ve form\u00e1tu MP3",incorrect_try_again:"\u0160patn\u011b. Zkuste to znovu.",image_alt_text:"Obr\u00e1zek reCAPTCHA",privacy_and_terms:"Ochrana soukrom\u00ed a smluvn\u00ed podm\u00ednky"},da:{visual_challenge:"Hent en visuel udfordring",audio_challenge:"Hent en lydudfordring",refresh_btn:"Hent en ny udfordring",instructions_visual:"Indtast teksten:",instructions_audio:"Indtast det, du h\u00f8rer:",help_btn:"Hj\u00e6lp",play_again:"Afspil lyden igen",cant_hear_this:"Download lyd som MP3", +incorrect_try_again:"Forkert. Pr\u00f8v igen.",image_alt_text:"reCAPTCHA-udfordringsbillede",privacy_and_terms:"Privatliv og vilk\u00e5r"},de:{visual_challenge:"Captcha abrufen",audio_challenge:"Audio-Captcha abrufen",refresh_btn:"Neues Captcha abrufen",instructions_visual:"Geben Sie den angezeigten Text ein:",instructions_audio:"Geben Sie das Geh\u00f6rte ein:",help_btn:"Hilfe",play_again:"Wort erneut abspielen",cant_hear_this:"Wort als MP3 herunterladen",incorrect_try_again:"Falsch. Bitte versuchen Sie es erneut.", +image_alt_text:"reCAPTCHA-Bild",privacy_and_terms:"Datenschutzerkl\u00e4rung & Nutzungsbedingungen"},el:{visual_challenge:"\u039f\u03c0\u03c4\u03b9\u03ba\u03ae \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",audio_challenge:"\u0397\u03c7\u03b7\u03c4\u03b9\u03ba\u03ae \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",refresh_btn:"\u039d\u03ad\u03b1 \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",instructions_visual:"\u03a0\u03bb\u03b7\u03ba\u03c4\u03c1\u03bf\u03bb\u03bf\u03b3\u03ae\u03c3\u03c4\u03b5 \u03c4\u03bf \u03ba\u03b5\u03af\u03bc\u03b5\u03bd\u03bf:", +instructions_audio:"\u03a0\u03bb\u03b7\u03ba\u03c4\u03c1\u03bf\u03bb\u03bf\u03b3\u03ae\u03c3\u03c4\u03b5 \u03cc\u03c4\u03b9 \u03b1\u03ba\u03bf\u03cd\u03c4\u03b5:",help_btn:"\u0392\u03bf\u03ae\u03b8\u03b5\u03b9\u03b1",play_again:"\u0391\u03bd\u03b1\u03c0\u03b1\u03c1\u03b1\u03b3\u03c9\u03b3\u03ae \u03ae\u03c7\u03bf\u03c5 \u03be\u03b1\u03bd\u03ac",cant_hear_this:"\u039b\u03ae\u03c8\u03b7 \u03ae\u03c7\u03bf\u03c5 \u03c9\u03c2 \u039c\u03a13",incorrect_try_again:"\u039b\u03ac\u03b8\u03bf\u03c2. \u0394\u03bf\u03ba\u03b9\u03bc\u03ac\u03c3\u03c4\u03b5 \u03be\u03b1\u03bd\u03ac.", +image_alt_text:"\u0395\u03b9\u03ba\u03cc\u03bd\u03b1 \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7\u03c2 reCAPTCHA",privacy_and_terms:"\u0391\u03c0\u03cc\u03c1\u03c1\u03b7\u03c4\u03bf \u03ba\u03b1\u03b9 \u03cc\u03c1\u03bf\u03b9"},"en-GB":w,"en-US":w,es:ka,"es-419":{visual_challenge:"Enfrentar un desaf\u00edo visual",audio_challenge:"Enfrentar un desaf\u00edo de audio",refresh_btn:"Enfrentar un nuevo desaf\u00edo",instructions_visual:"Escriba el texto:",instructions_audio:"Escribe lo que escuchas:", +help_btn:"Ayuda",play_again:"Reproducir sonido de nuevo",cant_hear_this:"Descargar sonido en formato MP3",incorrect_try_again:"Incorrecto. Vuelve a intentarlo.",image_alt_text:"Imagen del desaf\u00edo de la reCAPTCHA",privacy_and_terms:"Privacidad y condiciones"},"es-ES":ka,et:{visual_challenge:"Kuva kuvap\u00f5hine robotil\u00f5ks",audio_challenge:"Kuva helip\u00f5hine robotil\u00f5ks",refresh_btn:"Kuva uus robotil\u00f5ks",instructions_visual:"Tippige tekst:",instructions_audio:"Tippige, mida kuulete.", +help_btn:"Abi",play_again:"Esita heli uuesti",cant_hear_this:"Laadi heli alla MP3-vormingus",incorrect_try_again:"Vale. Proovige uuesti.",image_alt_text:"reCAPTCHA robotil\u00f5ksu kujutis",privacy_and_terms:"Privaatsus ja tingimused"},eu:{visual_challenge:"Eskuratu ikusizko erronka",audio_challenge:"Eskuratu audio-erronka",refresh_btn:"Eskuratu erronka berria",instructions_visual:"",instructions_audio:"Idatzi entzuten duzuna:",help_btn:"Laguntza",play_again:"Erreproduzitu soinua berriro",cant_hear_this:"Deskargatu soinua MP3 gisa", +incorrect_try_again:"Ez da zuzena. Saiatu berriro.",image_alt_text:"reCAPTCHA erronkaren irudia",privacy_and_terms:"Pribatutasuna eta baldintzak"},fa:{visual_challenge:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u062f\u06cc\u062f\u0627\u0631\u06cc",audio_challenge:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u0635\u0648\u062a\u06cc",refresh_btn:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u062c\u062f\u06cc\u062f", +instructions_visual:"",instructions_audio:"\u0622\u0646\u0686\u0647 \u0631\u0627 \u06a9\u0647 \u0645\u06cc\u200c\u0634\u0646\u0648\u06cc\u062f \u062a\u0627\u06cc\u067e \u06a9\u0646\u06cc\u062f:",help_btn:"\u0631\u0627\u0647\u0646\u0645\u0627\u06cc\u06cc",play_again:"\u067e\u062e\u0634 \u0645\u062c\u062f\u062f \u0635\u062f\u0627",cant_hear_this:"\u062f\u0627\u0646\u0644\u0648\u062f \u0635\u062f\u0627 \u0628\u0647 \u0635\u0648\u0631\u062a MP3",incorrect_try_again:"\u0646\u0627\u062f\u0631\u0633\u062a. \u062f\u0648\u0628\u0627\u0631\u0647 \u0627\u0645\u062a\u062d\u0627\u0646 \u06a9\u0646\u06cc\u062f.", +image_alt_text:"\u062a\u0635\u0648\u06cc\u0631 \u0686\u0627\u0644\u0634\u06cc reCAPTCHA",privacy_and_terms:"\u062d\u0631\u06cc\u0645 \u062e\u0635\u0648\u0635\u06cc \u0648 \u0634\u0631\u0627\u06cc\u0637"},fi:{visual_challenge:"Kuvavahvistus",audio_challenge:"\u00c4\u00e4nivahvistus",refresh_btn:"Uusi kuva",instructions_visual:"Kirjoita teksti:",instructions_audio:"Kirjoita kuulemasi:",help_btn:"Ohje",play_again:"Toista \u00e4\u00e4ni uudelleen",cant_hear_this:"Lataa \u00e4\u00e4ni MP3-tiedostona", +incorrect_try_again:"V\u00e4\u00e4rin. Yrit\u00e4 uudelleen.",image_alt_text:"reCAPTCHA-kuva",privacy_and_terms:"Tietosuoja ja k\u00e4ytt\u00f6ehdot"},fil:la,fr:ma,"fr-CA":{visual_challenge:"Obtenir un test visuel",audio_challenge:"Obtenir un test audio",refresh_btn:"Obtenir un nouveau test",instructions_visual:"Saisissez le texte\u00a0:",instructions_audio:"Tapez ce que vous entendez\u00a0:",help_btn:"Aide",play_again:"Jouer le son de nouveau",cant_hear_this:"T\u00e9l\u00e9charger le son en format MP3", +incorrect_try_again:"Erreur, essayez \u00e0 nouveau",image_alt_text:"Image reCAPTCHA",privacy_and_terms:"Confidentialit\u00e9 et conditions d'utilisation"},"fr-FR":ma,gl:{visual_challenge:"Obter unha proba visual",audio_challenge:"Obter unha proba de audio",refresh_btn:"Obter unha proba nova",instructions_visual:"",instructions_audio:"Escribe o que escoitas:",help_btn:"Axuda",play_again:"Reproducir o son de novo",cant_hear_this:"Descargar son como MP3",incorrect_try_again:"Incorrecto. T\u00e9ntao de novo.", +image_alt_text:"Imaxe de proba de reCAPTCHA",privacy_and_terms:"Privacidade e condici\u00f3ns"},gu:{visual_challenge:"\u0a8f\u0a95 \u0aa6\u0ac3\u0ab6\u0acd\u0aaf\u0abe\u0aa4\u0acd\u0aae\u0a95 \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",audio_challenge:"\u0a8f\u0a95 \u0a91\u0aa1\u0abf\u0a93 \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",refresh_btn:"\u0a8f\u0a95 \u0aa8\u0ab5\u0acb \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",instructions_visual:"", +instructions_audio:"\u0aa4\u0aae\u0ac7 \u0a9c\u0ac7 \u0ab8\u0abe\u0a82\u0aad\u0ab3\u0acb \u0a9b\u0acb \u0aa4\u0ac7 \u0ab2\u0a96\u0acb:",help_btn:"\u0ab8\u0ab9\u0abe\u0aaf",play_again:"\u0aa7\u0acd\u0ab5\u0aa8\u0abf \u0aab\u0ab0\u0ac0\u0aa5\u0ac0 \u0a9a\u0ab2\u0abe\u0ab5\u0acb",cant_hear_this:"MP3 \u0aa4\u0ab0\u0ac0\u0a95\u0ac7 \u0aa7\u0acd\u0ab5\u0aa8\u0abf\u0aa8\u0ac7 \u0aa1\u0abe\u0a89\u0aa8\u0ab2\u0acb\u0aa1 \u0a95\u0ab0\u0acb",incorrect_try_again:"\u0a96\u0acb\u0a9f\u0ac1\u0a82. \u0aab\u0ab0\u0ac0 \u0aaa\u0acd\u0ab0\u0aaf\u0abe\u0ab8 \u0a95\u0ab0\u0acb.", +image_alt_text:"reCAPTCHA \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0a9b\u0aac\u0ac0",privacy_and_terms:"\u0a97\u0acb\u0aaa\u0aa8\u0ac0\u0aaf\u0aa4\u0abe \u0a85\u0aa8\u0ac7 \u0ab6\u0ab0\u0aa4\u0acb"},hi:{visual_challenge:"\u0915\u094b\u0908 \u0935\u093f\u091c\u0941\u0905\u0932 \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",audio_challenge:"\u0915\u094b\u0908 \u0911\u0921\u093f\u092f\u094b \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",refresh_btn:"\u0915\u094b\u0908 \u0928\u0908 \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902", +instructions_visual:"\u091f\u0947\u0915\u094d\u0938\u094d\u091f \u091f\u093e\u0907\u092a \u0915\u0930\u0947\u0902:",instructions_audio:"\u091c\u094b \u0906\u092a \u0938\u0941\u0928 \u0930\u0939\u0947 \u0939\u0948\u0902 \u0909\u0938\u0947 \u0932\u093f\u0916\u0947\u0902:",help_btn:"\u0938\u0939\u093e\u092f\u0924\u093e",play_again:"\u0927\u094d\u200d\u0935\u0928\u093f \u092a\u0941\u0928: \u091a\u0932\u093e\u090f\u0902",cant_hear_this:"\u0927\u094d\u200d\u0935\u0928\u093f \u0915\u094b MP3 \u0915\u0947 \u0930\u0942\u092a \u092e\u0947\u0902 \u0921\u093e\u0909\u0928\u0932\u094b\u0921 \u0915\u0930\u0947\u0902", +incorrect_try_again:"\u0917\u0932\u0924. \u092a\u0941\u0928: \u092a\u094d\u0930\u092f\u093e\u0938 \u0915\u0930\u0947\u0902.",image_alt_text:"reCAPTCHA \u091a\u0941\u0928\u094c\u0924\u0940 \u091a\u093f\u0924\u094d\u0930",privacy_and_terms:"\u0917\u094b\u092a\u0928\u0940\u092f\u0924\u093e \u0914\u0930 \u0936\u0930\u094d\u0924\u0947\u0902"},hr:{visual_challenge:"Dohvati vizualni upit",audio_challenge:"Dohvati zvu\u010dni upit",refresh_btn:"Dohvati novi upit",instructions_visual:"Unesite tekst:",instructions_audio:"Upi\u0161ite \u0161to \u010dujete:", +help_btn:"Pomo\u0107",play_again:"Ponovi zvuk",cant_hear_this:"Preuzmi zvuk u MP3 formatu",incorrect_try_again:"Nije to\u010dno. Poku\u0161ajte ponovno.",image_alt_text:"Slikovni izazov reCAPTCHA",privacy_and_terms:"Privatnost i odredbe"},hu:{visual_challenge:"Vizu\u00e1lis kih\u00edv\u00e1s k\u00e9r\u00e9se",audio_challenge:"Hangkih\u00edv\u00e1s k\u00e9r\u00e9se",refresh_btn:"\u00daj kih\u00edv\u00e1s k\u00e9r\u00e9se",instructions_visual:"\u00cdrja be a sz\u00f6veget:",instructions_audio:"\u00cdrja le, amit hall:", +help_btn:"S\u00fag\u00f3",play_again:"Hang ism\u00e9telt lej\u00e1tsz\u00e1sa",cant_hear_this:"Hang let\u00f6lt\u00e9se MP3 form\u00e1tumban",incorrect_try_again:"Hib\u00e1s. Pr\u00f3b\u00e1lkozzon \u00fajra.",image_alt_text:"reCAPTCHA ellen\u0151rz\u0151 k\u00e9p",privacy_and_terms:"Adatv\u00e9delem \u00e9s Szerz\u0151d\u00e9si Felt\u00e9telek"},hy:{visual_challenge:"\u054d\u057f\u0561\u0576\u0561\u056c \u057f\u0565\u057d\u0578\u0572\u0561\u056f\u0561\u0576 \u056d\u0576\u0564\u056b\u0580",audio_challenge:"\u054d\u057f\u0561\u0576\u0561\u056c \u0571\u0561\u0575\u0576\u0561\u0575\u056b\u0576 \u056d\u0576\u0564\u056b\u0580", +refresh_btn:"\u054d\u057f\u0561\u0576\u0561\u056c \u0576\u0578\u0580 \u056d\u0576\u0564\u056b\u0580",instructions_visual:"\u0544\u0578\u0582\u057f\u0584\u0561\u0563\u0580\u0565\u0584 \u057f\u0565\u0584\u057d\u057f\u0568\u055d",instructions_audio:"\u0544\u0578\u0582\u057f\u0584\u0561\u0563\u0580\u0565\u0584 \u0561\u0575\u0576, \u056b\u0576\u0579 \u056c\u057d\u0578\u0582\u0574 \u0565\u0584\u055d",help_btn:"\u0555\u0563\u0576\u0578\u0582\u0569\u0575\u0578\u0582\u0576",play_again:"\u0546\u057e\u0561\u0563\u0561\u0580\u056f\u0565\u056c \u0571\u0561\u0575\u0576\u0568 \u056f\u0580\u056f\u056b\u0576", +cant_hear_this:"\u0532\u0565\u057c\u0576\u0565\u056c \u0571\u0561\u0575\u0576\u0568 \u0578\u0580\u057a\u0565\u057d MP3",incorrect_try_again:"\u054d\u056d\u0561\u056c \u0567: \u0553\u0578\u0580\u0571\u0565\u0584 \u056f\u0580\u056f\u056b\u0576:",image_alt_text:"reCAPTCHA \u057a\u0561\u057f\u056f\u0565\u0580\u0578\u057e \u056d\u0576\u0564\u056b\u0580",privacy_and_terms:"\u0533\u0561\u0572\u057f\u0576\u056b\u0578\u0582\u0569\u0575\u0561\u0576 & \u057a\u0561\u0575\u0574\u0561\u0576\u0576\u0565\u0580"}, +id:na,is:{visual_challenge:"F\u00e1 a\u00f0gangspr\u00f3f sem mynd",audio_challenge:"F\u00e1 a\u00f0gangspr\u00f3f sem hlj\u00f3\u00f0skr\u00e1",refresh_btn:"F\u00e1 n\u00fdtt a\u00f0gangspr\u00f3f",instructions_visual:"",instructions_audio:"Sl\u00e1\u00f0u inn \u00fea\u00f0 sem \u00fe\u00fa heyrir:",help_btn:"Hj\u00e1lp",play_again:"Spila hlj\u00f3\u00f0 aftur",cant_hear_this:"S\u00e6kja hlj\u00f3\u00f0 sem MP3",incorrect_try_again:"Rangt. Reyndu aftur.",image_alt_text:"mynd reCAPTCHA a\u00f0gangspr\u00f3fs", +privacy_and_terms:"Pers\u00f3nuvernd og skilm\u00e1lar"},it:{visual_challenge:"Verifica visiva",audio_challenge:"Verifica audio",refresh_btn:"Nuova verifica",instructions_visual:"Digita il testo:",instructions_audio:"Digita ci\u00f2 che senti:",help_btn:"Guida",play_again:"Riproduci di nuovo audio",cant_hear_this:"Scarica audio in MP3",incorrect_try_again:"Sbagliato. Riprova.",image_alt_text:"Immagine di verifica reCAPTCHA",privacy_and_terms:"Privacy e Termini"},iw:oa,ja:{visual_challenge:"\u753b\u50cf\u3067\u78ba\u8a8d\u3057\u307e\u3059", +audio_challenge:"\u97f3\u58f0\u3067\u78ba\u8a8d\u3057\u307e\u3059",refresh_btn:"\u5225\u306e\u5358\u8a9e\u3067\u3084\u308a\u76f4\u3057\u307e\u3059",instructions_visual:"\u30c6\u30ad\u30b9\u30c8\u3092\u5165\u529b:",instructions_audio:"\u805e\u3053\u3048\u305f\u5358\u8a9e\u3092\u5165\u529b\u3057\u307e\u3059:",help_btn:"\u30d8\u30eb\u30d7",play_again:"\u3082\u3046\u4e00\u5ea6\u805e\u304f",cant_hear_this:"MP3 \u3067\u97f3\u58f0\u3092\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9",incorrect_try_again:"\u6b63\u3057\u304f\u3042\u308a\u307e\u305b\u3093\u3002\u3082\u3046\u4e00\u5ea6\u3084\u308a\u76f4\u3057\u3066\u304f\u3060\u3055\u3044\u3002", +image_alt_text:"reCAPTCHA \u78ba\u8a8d\u7528\u753b\u50cf",privacy_and_terms:"\u30d7\u30e9\u30a4\u30d0\u30b7\u30fc\u3068\u5229\u7528\u898f\u7d04"},kn:{visual_challenge:"\u0ca6\u0cc3\u0cb6\u0ccd\u0caf \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb8\u0ccd\u0cb5\u0cc0\u0c95\u0cb0\u0cbf\u0cb8\u0cbf",audio_challenge:"\u0c86\u0ca1\u0cbf\u0caf\u0ccb \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb8\u0ccd\u0cb5\u0cc0\u0c95\u0cb0\u0cbf\u0cb8\u0cbf",refresh_btn:"\u0cb9\u0cca\u0cb8 \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0caa\u0ca1\u0cc6\u0caf\u0cbf\u0cb0\u0cbf", +instructions_visual:"",instructions_audio:"\u0ca8\u0cbf\u0cae\u0c97\u0cc6 \u0c95\u0cc7\u0cb3\u0cbf\u0cb8\u0cc1\u0cb5\u0cc1\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0c9f\u0cc8\u0caa\u0ccd\u200c \u0cae\u0cbe\u0ca1\u0cbf:",help_btn:"\u0cb8\u0cb9\u0cbe\u0caf",play_again:"\u0ca7\u0ccd\u0cb5\u0ca8\u0cbf\u0caf\u0ca8\u0ccd\u0ca8\u0cc1 \u0cae\u0ca4\u0ccd\u0ca4\u0cc6 \u0caa\u0ccd\u0cb2\u0cc7 \u0cae\u0cbe\u0ca1\u0cbf",cant_hear_this:"\u0ca7\u0ccd\u0cb5\u0ca8\u0cbf\u0caf\u0ca8\u0ccd\u0ca8\u0cc1 MP3 \u0cb0\u0cc2\u0caa\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf \u0ca1\u0ccc\u0ca8\u0ccd\u200c\u0cb2\u0ccb\u0ca1\u0ccd \u0cae\u0cbe\u0ca1\u0cbf", +incorrect_try_again:"\u0ca4\u0caa\u0ccd\u0caa\u0cbe\u0c97\u0cbf\u0ca6\u0cc6. \u0cae\u0ca4\u0ccd\u0ca4\u0cca\u0cae\u0ccd\u0cae\u0cc6 \u0caa\u0ccd\u0cb0\u0caf\u0ca4\u0ccd\u0ca8\u0cbf\u0cb8\u0cbf.",image_alt_text:"reCAPTCHA \u0cb8\u0cb5\u0cbe\u0cb2\u0cc1 \u0c9a\u0cbf\u0ca4\u0ccd\u0cb0",privacy_and_terms:"\u0c97\u0ccc\u0caa\u0ccd\u0caf\u0ca4\u0cc6 \u0cae\u0ca4\u0ccd\u0ca4\u0cc1 \u0ca8\u0cbf\u0caf\u0cae\u0c97\u0cb3\u0cc1"},ko:{visual_challenge:"\uadf8\ub9bc\uc73c\ub85c \ubcf4\uc548\ubb38\uc790 \ubc1b\uae30", +audio_challenge:"\uc74c\uc131\uc73c\ub85c \ubcf4\uc548\ubb38\uc790 \ubc1b\uae30",refresh_btn:"\ubcf4\uc548\ubb38\uc790 \uc0c8\ub85c \ubc1b\uae30",instructions_visual:"\ud14d\uc2a4\ud2b8 \uc785\ub825:",instructions_audio:"\uc74c\uc131 \ubcf4\uc548\ubb38\uc790 \uc785\ub825:",help_btn:"\ub3c4\uc6c0\ub9d0",play_again:"\uc74c\uc131 \ub2e4\uc2dc \ub4e3\uae30",cant_hear_this:"\uc74c\uc131\uc744 MP3\ub85c \ub2e4\uc6b4\ub85c\ub4dc",incorrect_try_again:"\ud2c0\ub838\uc2b5\ub2c8\ub2e4. \ub2e4\uc2dc \uc2dc\ub3c4\ud574 \uc8fc\uc138\uc694.", +image_alt_text:"reCAPTCHA \ubcf4\uc548\ubb38\uc790 \uc774\ubbf8\uc9c0",privacy_and_terms:"\uac1c\uc778\uc815\ubcf4 \ubcf4\ud638 \ubc0f \uc57d\uad00"},ln:ma,lt:{visual_challenge:"Gauti vaizdin\u012f atpa\u017einimo test\u0105",audio_challenge:"Gauti garso atpa\u017einimo test\u0105",refresh_btn:"Gauti nauj\u0105 atpa\u017einimo test\u0105",instructions_visual:"\u012eveskite tekst\u0105:",instructions_audio:"\u012eveskite tai, k\u0105 girdite:",help_btn:"Pagalba",play_again:"Dar kart\u0105 paleisti gars\u0105", +cant_hear_this:"Atsisi\u0173sti gars\u0105 kaip MP3",incorrect_try_again:"Neteisingai. Bandykite dar kart\u0105.",image_alt_text:"Testo \u201ereCAPTCHA\u201c vaizdas",privacy_and_terms:"Privatumas ir s\u0105lygos"},lv:{visual_challenge:"Sa\u0146emt vizu\u0101lu izaicin\u0101jumu",audio_challenge:"Sa\u0146emt audio izaicin\u0101jumu",refresh_btn:"Sa\u0146emt jaunu izaicin\u0101jumu",instructions_visual:"Ievadiet tekstu:",instructions_audio:"Ierakstiet dzirdamo:",help_btn:"Pal\u012bdz\u012bba",play_again:"V\u0113lreiz atska\u0146ot ska\u0146u", +cant_hear_this:"Lejupiel\u0101d\u0113t ska\u0146u MP3\u00a0form\u0101t\u0101",incorrect_try_again:"Nepareizi. M\u0113\u0123iniet v\u0113lreiz.",image_alt_text:"reCAPTCHA izaicin\u0101juma att\u0113ls",privacy_and_terms:"Konfidencialit\u0101te un noteikumi"},ml:{visual_challenge:"\u0d12\u0d30\u0d41 \u0d26\u0d43\u0d36\u0d4d\u0d2f \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",audio_challenge:"\u0d12\u0d30\u0d41 \u0d13\u0d21\u0d3f\u0d2f\u0d4b \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15", +refresh_btn:"\u0d12\u0d30\u0d41 \u0d2a\u0d41\u0d24\u0d3f\u0d2f \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",instructions_visual:"",instructions_audio:"\u0d15\u0d47\u0d7e\u0d15\u0d4d\u0d15\u0d41\u0d28\u0d4d\u0d28\u0d24\u0d4d \u0d1f\u0d48\u0d2a\u0d4d\u0d2a\u0d4d \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d42:",help_btn:"\u0d38\u0d39\u0d3e\u0d2f\u0d02",play_again:"\u0d36\u0d2c\u0d4d\u200c\u0d26\u0d02 \u0d35\u0d40\u0d23\u0d4d\u0d1f\u0d41\u0d02 \u0d2a\u0d4d\u0d32\u0d47 \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d41\u0d15", +cant_hear_this:"\u0d36\u0d2c\u0d4d\u200c\u0d26\u0d02 MP3 \u0d06\u0d2f\u0d3f \u0d21\u0d57\u0d7a\u0d32\u0d4b\u0d21\u0d4d \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d41\u0d15",incorrect_try_again:"\u0d24\u0d46\u0d31\u0d4d\u0d31\u0d3e\u0d23\u0d4d. \u0d35\u0d40\u0d23\u0d4d\u0d1f\u0d41\u0d02 \u0d36\u0d4d\u0d30\u0d2e\u0d3f\u0d15\u0d4d\u0d15\u0d41\u0d15.",image_alt_text:"reCAPTCHA \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d07\u0d2e\u0d47\u0d1c\u0d4d",privacy_and_terms:"\u0d38\u0d4d\u0d35\u0d15\u0d3e\u0d30\u0d4d\u0d2f\u0d24\u0d2f\u0d41\u0d02 \u0d28\u0d3f\u0d2c\u0d28\u0d4d\u0d27\u0d28\u0d15\u0d33\u0d41\u0d02"}, +mr:{visual_challenge:"\u0926\u0943\u0936\u094d\u200d\u092f\u092e\u093e\u0928 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",audio_challenge:"\u0911\u0921\u0940\u0913 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",refresh_btn:"\u090f\u0915 \u0928\u0935\u0940\u0928 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",instructions_visual:"",instructions_audio:"\u0906\u092a\u0932\u094d\u092f\u093e\u0932\u093e \u091c\u0947 \u0910\u0915\u0942 \u092f\u0947\u0908\u0932 \u0924\u0947 \u091f\u093e\u0907\u092a \u0915\u0930\u093e:", +help_btn:"\u092e\u0926\u0924",play_again:"\u0927\u094d\u200d\u0935\u0928\u0940 \u092a\u0941\u0928\u094d\u0939\u093e \u092a\u094d\u200d\u0932\u0947 \u0915\u0930\u093e",cant_hear_this:"MP3 \u0930\u0941\u092a\u093e\u0924 \u0927\u094d\u200d\u0935\u0928\u0940 \u0921\u093e\u0909\u0928\u0932\u094b\u0921 \u0915\u0930\u093e",incorrect_try_again:"\u0905\u092f\u094b\u0917\u094d\u200d\u092f. \u092a\u0941\u0928\u094d\u200d\u0939\u093e \u092a\u094d\u0930\u092f\u0924\u094d\u200d\u0928 \u0915\u0930\u093e.",image_alt_text:"reCAPTCHA \u0906\u0935\u094d\u200d\u0939\u093e\u0928 \u092a\u094d\u0930\u0924\u093f\u092e\u093e", +privacy_and_terms:"\u0917\u094b\u092a\u0928\u0940\u092f\u0924\u093e \u0906\u0923\u093f \u0905\u091f\u0940"},ms:{visual_challenge:"Dapatkan cabaran visual",audio_challenge:"Dapatkan cabaran audio",refresh_btn:"Dapatkan cabaran baru",instructions_visual:"Taipkan teksnya:",instructions_audio:"Taip apa yang didengari:",help_btn:"Bantuan",play_again:"Mainkan bunyi sekali lagi",cant_hear_this:"Muat turun bunyi sebagai MP3",incorrect_try_again:"Tidak betul. Cuba lagi.",image_alt_text:"Imej cabaran reCAPTCHA", +privacy_and_terms:"Privasi & Syarat"},nl:{visual_challenge:"Een visuele uitdaging proberen",audio_challenge:"Een audio-uitdaging proberen",refresh_btn:"Een nieuwe uitdaging proberen",instructions_visual:"Typ de tekst:",instructions_audio:"Typ wat u hoort:",help_btn:"Help",play_again:"Geluid opnieuw afspelen",cant_hear_this:"Geluid downloaden als MP3",incorrect_try_again:"Onjuist. Probeer het opnieuw.",image_alt_text:"reCAPTCHA-uitdagingsafbeelding",privacy_and_terms:"Privacy en voorwaarden"},no:{visual_challenge:"F\u00e5 en bildeutfordring", +audio_challenge:"F\u00e5 en lydutfordring",refresh_btn:"F\u00e5 en ny utfordring",instructions_visual:"Skriv inn teksten:",instructions_audio:"Skriv inn det du h\u00f8rer:",help_btn:"Hjelp",play_again:"Spill av lyd p\u00e5 nytt",cant_hear_this:"Last ned lyd som MP3",incorrect_try_again:"Feil. Pr\u00f8v p\u00e5 nytt.",image_alt_text:"reCAPTCHA-utfordringsbilde",privacy_and_terms:"Personvern og vilk\u00e5r"},pl:{visual_challenge:"Poka\u017c podpowied\u017a wizualn\u0105",audio_challenge:"Odtw\u00f3rz podpowied\u017a d\u017awi\u0119kow\u0105", +refresh_btn:"Nowa podpowied\u017a",instructions_visual:"Przepisz tekst:",instructions_audio:"Wpisz us\u0142yszane s\u0142owa:",help_btn:"Pomoc",play_again:"Odtw\u00f3rz d\u017awi\u0119k ponownie",cant_hear_this:"Pobierz d\u017awi\u0119k jako plik MP3",incorrect_try_again:"Nieprawid\u0142owo. Spr\u00f3buj ponownie.",image_alt_text:"Zadanie obrazkowe reCAPTCHA",privacy_and_terms:"Prywatno\u015b\u0107 i warunki"},pt:pa,"pt-BR":pa,"pt-PT":{visual_challenge:"Obter um desafio visual",audio_challenge:"Obter um desafio de \u00e1udio", +refresh_btn:"Obter um novo desafio",instructions_visual:"Introduza o texto:",instructions_audio:"Escreva o que ouvir:",help_btn:"Ajuda",play_again:"Reproduzir som novamente",cant_hear_this:"Transferir som como MP3",incorrect_try_again:"Incorreto. Tente novamente.",image_alt_text:"Imagem de teste reCAPTCHA",privacy_and_terms:"Privacidade e Termos de Utiliza\u00e7\u00e3o"},ro:qa,ru:{visual_challenge:"\u0412\u0438\u0437\u0443\u0430\u043b\u044c\u043d\u0430\u044f \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430", +audio_challenge:"\u0417\u0432\u0443\u043a\u043e\u0432\u0430\u044f \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",refresh_btn:"\u041e\u0431\u043d\u043e\u0432\u0438\u0442\u044c",instructions_visual:"\u0412\u0432\u0435\u0434\u0438\u0442\u0435 \u0442\u0435\u043a\u0441\u0442:",instructions_audio:"\u0412\u0432\u0435\u0434\u0438\u0442\u0435 \u0442\u043e, \u0447\u0442\u043e \u0441\u043b\u044b\u0448\u0438\u0442\u0435:",help_btn:"\u0421\u043f\u0440\u0430\u0432\u043a\u0430",play_again:"\u041f\u0440\u043e\u0441\u043b\u0443\u0448\u0430\u0442\u044c \u0435\u0449\u0435 \u0440\u0430\u0437", +cant_hear_this:"\u0417\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044c MP3-\u0444\u0430\u0439\u043b",incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e. \u041f\u043e\u0432\u0442\u043e\u0440\u0438\u0442\u0435 \u043f\u043e\u043f\u044b\u0442\u043a\u0443.",image_alt_text:"\u041f\u0440\u043e\u0432\u0435\u0440\u043a\u0430 \u043f\u043e \u0441\u043b\u043e\u0432\u0443 reCAPTCHA",privacy_and_terms:"\u041f\u0440\u0430\u0432\u0438\u043b\u0430 \u0438 \u043f\u0440\u0438\u043d\u0446\u0438\u043f\u044b"}, +sk:{visual_challenge:"Zobrazi\u0165 vizu\u00e1lnu podobu",audio_challenge:"Prehra\u0165 zvukov\u00fa podobu",refresh_btn:"Zobrazi\u0165 nov\u00fd v\u00fdraz",instructions_visual:"Zadajte text:",instructions_audio:"Zadajte, \u010do po\u010dujete:",help_btn:"Pomocn\u00edk",play_again:"Znova prehra\u0165 zvuk",cant_hear_this:"Prevzia\u0165 zvuk v podobe s\u00faboru MP3",incorrect_try_again:"Nespr\u00e1vne. Sk\u00faste to znova.",image_alt_text:"Obr\u00e1zok zadania reCAPTCHA",privacy_and_terms:"Ochrana osobn\u00fdch \u00fadajov a Zmluvn\u00e9 podmienky"}, +sl:{visual_challenge:"Vizualni preskus",audio_challenge:"Zvo\u010dni preskus",refresh_btn:"Nov preskus",instructions_visual:"Vnesite besedilo:",instructions_audio:"Natipkajte, kaj sli\u0161ite:",help_btn:"Pomo\u010d",play_again:"Znova predvajaj zvok",cant_hear_this:"Prenesi zvok kot MP3",incorrect_try_again:"Napa\u010dno. Poskusite znova.",image_alt_text:"Slika izziva reCAPTCHA",privacy_and_terms:"Zasebnost in pogoji"},sr:{visual_challenge:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u0432\u0438\u0437\u0443\u0435\u043b\u043d\u0438 \u0443\u043f\u0438\u0442", +audio_challenge:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u0430\u0443\u0434\u0438\u043e \u0443\u043f\u0438\u0442",refresh_btn:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u043d\u043e\u0432\u0438 \u0443\u043f\u0438\u0442",instructions_visual:"\u0423\u043d\u0435\u0441\u0438\u0442\u0435 \u0442\u0435\u043a\u0441\u0442:",instructions_audio:"\u041e\u0442\u043a\u0443\u0446\u0430\u0458\u0442\u0435 \u043e\u043d\u043e \u0448\u0442\u043e \u0447\u0443\u0458\u0435\u0442\u0435:",help_btn:"\u041f\u043e\u043c\u043e\u045b", +play_again:"\u041f\u043e\u043d\u043e\u0432\u043e \u043f\u0443\u0441\u0442\u0438 \u0437\u0432\u0443\u043a",cant_hear_this:"\u041f\u0440\u0435\u0443\u0437\u043c\u0438 \u0437\u0432\u0443\u043a \u043a\u0430\u043e MP3 \u0441\u043d\u0438\u043c\u0430\u043a",incorrect_try_again:"\u041d\u0435\u0442\u0430\u0447\u043d\u043e. \u041f\u043e\u043a\u0443\u0448\u0430\u0458\u0442\u0435 \u043f\u043e\u043d\u043e\u0432\u043e.",image_alt_text:"\u0421\u043b\u0438\u043a\u0430 reCAPTCHA \u043f\u0440\u043e\u0432\u0435\u0440\u0435", +privacy_and_terms:"\u041f\u0440\u0438\u0432\u0430\u0442\u043d\u043e\u0441\u0442 \u0438 \u0443\u0441\u043b\u043e\u0432\u0438"},sv:{visual_challenge:"H\u00e4mta captcha i bildformat",audio_challenge:"H\u00e4mta captcha i ljudformat",refresh_btn:"H\u00e4mta ny captcha",instructions_visual:"Skriv texten:",instructions_audio:"Skriv det du h\u00f6r:",help_btn:"Hj\u00e4lp",play_again:"Spela upp ljudet igen",cant_hear_this:"H\u00e4mta ljud som MP3",incorrect_try_again:"Fel. F\u00f6rs\u00f6k igen.",image_alt_text:"reCAPTCHA-bild", +privacy_and_terms:"Sekretess och villkor"},sw:{visual_challenge:"Pata herufi za kusoma",audio_challenge:"Pata herufi za kusikiliza",refresh_btn:"Pata herufi mpya",instructions_visual:"",instructions_audio:"Charaza unachosikia:",help_btn:"Usaidizi",play_again:"Cheza sauti tena",cant_hear_this:"Pakua sauti kama MP3",incorrect_try_again:"Sio sahihi. Jaribu tena.",image_alt_text:"picha ya changamoto ya reCAPTCHA",privacy_and_terms:"Faragha & Masharti"},ta:{visual_challenge:"\u0baa\u0bbe\u0bb0\u0bcd\u0bb5\u0bc8 \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95", +audio_challenge:"\u0b86\u0b9f\u0bbf\u0baf\u0bcb \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",refresh_btn:"\u0baa\u0bc1\u0ba4\u0bbf\u0baf \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",instructions_visual:"",instructions_audio:"\u0b95\u0bc7\u0b9f\u0bcd\u0baa\u0ba4\u0bc8 \u0b9f\u0bc8\u0baa\u0bcd \u0b9a\u0bc6\u0baf\u0bcd\u0b95:",help_btn:"\u0b89\u0ba4\u0bb5\u0bbf",play_again:"\u0b92\u0bb2\u0bbf\u0baf\u0bc8 \u0bae\u0bc0\u0ba3\u0bcd\u0b9f\u0bc1\u0bae\u0bcd \u0b87\u0baf\u0b95\u0bcd\u0b95\u0bc1", +cant_hear_this:"\u0b92\u0bb2\u0bbf\u0baf\u0bc8 MP3 \u0b86\u0b95 \u0baa\u0ba4\u0bbf\u0bb5\u0bbf\u0bb1\u0b95\u0bcd\u0b95\u0bc1\u0b95",incorrect_try_again:"\u0ba4\u0bb5\u0bb1\u0bbe\u0ba9\u0ba4\u0bc1. \u0bae\u0bc0\u0ba3\u0bcd\u0b9f\u0bc1\u0bae\u0bcd \u0bae\u0bc1\u0baf\u0bb2\u0bb5\u0bc1\u0bae\u0bcd.",image_alt_text:"reCAPTCHA \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bcd \u0baa\u0b9f\u0bae\u0bcd",privacy_and_terms:"\u0ba4\u0ba9\u0bbf\u0baf\u0bc1\u0bb0\u0bbf\u0bae\u0bc8 & \u0bb5\u0bbf\u0ba4\u0bbf\u0bae\u0bc1\u0bb1\u0bc8\u0b95\u0bb3\u0bcd"}, +te:{visual_challenge:"\u0c12\u0c15 \u0c26\u0c43\u0c36\u0c4d\u0c2f\u0c2e\u0c3e\u0c28 \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",audio_challenge:"\u0c12\u0c15 \u0c06\u0c21\u0c3f\u0c2f\u0c4b \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",refresh_btn:"\u0c15\u0c4d\u0c30\u0c4a\u0c24\u0c4d\u0c24 \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f", +instructions_visual:"",instructions_audio:"\u0c2e\u0c40\u0c30\u0c41 \u0c35\u0c3f\u0c28\u0c4d\u0c28\u0c26\u0c3f \u0c1f\u0c48\u0c2a\u0c4d \u0c1a\u0c47\u0c2f\u0c02\u0c21\u0c3f:",help_btn:"\u0c38\u0c39\u0c3e\u0c2f\u0c02",play_again:"\u0c27\u0c4d\u0c35\u0c28\u0c3f\u0c28\u0c3f \u0c2e\u0c33\u0c4d\u0c32\u0c40 \u0c2a\u0c4d\u0c32\u0c47 \u0c1a\u0c47\u0c2f\u0c3f",cant_hear_this:"\u0c27\u0c4d\u0c35\u0c28\u0c3f\u0c28\u0c3f MP3 \u0c35\u0c32\u0c46 \u0c21\u0c4c\u0c28\u0c4d\u200c\u0c32\u0c4b\u0c21\u0c4d \u0c1a\u0c47\u0c2f\u0c3f", +incorrect_try_again:"\u0c24\u0c2a\u0c4d\u0c2a\u0c41. \u0c2e\u0c33\u0c4d\u0c32\u0c40 \u0c2a\u0c4d\u0c30\u0c2f\u0c24\u0c4d\u0c28\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f.",image_alt_text:"reCAPTCHA \u0c38\u0c35\u0c3e\u0c32\u0c41 \u0c1a\u0c3f\u0c24\u0c4d\u0c30\u0c02",privacy_and_terms:"\u0c17\u0c4b\u0c2a\u0c4d\u0c2f\u0c24 & \u0c28\u0c3f\u0c2c\u0c02\u0c27\u0c28\u0c32\u0c41"},th:{visual_challenge:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e14\u0e49\u0e32\u0e19\u0e20\u0e32\u0e1e", +audio_challenge:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e14\u0e49\u0e32\u0e19\u0e40\u0e2a\u0e35\u0e22\u0e07",refresh_btn:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e43\u0e2b\u0e21\u0e48",instructions_visual:"\u0e1e\u0e34\u0e21\u0e1e\u0e4c\u0e02\u0e49\u0e2d\u0e04\u0e27\u0e32\u0e21\u0e19\u0e35\u0e49:",instructions_audio:"\u0e1e\u0e34\u0e21\u0e1e\u0e4c\u0e2a\u0e34\u0e48\u0e07\u0e17\u0e35\u0e48\u0e04\u0e38\u0e13\u0e44\u0e14\u0e49\u0e22\u0e34\u0e19:", +help_btn:"\u0e04\u0e27\u0e32\u0e21\u0e0a\u0e48\u0e27\u0e22\u0e40\u0e2b\u0e25\u0e37\u0e2d",play_again:"\u0e40\u0e25\u0e48\u0e19\u0e40\u0e2a\u0e35\u0e22\u0e07\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07",cant_hear_this:"\u0e14\u0e32\u0e27\u0e42\u0e2b\u0e25\u0e14\u0e40\u0e2a\u0e35\u0e22\u0e07\u0e40\u0e1b\u0e47\u0e19 MP3",incorrect_try_again:"\u0e44\u0e21\u0e48\u0e16\u0e39\u0e01\u0e15\u0e49\u0e2d\u0e07 \u0e25\u0e2d\u0e07\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07",image_alt_text:"\u0e23\u0e2b\u0e31\u0e2a\u0e20\u0e32\u0e1e reCAPTCHA", +privacy_and_terms:"\u0e19\u0e42\u0e22\u0e1a\u0e32\u0e22\u0e2a\u0e48\u0e27\u0e19\u0e1a\u0e38\u0e04\u0e04\u0e25\u0e41\u0e25\u0e30\u0e02\u0e49\u0e2d\u0e01\u0e33\u0e2b\u0e19\u0e14"},tr:{visual_challenge:"G\u00f6rsel sorgu al",audio_challenge:"Sesli sorgu al",refresh_btn:"Yeniden y\u00fckle",instructions_visual:"Metni yaz\u0131n:",instructions_audio:"Duydu\u011funuzu yaz\u0131n:",help_btn:"Yard\u0131m",play_again:"Sesi tekrar \u00e7al",cant_hear_this:"Sesi MP3 olarak indir",incorrect_try_again:"Yanl\u0131\u015f. Tekrar deneyin.", +image_alt_text:"reCAPTCHA sorusu resmi",privacy_and_terms:"Gizlilik ve \u015eartlar"},uk:{visual_challenge:"\u041e\u0442\u0440\u0438\u043c\u0430\u0442\u0438 \u0432\u0456\u0437\u0443\u0430\u043b\u044c\u043d\u0438\u0439 \u0442\u0435\u043a\u0441\u0442",audio_challenge:"\u041e\u0442\u0440\u0438\u043c\u0430\u0442\u0438 \u0430\u0443\u0434\u0456\u043e\u0437\u0430\u043f\u0438\u0441",refresh_btn:"\u041e\u043d\u043e\u0432\u0438\u0442\u0438 \u0442\u0435\u043a\u0441\u0442",instructions_visual:"\u0412\u0432\u0435\u0434\u0456\u0442\u044c \u0442\u0435\u043a\u0441\u0442:", +instructions_audio:"\u0412\u0432\u0435\u0434\u0456\u0442\u044c \u043f\u043e\u0447\u0443\u0442\u0435:",help_btn:"\u0414\u043e\u0432\u0456\u0434\u043a\u0430",play_again:"\u0412\u0456\u0434\u0442\u0432\u043e\u0440\u0438\u0442\u0438 \u0437\u0430\u043f\u0438\u0441 \u0449\u0435 \u0440\u0430\u0437",cant_hear_this:"\u0417\u0430\u0432\u0430\u043d\u0442\u0430\u0436\u0438\u0442\u0438 \u0437\u0430\u043f\u0438\u0441 \u044f\u043a MP3",incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e. \u0421\u043f\u0440\u043e\u0431\u0443\u0439\u0442\u0435 \u0449\u0435 \u0440\u0430\u0437.", +image_alt_text:"\u0417\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u043d\u044f \u0437\u0430\u0432\u0434\u0430\u043d\u043d\u044f reCAPTCHA",privacy_and_terms:"\u041a\u043e\u043d\u0444\u0456\u0434\u0435\u043d\u0446\u0456\u0439\u043d\u0456\u0441\u0442\u044c \u0456 \u0443\u043c\u043e\u0432\u0438"},ur:{visual_challenge:"\u0627\u06cc\u06a9 \u0645\u0631\u0626\u06cc \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",audio_challenge:"\u0627\u06cc\u06a9 \u0622\u0688\u06cc\u0648 \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba", +refresh_btn:"\u0627\u06cc\u06a9 \u0646\u06cc\u0627 \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",instructions_visual:"",instructions_audio:"\u062c\u0648 \u0633\u0646\u0627\u0626\u06cc \u062f\u06cc\u062a\u0627 \u06c1\u06d2 \u0648\u06c1 \u0679\u0627\u0626\u067e \u06a9\u0631\u06cc\u06ba:",help_btn:"\u0645\u062f\u062f",play_again:"\u0622\u0648\u0627\u0632 \u062f\u0648\u0628\u0627\u0631\u06c1 \u0686\u0644\u0627\u0626\u06cc\u06ba",cant_hear_this:"\u0622\u0648\u0627\u0632 \u06a9\u0648 MP3 \u06a9\u06d2 \u0628\u0637\u0648\u0631 \u0688\u0627\u0624\u0646 \u0644\u0648\u0688 \u06a9\u0631\u06cc\u06ba", +incorrect_try_again:"\u063a\u0644\u0637\u06d4 \u062f\u0648\u0628\u0627\u0631\u06c1 \u06a9\u0648\u0634\u0634 \u06a9\u0631\u06cc\u06ba\u06d4",image_alt_text:"reCAPTCHA \u0686\u06cc\u0644\u0646\u062c \u0648\u0627\u0644\u06cc \u0634\u0628\u06cc\u06c1",privacy_and_terms:"\u0631\u0627\u0632\u062f\u0627\u0631\u06cc \u0648 \u0634\u0631\u0627\u0626\u0637"},vi:{visual_challenge:"Nh\u1eadn th\u1eed th\u00e1ch h\u00ecnh \u1ea3nh",audio_challenge:"Nh\u1eadn th\u1eed th\u00e1ch \u00e2m thanh",refresh_btn:"Nh\u1eadn th\u1eed th\u00e1ch m\u1edbi", +instructions_visual:"Nh\u1eadp v\u0103n b\u1ea3n:",instructions_audio:"Nh\u1eadp n\u1ed9i dung b\u1ea1n nghe th\u1ea5y:",help_btn:"Tr\u1ee3 gi\u00fap",play_again:"Ph\u00e1t l\u1ea1i \u00e2m thanh",cant_hear_this:"T\u1ea3i \u00e2m thanh xu\u1ed1ng d\u01b0\u1edbi d\u1ea1ng MP3",incorrect_try_again:"Kh\u00f4ng ch\u00ednh x\u00e1c. H\u00e3y th\u1eed l\u1ea1i.",image_alt_text:"H\u00ecnh x\u00e1c th\u1ef1c reCAPTCHA",privacy_and_terms:"B\u1ea3o m\u1eadt v\u00e0 \u0111i\u1ec1u kho\u1ea3n"},"zh-CN":ra,"zh-HK":{visual_challenge:"\u56de\u7b54\u5716\u50cf\u9a57\u8b49\u554f\u984c", +audio_challenge:"\u53d6\u5f97\u8a9e\u97f3\u9a57\u8b49\u554f\u984c",refresh_btn:"\u63db\u4e00\u500b\u9a57\u8b49\u554f\u984c",instructions_visual:"\u8f38\u5165\u6587\u5b57\uff1a",instructions_audio:"\u9375\u5165\u60a8\u6240\u807d\u5230\u7684\uff1a",help_btn:"\u8aaa\u660e",play_again:"\u518d\u6b21\u64ad\u653e\u8072\u97f3",cant_hear_this:"\u5c07\u8072\u97f3\u4e0b\u8f09\u70ba MP3",incorrect_try_again:"\u4e0d\u6b63\u78ba\uff0c\u518d\u8a66\u4e00\u6b21\u3002",image_alt_text:"reCAPTCHA \u9a57\u8b49\u6587\u5b57\u5716\u7247", +privacy_and_terms:"\u79c1\u96b1\u6b0a\u8207\u689d\u6b3e"},"zh-TW":{visual_challenge:"\u53d6\u5f97\u5716\u7247\u9a57\u8b49\u554f\u984c",audio_challenge:"\u53d6\u5f97\u8a9e\u97f3\u9a57\u8b49\u554f\u984c",refresh_btn:"\u53d6\u5f97\u65b0\u7684\u9a57\u8b49\u554f\u984c",instructions_visual:"\u8acb\u8f38\u5165\u5716\u7247\u4e2d\u7684\u6587\u5b57\uff1a",instructions_audio:"\u8acb\u8f38\u5165\u8a9e\u97f3\u5167\u5bb9\uff1a",help_btn:"\u8aaa\u660e",play_again:"\u518d\u6b21\u64ad\u653e",cant_hear_this:"\u4ee5 MP3 \u683c\u5f0f\u4e0b\u8f09\u8072\u97f3", +incorrect_try_again:"\u9a57\u8b49\u78bc\u6709\u8aa4\uff0c\u8acb\u518d\u8a66\u4e00\u6b21\u3002",image_alt_text:"reCAPTCHA \u9a57\u8b49\u6587\u5b57\u5716\u7247",privacy_and_terms:"\u96b1\u79c1\u6b0a\u8207\u689d\u6b3e"},zu:{visual_challenge:"Thola inselelo ebonakalayo",audio_challenge:"Thola inselelo yokulalelwayo",refresh_btn:"Thola inselelo entsha",instructions_visual:"",instructions_audio:"Bhala okuzwayo:",help_btn:"Usizo",play_again:"Phinda udlale okulalelwayo futhi",cant_hear_this:"Layisha umsindo njenge-MP3", +incorrect_try_again:"Akulungile. Zama futhi.",image_alt_text:"umfanekiso oyinselelo we-reCAPTCHA",privacy_and_terms:"Okwangasese kanye nemigomo"},tl:la,he:oa,"in":na,mo:qa,zh:ra};var x=function(a){if(Error.captureStackTrace)Error.captureStackTrace(this,x);else{var b=Error().stack;b&&(this.stack=b)}a&&(this.message=String(a))};u(x,Error);x.prototype.name="CustomError";var ta;var ua=function(a,b){for(var c=a.split("%s"),d="",e=Array.prototype.slice.call(arguments,1);e.length&&1")&&(a=a.replace(za,">"));-1!=a.indexOf('"')&&(a=a.replace(Aa,"""));-1!=a.indexOf("'")&& +(a=a.replace(Ba,"'"));-1!=a.indexOf("\x00")&&(a=a.replace(Ca,"�"));return a},xa=/&/g,ya=//g,Aa=/"/g,Ba=/'/g,Ca=/\x00/g,wa=/[\x00&<>"']/,Ea=function(a,b){return ab?1:0},Fa=function(a){return String(a).replace(/\-([a-z])/g,function(a,c){return c.toUpperCase()})},Ga=function(a){var b=q(void 0)?"undefined".replace(/([-()\[\]{}+?*.$\^|,:#c?Math.max(0,a.length+c):c;if(q(a))return q(b)&&1==b.length?a.indexOf(b,c):-1;for(;c=arguments.length?z.slice.call(a,b):z.slice.call(a,b,c)};var Ta=function(a,b){for(var c in a)b.call(void 0,a[c],c,a)},Ua=function(a){var b=[],c=0,d;for(d in a)b[c++]=d;return b},Va=function(a){for(var b in a)return!1;return!0},Xa=function(){var a=Wa()?k.google_ad:null,b={},c;for(c in a)b[c]=a[c];return b},Ya="constructor hasOwnProperty isPrototypeOf propertyIsEnumerable toLocaleString toString valueOf".split(" "),Za=function(a,b){for(var c,d,e=1;eparseFloat(a))?String(b):a}(),fb={},F=function(a){var b; +if(!(b=fb[a])){b=0;for(var c=va(String(eb)).split("."),d=va(String(a)).split("."),e=Math.max(c.length,d.length),g=0;0==b&&ga.length)return String.fromCharCode.apply(null,a);for(var b="",c=0;cb;b++)kb[b]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".charAt(b),lb[kb[b]]=b,62<=b&&(lb["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.".charAt(b)]=b)}for(var b=lb,c=[],d=0;d> +4);64!=f&&(c.push(g<<4&240|f>>2),64!=m&&c.push(f<<6&192|m))}return c};var H=function(){this.disposed_=this.disposed_;this.onDisposeCallbacks_=this.onDisposeCallbacks_};H.prototype.disposed_=!1;H.prototype.dispose=function(){this.disposed_||(this.disposed_=!0,this.disposeInternal())};var nb=function(a,b){a.onDisposeCallbacks_||(a.onDisposeCallbacks_=[]);a.onDisposeCallbacks_.push(l(void 0)?s(b,void 0):b)};H.prototype.disposeInternal=function(){if(this.onDisposeCallbacks_)for(;this.onDisposeCallbacks_.length;)this.onDisposeCallbacks_.shift()()}; +var ob=function(a){a&&"function"==typeof a.dispose&&a.dispose()};var pb=!C||C&&9<=hb;!D&&!C||C&&C&&9<=hb||D&&F("1.9.1");C&&F("9");var sb=function(a){return a?new qb(rb(a)):ta||(ta=new qb)},tb=function(a,b){return q(b)?a.getElementById(b):b},vb=function(a,b){Ta(b,function(b,d){"style"==d?a.style.cssText=b:"class"==d?a.className=b:"for"==d?a.htmlFor=b:d in ub?a.setAttribute(ub[d],b):0==d.lastIndexOf("aria-",0)||0==d.lastIndexOf("data-",0)?a.setAttribute(d,b):a[d]=b})},ub={cellpadding:"cellPadding",cellspacing:"cellSpacing",colspan:"colSpan",frameborder:"frameBorder",height:"height",maxlength:"maxLength",role:"role",rowspan:"rowSpan", +type:"type",usemap:"useMap",valign:"vAlign",width:"width"},xb=function(a,b,c){return wb(document,arguments)},wb=function(a,b){var c=b[0],d=b[1];if(!pb&&d&&(d.name||d.type)){c=["<",c];d.name&&c.push(' name="',Da(d.name),'"');if(d.type){c.push(' type="',Da(d.type),'"');var e={};Za(e,d);delete e.type;d=e}c.push(">");c=c.join("")}c=a.createElement(c);d&&(q(d)?c.className=d:p(d)?c.className=d.join(" "):vb(c,d));2=b.branches_&&b.cancel())}this.onCancelFunction_?this.onCancelFunction_.call(this.defaultScope_,this):this.silentlyCanceled_=!0;this.fired_||Wb(this,new Xb)}};M.prototype.continue_=function(a,b){this.blocked_=!1;Yb(this,a,b)}; +var Yb=function(a,b,c){a.fired_=!0;a.result_=c;a.hadError_=!b;Zb(a)},ac=function(a){if(a.fired_){if(!a.silentlyCanceled_)throw new $b;a.silentlyCanceled_=!1}};M.prototype.callback=function(a){ac(this);bc(a);Yb(this,!0,a)};var Wb=function(a,b){ac(a);bc(b);Yb(a,!1,b)},bc=function(a){y(!(a instanceof M),"An execution sequence may not be initiated with a blocking Deferred.")},cc=function(a,b,c,d){y(!a.blocking_,"Blocking Deferreds can not be re-used");a.sequence_.push([b,c,d]);a.fired_&&Zb(a)}; +M.prototype.then=function(a,b,c){var d,e,g=new L(function(a,b){d=a;e=b});cc(this,d,function(a){a instanceof Xb?g.cancel():e(a)});return g.then(a,b,c)};Mb(M); +var dc=function(a){return Pa(a.sequence_,function(a){return r(a[1])})},Zb=function(a){if(a.unhandledErrorId_&&a.fired_&&dc(a)){var b=a.unhandledErrorId_,c=ec[b];c&&(k.clearTimeout(c.id_),delete ec[b]);a.unhandledErrorId_=0}a.parent_&&(a.parent_.branches_--,delete a.parent_);for(var b=a.result_,d=c=!1;a.sequence_.length&&!a.blocked_;){var e=a.sequence_.shift(),g=e[0],f=e[1],e=e[2];if(g=a.hadError_?f:g)try{var m=g.call(e||a.defaultScope_,b);l(m)&&(a.hadError_=a.hadError_&&(m==b||m instanceof Error), +a.result_=b=m);Nb(b)&&(d=!0,a.blocked_=!0)}catch($){b=$,a.hadError_=!0,dc(a)||(c=!0)}}a.result_=b;d&&(m=s(a.continue_,a,!0),d=s(a.continue_,a,!1),b instanceof M?(cc(b,m,d),b.blocking_=!0):b.then(m,d));c&&(b=new fc(b),ec[b.id_]=b,a.unhandledErrorId_=b.id_)},$b=function(){x.call(this)};u($b,x);$b.prototype.message="Deferred has already fired";$b.prototype.name="AlreadyCalledError";var Xb=function(){x.call(this)};u(Xb,x);Xb.prototype.message="Deferred was canceled";Xb.prototype.name="CanceledError"; +var fc=function(a){this.id_=k.setTimeout(s(this.throwError,this),0);this.error_=a};fc.prototype.throwError=function(){y(ec[this.id_],"Cannot throw an error that is not scheduled.");delete ec[this.id_];throw this.error_;};var ec={};var kc=function(a){var b={},c=b.document||document,d=document.createElement("SCRIPT"),e={script_:d,timeout_:void 0},g=new M(gc,e),f=null,m=null!=b.timeout?b.timeout:5E3;0=a.keyCode)a.keyCode=-1}catch(b){}};O.prototype.disposeInternal=function(){};var oc="closure_listenable_"+(1E6*Math.random()|0),pc=0;var qc=function(a,b,c,d,e){this.listener=a;this.proxy=null;this.src=b;this.type=c;this.capture=!!d;this.handler=e;this.key=++pc;this.removed=this.callOnce=!1},rc=function(a){a.removed=!0;a.listener=null;a.proxy=null;a.src=null;a.handler=null};var P=function(a){this.src=a;this.listeners={};this.typeCount_=0};P.prototype.add=function(a,b,c,d,e){var g=a.toString();a=this.listeners[g];a||(a=this.listeners[g]=[],this.typeCount_++);var f=sc(a,b,d,e);-1e.keyCode||void 0!=e.returnValue)){t:{var g=!1;if(0==e.keyCode)try{e.keyCode=-1;break t}catch(f){g=!0}if(g||void 0==e.returnValue)e.returnValue=!0}e=[];for(g=c.currentTarget;g;g=g.parentNode)e.push(g);for(var g=a.type,m=e.length-1;!c.propagationStopped_&&0<=m;m--)c.currentTarget=e[m],d&=Gc(e[m],g,!0,c);for(m=0;!c.propagationStopped_&&m>>0),yc=function(a){y(a,"Listener can not be null.");if(r(a))return a;y(a.handleEvent,"An object listener must have handleEvent method.");a[Hc]||(a[Hc]=function(b){return a.handleEvent(b)});return a[Hc]};var Q=function(a){H.call(this);this.handler_=a;this.keys_={}};u(Q,H);var Ic=[];h=Q.prototype;h.listen=function(a,b,c,d){p(b)||(b&&(Ic[0]=b.toString()),b=Ic);for(var e=0;e++d,"infinite loop")}c=this.actualEventTarget_;d=a.type||a;if(q(a))a=new N(a,c);else if(a instanceof N)a.target=a.target||c;else{var e=a;a=new N(d,c);Za(a,e)}var e=!0,g;if(b)for(var f=b.length-1;!a.propagationStopped_&&0<=f;f--)g=a.currentTarget=b[f],e=Kc(g,d,!0,a)&&e;a.propagationStopped_||(g=a.currentTarget=c,e=Kc(g,d,!0,a)&&e,a.propagationStopped_||(e=Kc(g,d,!1,a)&& +e));if(b)for(f=0;!a.propagationStopped_&&f
AdChoices
AdChoices
'))}; +var Rc=function(a){var b=Sc(a,"visibility");a=Sc(a,"display");return"hidden"!=b&&"none"!=a},Sc=function(a,b){var c;t:{c=rb(a);if(c.defaultView&&c.defaultView.getComputedStyle&&(c=c.defaultView.getComputedStyle(a,null))){c=c[b]||c.getPropertyValue(b)||"";break t}c=""}if(!(c=c||(a.currentStyle?a.currentStyle[b]:null))&&(c=a.style[Fa(b)],"undefined"===typeof c)){c=a.style;var d;t:if(d=Fa(b),void 0===a.style[d]){var e=(E?"Webkit":D?"Moz":C?"ms":cb?"O":null)+Ga(d);if(void 0!==a.style[e]){d=e;break t}}c= +c[d]||""}return c};V.prototype.disposeInternal=function(){delete this.captchaImage_;delete this.adImage_;V.superClass_.disposeInternal.call(this)};var Tc=function(a,b,c){H.call(this);this.listener_=a;this.interval_=b||0;this.handler_=c;this.callback_=s(this.doAction_,this)};u(Tc,H);h=Tc.prototype;h.id_=0;h.disposeInternal=function(){Tc.superClass_.disposeInternal.call(this);this.stop();delete this.listener_;delete this.handler_}; +h.start=function(a){this.stop();var b=this.callback_;a=l(a)?a:this.interval_;if(!r(b))if(b&&"function"==typeof b.handleEvent)b=s(b.handleEvent,b);else throw Error("Invalid listener argument");this.id_=2147483647c?-1:ha()+c;this.delay_.start(0>c?this.interval_:Math.min(this.interval_,c))};h.stop=function(){this.delay_.stop()}; +h.isActive=function(){return this.delay_.isActive()};h.onSuccess=function(){};h.onFailure=function(){};h.onTick_=function(){if(this.listener_.call(this.handler_))this.onSuccess();else if(0>this.runUntil_)this.delay_.start(this.interval_);else{var a=this.runUntil_-ha();if(0>=a)this.onFailure();else this.delay_.start(Math.min(this.interval_,a))}};$a("area base br col command embed hr img input keygen link meta param source track wbr".split(" "));$a("action","cite","data","formaction","href","manifest","poster","src");$a("link","script","style");var Vc={sanitizedContentKindHtml:!0},Wc={sanitizedContentKindText:!0},Xc=function(){throw Error("Do not instantiate directly");};Xc.prototype.contentDir=null;Xc.prototype.toString=function(){return this.content};var bd=function(a){var b=Yc;y(b,"Soy template may not be null.");var c=sb().createElement("DIV");a=Zc(b(a||$c,void 0,void 0));b=a.match(ad);y(!b,"This template starts with a %s, which cannot be a child of a
, as required by soy internals. Consider using goog.soy.renderElement instead.\nTemplate output: %s",b&&b[0],a);c.innerHTML=a;return 1==c.childNodes.length&&(a=c.firstChild,1==a.nodeType)?a:c},Zc=function(a){if(!da(a))return String(a);if(a instanceof Xc){if(a.contentKind===Vc)return Ka(a.content); +if(a.contentKind===Wc)return Da(a.content)}Ja("Soy template output is unsafe for use as HTML: "+a);return"zSoyz"},ad=/^<(body|caption|col|colgroup|head|html|tr|td|tbody|thead|tfoot)>/i,$c={};C&&F(8);var cd=function(){Xc.call(this)};u(cd,Xc);cd.prototype.contentKind=Vc;var dd=function(a){function b(a){this.content=a}b.prototype=a.prototype;return function(a,d){var e=new b(String(a));void 0!==d&&(e.contentDir=d);return e}}(cd);(function(a){function b(a){this.content=a}b.prototype=a.prototype;return function(a,d){var e=String(a);if(!e)return"";e=new b(e);void 0!==d&&(e.contentDir=d);return e}})(cd); +var ed={"\x00":"\\x00","\b":"\\x08","\t":"\\t","\n":"\\n","\x0B":"\\x0b","\f":"\\f","\r":"\\r",'"':"\\x22",$:"\\x24","&":"\\x26","'":"\\x27","(":"\\x28",")":"\\x29","*":"\\x2a","+":"\\x2b",",":"\\x2c","-":"\\x2d",".":"\\x2e","/":"\\/",":":"\\x3a","<":"\\x3c","=":"\\x3d",">":"\\x3e","?":"\\x3f","[":"\\x5b","\\":"\\\\","]":"\\x5d","^":"\\x5e","{":"\\x7b","|":"\\x7c","}":"\\x7d","\u0085":"\\x85","\u2028":"\\u2028","\u2029":"\\u2029"},fd=function(a){return ed[a]},gd=/[\x00\x08-\x0d\x22\x26\x27\/\x3c-\x3e\\\x85\u2028\u2029]/g;var Yc=function(a){return dd(' @@ -18,16 +19,11 @@

+ +
+
-
- - -
- -
- - -
Your home server stores all your conversation and account data.
+
diff --git a/static/client/register/js/register.js b/static/client/register/js/register.js index b8a9247e6..38408f731 100644 --- a/static/client/register/js/register.js +++ b/static/client/register/js/register.js @@ -1,23 +1,116 @@ -window.matrixRegistration = {}; +window.matrixRegistration = { + endpoint: location.origin + "/_matrix/client/api/v1/register" +}; var setupCaptcha = function() { - if (!window.matrixRegistrationConfig) { - return; - } - console.log("Setting up ReCaptcha"); - var public_key = window.matrixRegistrationConfig.recaptcha_public_key; - if (public_key === undefined) { - console.error("No public key defined for captcha!"); + if (!window.matrixRegistrationConfig) { return; } - Recaptcha.create(public_key, - "regcaptcha", - { - theme: "red", - callback: Recaptcha.focus_response_field + $.get(matrixRegistration.endpoint, function(response) { + var serverExpectsCaptcha = false; + for (var i=0; i "+JSON.stringify(response)); + submitPassword(user, pwd, response.session); + }).error(function(err) { + Recaptcha.reload(); + errorFunc(err); }); }; +var submitPassword = function(user, pwd, session) { + console.log("Registering..."); + var data = { + type: "m.login.password", + user: user, + password: pwd, + session: session + }; + $.post(matrixRegistration.endpoint, JSON.stringify(data), function(response) { + matrixRegistration.onRegistered( + response.home_server, response.user_id, response.access_token + ); + }).error(errorFunc); +}; + +var errorFunc = function(err) { + if (err.responseJSON && err.responseJSON.error) { + setFeedbackString(err.responseJSON.error + " (" + err.responseJSON.errcode + ")"); + } + else { + setFeedbackString("Request failed: " + err.status); + } +}; + +var setFeedbackString = function(text) { + $("#feedback").text(text); +}; + matrixRegistration.onLoad = function() { - setupCaptcha(); -}; \ No newline at end of file + setupCaptcha(); +}; + +matrixRegistration.signUp = function() { + var user = $("#desired_user_id").val(); + if (user.length == 0) { + setFeedbackString("Must specify a username."); + return; + } + var pwd1 = $("#pwd1").val(); + var pwd2 = $("#pwd2").val(); + if (pwd1.length < 6) { + setFeedbackString("Password: min. 6 characters."); + return; + } + if (pwd1 != pwd2) { + setFeedbackString("Passwords do not match."); + return; + } + if (window.matrixRegistration.isUsingRecaptcha) { + submitCaptcha(user, pwd1); + } + else { + submitPassword(user, pwd1); + } +}; + +matrixRegistration.onRegistered = function(hs_url, user_id, access_token) { + // clobber this function + console.log("onRegistered - This function should be replaced to proceed."); +}; From f30d4d5308a2ce7e535eea01d0f0a74181820359 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 23 Feb 2015 17:12:15 +0000 Subject: [PATCH 521/630] Add jquery --- static/client/register/js/jquery-2.1.3.min.js | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 static/client/register/js/jquery-2.1.3.min.js diff --git a/static/client/register/js/jquery-2.1.3.min.js b/static/client/register/js/jquery-2.1.3.min.js new file mode 100644 index 000000000..25714ed29 --- /dev/null +++ b/static/client/register/js/jquery-2.1.3.min.js @@ -0,0 +1,4 @@ +/*! jQuery v2.1.3 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ +!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k={},l=a.document,m="2.1.3",n=function(a,b){return new n.fn.init(a,b)},o=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,p=/^-ms-/,q=/-([\da-z])/gi,r=function(a,b){return b.toUpperCase()};n.fn=n.prototype={jquery:m,constructor:n,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=n.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return n.each(this,a,b)},map:function(a){return this.pushStack(n.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},n.extend=n.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||n.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(a=arguments[h]))for(b in a)c=g[b],d=a[b],g!==d&&(j&&d&&(n.isPlainObject(d)||(e=n.isArray(d)))?(e?(e=!1,f=c&&n.isArray(c)?c:[]):f=c&&n.isPlainObject(c)?c:{},g[b]=n.extend(j,f,d)):void 0!==d&&(g[b]=d));return g},n.extend({expando:"jQuery"+(m+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===n.type(a)},isArray:Array.isArray,isWindow:function(a){return null!=a&&a===a.window},isNumeric:function(a){return!n.isArray(a)&&a-parseFloat(a)+1>=0},isPlainObject:function(a){return"object"!==n.type(a)||a.nodeType||n.isWindow(a)?!1:a.constructor&&!j.call(a.constructor.prototype,"isPrototypeOf")?!1:!0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(a){var b,c=eval;a=n.trim(a),a&&(1===a.indexOf("use strict")?(b=l.createElement("script"),b.text=a,l.head.appendChild(b).parentNode.removeChild(b)):c(a))},camelCase:function(a){return a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=s(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(o,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(s(Object(a))?n.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){return null==b?-1:g.call(b,a,c)},merge:function(a,b){for(var c=+b.length,d=0,e=a.length;c>d;d++)a[e++]=b[d];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=s(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(c=a[b],b=a,a=c),n.isFunction(a)?(e=d.call(arguments,2),f=function(){return a.apply(b||this,e.concat(d.call(arguments)))},f.guid=a.guid=a.guid||n.guid++,f):void 0},now:Date.now,support:k}),n.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function s(a){var b=a.length,c=n.type(a);return"function"===c||n.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var t=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=hb(),z=hb(),A=hb(),B=function(a,b){return a===b&&(l=!0),0},C=1<<31,D={}.hasOwnProperty,E=[],F=E.pop,G=E.push,H=E.push,I=E.slice,J=function(a,b){for(var c=0,d=a.length;d>c;c++)if(a[c]===b)return c;return-1},K="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",L="[\\x20\\t\\r\\n\\f]",M="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",N=M.replace("w","w#"),O="\\["+L+"*("+M+")(?:"+L+"*([*^$|!~]?=)"+L+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+N+"))|)"+L+"*\\]",P=":("+M+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+O+")*)|.*)\\)|)",Q=new RegExp(L+"+","g"),R=new RegExp("^"+L+"+|((?:^|[^\\\\])(?:\\\\.)*)"+L+"+$","g"),S=new RegExp("^"+L+"*,"+L+"*"),T=new RegExp("^"+L+"*([>+~]|"+L+")"+L+"*"),U=new RegExp("="+L+"*([^\\]'\"]*?)"+L+"*\\]","g"),V=new RegExp(P),W=new RegExp("^"+N+"$"),X={ID:new RegExp("^#("+M+")"),CLASS:new RegExp("^\\.("+M+")"),TAG:new RegExp("^("+M.replace("w","w*")+")"),ATTR:new RegExp("^"+O),PSEUDO:new RegExp("^"+P),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+L+"*(even|odd|(([+-]|)(\\d*)n|)"+L+"*(?:([+-]|)"+L+"*(\\d+)|))"+L+"*\\)|)","i"),bool:new RegExp("^(?:"+K+")$","i"),needsContext:new RegExp("^"+L+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+L+"*((?:-\\d)?\\d*)"+L+"*\\)|)(?=[^-]|$)","i")},Y=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,$=/^[^{]+\{\s*\[native \w/,_=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ab=/[+~]/,bb=/'|\\/g,cb=new RegExp("\\\\([\\da-f]{1,6}"+L+"?|("+L+")|.)","ig"),db=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},eb=function(){m()};try{H.apply(E=I.call(v.childNodes),v.childNodes),E[v.childNodes.length].nodeType}catch(fb){H={apply:E.length?function(a,b){G.apply(a,I.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function gb(a,b,d,e){var f,h,j,k,l,o,r,s,w,x;if((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,d=d||[],k=b.nodeType,"string"!=typeof a||!a||1!==k&&9!==k&&11!==k)return d;if(!e&&p){if(11!==k&&(f=_.exec(a)))if(j=f[1]){if(9===k){if(h=b.getElementById(j),!h||!h.parentNode)return d;if(h.id===j)return d.push(h),d}else if(b.ownerDocument&&(h=b.ownerDocument.getElementById(j))&&t(b,h)&&h.id===j)return d.push(h),d}else{if(f[2])return H.apply(d,b.getElementsByTagName(a)),d;if((j=f[3])&&c.getElementsByClassName)return H.apply(d,b.getElementsByClassName(j)),d}if(c.qsa&&(!q||!q.test(a))){if(s=r=u,w=b,x=1!==k&&a,1===k&&"object"!==b.nodeName.toLowerCase()){o=g(a),(r=b.getAttribute("id"))?s=r.replace(bb,"\\$&"):b.setAttribute("id",s),s="[id='"+s+"'] ",l=o.length;while(l--)o[l]=s+rb(o[l]);w=ab.test(a)&&pb(b.parentNode)||b,x=o.join(",")}if(x)try{return H.apply(d,w.querySelectorAll(x)),d}catch(y){}finally{r||b.removeAttribute("id")}}}return i(a.replace(R,"$1"),b,d,e)}function hb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ib(a){return a[u]=!0,a}function jb(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function kb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function lb(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||C)-(~a.sourceIndex||C);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function mb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function nb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function ob(a){return ib(function(b){return b=+b,ib(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function pb(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=gb.support={},f=gb.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=gb.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=g.documentElement,e=g.defaultView,e&&e!==e.top&&(e.addEventListener?e.addEventListener("unload",eb,!1):e.attachEvent&&e.attachEvent("onunload",eb)),p=!f(g),c.attributes=jb(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=jb(function(a){return a.appendChild(g.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=$.test(g.getElementsByClassName),c.getById=jb(function(a){return o.appendChild(a).id=u,!g.getElementsByName||!g.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=$.test(g.querySelectorAll))&&(jb(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+L+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+L+"*(?:value|"+K+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),jb(function(a){var b=g.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+L+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=$.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&jb(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",P)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=$.test(o.compareDocumentPosition),t=b||$.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===g||a.ownerDocument===v&&t(v,a)?-1:b===g||b.ownerDocument===v&&t(v,b)?1:k?J(k,a)-J(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,h=[a],i=[b];if(!e||!f)return a===g?-1:b===g?1:e?-1:f?1:k?J(k,a)-J(k,b):0;if(e===f)return lb(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)i.unshift(c);while(h[d]===i[d])d++;return d?lb(h[d],i[d]):h[d]===v?-1:i[d]===v?1:0},g):n},gb.matches=function(a,b){return gb(a,null,null,b)},gb.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(U,"='$1']"),!(!c.matchesSelector||!p||r&&r.test(b)||q&&q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return gb(b,n,null,[a]).length>0},gb.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},gb.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&D.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},gb.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},gb.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=gb.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=gb.selectors={cacheLength:50,createPseudo:ib,match:X,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(cb,db),a[3]=(a[3]||a[4]||a[5]||"").replace(cb,db),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||gb.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&gb.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return X.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&V.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(cb,db).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+L+")"+a+"("+L+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=gb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(Q," ")+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){k=q[u]||(q[u]={}),j=k[a]||[],n=j[0]===w&&j[1],m=j[0]===w&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[w,n,m];break}}else if(s&&(j=(b[u]||(b[u]={}))[a])&&j[0]===w)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(s&&((l[u]||(l[u]={}))[a]=[w,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||gb.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ib(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=J(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ib(function(a){var b=[],c=[],d=h(a.replace(R,"$1"));return d[u]?ib(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ib(function(a){return function(b){return gb(a,b).length>0}}),contains:ib(function(a){return a=a.replace(cb,db),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ib(function(a){return W.test(a||"")||gb.error("unsupported lang: "+a),a=a.replace(cb,db).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Z.test(a.nodeName)},input:function(a){return Y.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:ob(function(){return[0]}),last:ob(function(a,b){return[b-1]}),eq:ob(function(a,b,c){return[0>c?c+b:c]}),even:ob(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:ob(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:ob(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:ob(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function sb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[u]||(b[u]={}),(h=i[d])&&h[0]===w&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function tb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function ub(a,b,c){for(var d=0,e=b.length;e>d;d++)gb(a,b[d],c);return c}function vb(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function wb(a,b,c,d,e,f){return d&&!d[u]&&(d=wb(d)),e&&!e[u]&&(e=wb(e,f)),ib(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||ub(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:vb(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=vb(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?J(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=vb(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):H.apply(g,r)})}function xb(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=sb(function(a){return a===b},h,!0),l=sb(function(a){return J(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];f>i;i++)if(c=d.relative[a[i].type])m=[sb(tb(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return wb(i>1&&tb(m),i>1&&rb(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(R,"$1"),c,e>i&&xb(a.slice(i,e)),f>e&&xb(a=a.slice(e)),f>e&&rb(a))}m.push(c)}return tb(m)}function yb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,m,o,p=0,q="0",r=f&&[],s=[],t=j,u=f||e&&d.find.TAG("*",k),v=w+=null==t?1:Math.random()||.1,x=u.length;for(k&&(j=g!==n&&g);q!==x&&null!=(l=u[q]);q++){if(e&&l){m=0;while(o=a[m++])if(o(l,g,h)){i.push(l);break}k&&(w=v)}c&&((l=!o&&l)&&p--,f&&r.push(l))}if(p+=q,c&&q!==p){m=0;while(o=b[m++])o(r,s,g,h);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=F.call(i));s=vb(s)}H.apply(i,s),k&&!f&&s.length>0&&p+b.length>1&&gb.uniqueSort(i)}return k&&(w=v,j=t),r};return c?ib(f):f}return h=gb.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=xb(b[c]),f[u]?d.push(f):e.push(f);f=A(a,yb(e,d)),f.selector=a}return f},i=gb.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(cb,db),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=X.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(cb,db),ab.test(j[0].type)&&pb(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&rb(j),!a)return H.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,ab.test(a)&&pb(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=jb(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),jb(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||kb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&jb(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||kb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),jb(function(a){return null==a.getAttribute("disabled")})||kb(K,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),gb}(a);n.find=t,n.expr=t.selectors,n.expr[":"]=n.expr.pseudos,n.unique=t.uniqueSort,n.text=t.getText,n.isXMLDoc=t.isXML,n.contains=t.contains;var u=n.expr.match.needsContext,v=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,w=/^.[^:#\[\.,]*$/;function x(a,b,c){if(n.isFunction(b))return n.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return n.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(w.test(b))return n.filter(b,a,c);b=n.filter(b,a)}return n.grep(a,function(a){return g.call(b,a)>=0!==c})}n.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?n.find.matchesSelector(d,a)?[d]:[]:n.find.matches(a,n.grep(b,function(a){return 1===a.nodeType}))},n.fn.extend({find:function(a){var b,c=this.length,d=[],e=this;if("string"!=typeof a)return this.pushStack(n(a).filter(function(){for(b=0;c>b;b++)if(n.contains(e[b],this))return!0}));for(b=0;c>b;b++)n.find(a,e[b],d);return d=this.pushStack(c>1?n.unique(d):d),d.selector=this.selector?this.selector+" "+a:a,d},filter:function(a){return this.pushStack(x(this,a||[],!1))},not:function(a){return this.pushStack(x(this,a||[],!0))},is:function(a){return!!x(this,"string"==typeof a&&u.test(a)?n(a):a||[],!1).length}});var y,z=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,A=n.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a[0]&&">"===a[a.length-1]&&a.length>=3?[null,a,null]:z.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||y).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof n?b[0]:b,n.merge(this,n.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:l,!0)),v.test(c[1])&&n.isPlainObject(b))for(c in b)n.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}return d=l.getElementById(c[2]),d&&d.parentNode&&(this.length=1,this[0]=d),this.context=l,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):n.isFunction(a)?"undefined"!=typeof y.ready?y.ready(a):a(n):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),n.makeArray(a,this))};A.prototype=n.fn,y=n(l);var B=/^(?:parents|prev(?:Until|All))/,C={children:!0,contents:!0,next:!0,prev:!0};n.extend({dir:function(a,b,c){var d=[],e=void 0!==c;while((a=a[b])&&9!==a.nodeType)if(1===a.nodeType){if(e&&n(a).is(c))break;d.push(a)}return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),n.fn.extend({has:function(a){var b=n(a,this),c=b.length;return this.filter(function(){for(var a=0;c>a;a++)if(n.contains(this,b[a]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=u.test(a)||"string"!=typeof a?n(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&n.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?n.unique(f):f)},index:function(a){return a?"string"==typeof a?g.call(n(a),this[0]):g.call(this,a.jquery?a[0]:a):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(n.unique(n.merge(this.get(),n(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function D(a,b){while((a=a[b])&&1!==a.nodeType);return a}n.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return n.dir(a,"parentNode")},parentsUntil:function(a,b,c){return n.dir(a,"parentNode",c)},next:function(a){return D(a,"nextSibling")},prev:function(a){return D(a,"previousSibling")},nextAll:function(a){return n.dir(a,"nextSibling")},prevAll:function(a){return n.dir(a,"previousSibling")},nextUntil:function(a,b,c){return n.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return n.dir(a,"previousSibling",c)},siblings:function(a){return n.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return n.sibling(a.firstChild)},contents:function(a){return a.contentDocument||n.merge([],a.childNodes)}},function(a,b){n.fn[a]=function(c,d){var e=n.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=n.filter(d,e)),this.length>1&&(C[a]||n.unique(e),B.test(a)&&e.reverse()),this.pushStack(e)}});var E=/\S+/g,F={};function G(a){var b=F[a]={};return n.each(a.match(E)||[],function(a,c){b[c]=!0}),b}n.Callbacks=function(a){a="string"==typeof a?F[a]||G(a):n.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(b=a.memory&&l,c=!0,g=e||0,e=0,f=h.length,d=!0;h&&f>g;g++)if(h[g].apply(l[0],l[1])===!1&&a.stopOnFalse){b=!1;break}d=!1,h&&(i?i.length&&j(i.shift()):b?h=[]:k.disable())},k={add:function(){if(h){var c=h.length;!function g(b){n.each(b,function(b,c){var d=n.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&g(c)})}(arguments),d?f=h.length:b&&(e=c,j(b))}return this},remove:function(){return h&&n.each(arguments,function(a,b){var c;while((c=n.inArray(b,h,c))>-1)h.splice(c,1),d&&(f>=c&&f--,g>=c&&g--)}),this},has:function(a){return a?n.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],f=0,this},disable:function(){return h=i=b=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,b||k.disable(),this},locked:function(){return!i},fireWith:function(a,b){return!h||c&&!i||(b=b||[],b=[a,b.slice?b.slice():b],d?i.push(b):j(b)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!c}};return k},n.extend({Deferred:function(a){var b=[["resolve","done",n.Callbacks("once memory"),"resolved"],["reject","fail",n.Callbacks("once memory"),"rejected"],["notify","progress",n.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return n.Deferred(function(c){n.each(b,function(b,f){var g=n.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&n.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?n.extend(a,d):d}},e={};return d.pipe=d.then,n.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&n.isFunction(a.promise)?e:0,g=1===f?a:n.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&n.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var H;n.fn.ready=function(a){return n.ready.promise().done(a),this},n.extend({isReady:!1,readyWait:1,holdReady:function(a){a?n.readyWait++:n.ready(!0)},ready:function(a){(a===!0?--n.readyWait:n.isReady)||(n.isReady=!0,a!==!0&&--n.readyWait>0||(H.resolveWith(l,[n]),n.fn.triggerHandler&&(n(l).triggerHandler("ready"),n(l).off("ready"))))}});function I(){l.removeEventListener("DOMContentLoaded",I,!1),a.removeEventListener("load",I,!1),n.ready()}n.ready.promise=function(b){return H||(H=n.Deferred(),"complete"===l.readyState?setTimeout(n.ready):(l.addEventListener("DOMContentLoaded",I,!1),a.addEventListener("load",I,!1))),H.promise(b)},n.ready.promise();var J=n.access=function(a,b,c,d,e,f,g){var h=0,i=a.length,j=null==c;if("object"===n.type(c)){e=!0;for(h in c)n.access(a,b,h,c[h],!0,f,g)}else if(void 0!==d&&(e=!0,n.isFunction(d)||(g=!0),j&&(g?(b.call(a,d),b=null):(j=b,b=function(a,b,c){return j.call(n(a),c)})),b))for(;i>h;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f};n.acceptData=function(a){return 1===a.nodeType||9===a.nodeType||!+a.nodeType};function K(){Object.defineProperty(this.cache={},0,{get:function(){return{}}}),this.expando=n.expando+K.uid++}K.uid=1,K.accepts=n.acceptData,K.prototype={key:function(a){if(!K.accepts(a))return 0;var b={},c=a[this.expando];if(!c){c=K.uid++;try{b[this.expando]={value:c},Object.defineProperties(a,b)}catch(d){b[this.expando]=c,n.extend(a,b)}}return this.cache[c]||(this.cache[c]={}),c},set:function(a,b,c){var d,e=this.key(a),f=this.cache[e];if("string"==typeof b)f[b]=c;else if(n.isEmptyObject(f))n.extend(this.cache[e],b);else for(d in b)f[d]=b[d];return f},get:function(a,b){var c=this.cache[this.key(a)];return void 0===b?c:c[b]},access:function(a,b,c){var d;return void 0===b||b&&"string"==typeof b&&void 0===c?(d=this.get(a,b),void 0!==d?d:this.get(a,n.camelCase(b))):(this.set(a,b,c),void 0!==c?c:b)},remove:function(a,b){var c,d,e,f=this.key(a),g=this.cache[f];if(void 0===b)this.cache[f]={};else{n.isArray(b)?d=b.concat(b.map(n.camelCase)):(e=n.camelCase(b),b in g?d=[b,e]:(d=e,d=d in g?[d]:d.match(E)||[])),c=d.length;while(c--)delete g[d[c]]}},hasData:function(a){return!n.isEmptyObject(this.cache[a[this.expando]]||{})},discard:function(a){a[this.expando]&&delete this.cache[a[this.expando]]}};var L=new K,M=new K,N=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,O=/([A-Z])/g;function P(a,b,c){var d;if(void 0===c&&1===a.nodeType)if(d="data-"+b.replace(O,"-$1").toLowerCase(),c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:N.test(c)?n.parseJSON(c):c}catch(e){}M.set(a,b,c)}else c=void 0;return c}n.extend({hasData:function(a){return M.hasData(a)||L.hasData(a)},data:function(a,b,c){return M.access(a,b,c) +},removeData:function(a,b){M.remove(a,b)},_data:function(a,b,c){return L.access(a,b,c)},_removeData:function(a,b){L.remove(a,b)}}),n.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=M.get(f),1===f.nodeType&&!L.get(f,"hasDataAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=n.camelCase(d.slice(5)),P(f,d,e[d])));L.set(f,"hasDataAttrs",!0)}return e}return"object"==typeof a?this.each(function(){M.set(this,a)}):J(this,function(b){var c,d=n.camelCase(a);if(f&&void 0===b){if(c=M.get(f,a),void 0!==c)return c;if(c=M.get(f,d),void 0!==c)return c;if(c=P(f,d,void 0),void 0!==c)return c}else this.each(function(){var c=M.get(this,d);M.set(this,d,b),-1!==a.indexOf("-")&&void 0!==c&&M.set(this,a,b)})},null,b,arguments.length>1,null,!0)},removeData:function(a){return this.each(function(){M.remove(this,a)})}}),n.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=L.get(a,b),c&&(!d||n.isArray(c)?d=L.access(a,b,n.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=n.queue(a,b),d=c.length,e=c.shift(),f=n._queueHooks(a,b),g=function(){n.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return L.get(a,c)||L.access(a,c,{empty:n.Callbacks("once memory").add(function(){L.remove(a,[b+"queue",c])})})}}),n.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthx",k.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue}();var U="undefined";k.focusinBubbles="onfocusin"in a;var V=/^key/,W=/^(?:mouse|pointer|contextmenu)|click/,X=/^(?:focusinfocus|focusoutblur)$/,Y=/^([^.]*)(?:\.(.+)|)$/;function Z(){return!0}function $(){return!1}function _(){try{return l.activeElement}catch(a){}}n.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=L.get(a);if(r){c.handler&&(f=c,c=f.handler,e=f.selector),c.guid||(c.guid=n.guid++),(i=r.events)||(i=r.events={}),(g=r.handle)||(g=r.handle=function(b){return typeof n!==U&&n.event.triggered!==b.type?n.event.dispatch.apply(a,arguments):void 0}),b=(b||"").match(E)||[""],j=b.length;while(j--)h=Y.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o&&(l=n.event.special[o]||{},o=(e?l.delegateType:l.bindType)||o,l=n.event.special[o]||{},k=n.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&n.expr.match.needsContext.test(e),namespace:p.join(".")},f),(m=i[o])||(m=i[o]=[],m.delegateCount=0,l.setup&&l.setup.call(a,d,p,g)!==!1||a.addEventListener&&a.addEventListener(o,g,!1)),l.add&&(l.add.call(a,k),k.handler.guid||(k.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,k):m.push(k),n.event.global[o]=!0)}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=L.hasData(a)&&L.get(a);if(r&&(i=r.events)){b=(b||"").match(E)||[""],j=b.length;while(j--)if(h=Y.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=n.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,m=i[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),g=f=m.length;while(f--)k=m[f],!e&&q!==k.origType||c&&c.guid!==k.guid||h&&!h.test(k.namespace)||d&&d!==k.selector&&("**"!==d||!k.selector)||(m.splice(f,1),k.selector&&m.delegateCount--,l.remove&&l.remove.call(a,k));g&&!m.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||n.removeEvent(a,o,r.handle),delete i[o])}else for(o in i)n.event.remove(a,o+b[j],c,d,!0);n.isEmptyObject(i)&&(delete r.handle,L.remove(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,m,o,p=[d||l],q=j.call(b,"type")?b.type:b,r=j.call(b,"namespace")?b.namespace.split("."):[];if(g=h=d=d||l,3!==d.nodeType&&8!==d.nodeType&&!X.test(q+n.event.triggered)&&(q.indexOf(".")>=0&&(r=q.split("."),q=r.shift(),r.sort()),k=q.indexOf(":")<0&&"on"+q,b=b[n.expando]?b:new n.Event(q,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=r.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+r.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:n.makeArray(c,[b]),o=n.event.special[q]||{},e||!o.trigger||o.trigger.apply(d,c)!==!1)){if(!e&&!o.noBubble&&!n.isWindow(d)){for(i=o.delegateType||q,X.test(i+q)||(g=g.parentNode);g;g=g.parentNode)p.push(g),h=g;h===(d.ownerDocument||l)&&p.push(h.defaultView||h.parentWindow||a)}f=0;while((g=p[f++])&&!b.isPropagationStopped())b.type=f>1?i:o.bindType||q,m=(L.get(g,"events")||{})[b.type]&&L.get(g,"handle"),m&&m.apply(g,c),m=k&&g[k],m&&m.apply&&n.acceptData(g)&&(b.result=m.apply(g,c),b.result===!1&&b.preventDefault());return b.type=q,e||b.isDefaultPrevented()||o._default&&o._default.apply(p.pop(),c)!==!1||!n.acceptData(d)||k&&n.isFunction(d[q])&&!n.isWindow(d)&&(h=d[k],h&&(d[k]=null),n.event.triggered=q,d[q](),n.event.triggered=void 0,h&&(d[k]=h)),b.result}},dispatch:function(a){a=n.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(L.get(this,"events")||{})[a.type]||[],k=n.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=n.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,c=0;while((g=f.handlers[c++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(g.namespace))&&(a.handleObj=g,a.data=g.data,e=((n.event.special[g.origType]||{}).handle||g.handler).apply(f.elem,i),void 0!==e&&(a.result=e)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!==this;i=i.parentNode||this)if(i.disabled!==!0||"click"!==a.type){for(d=[],c=0;h>c;c++)f=b[c],e=f.selector+" ",void 0===d[e]&&(d[e]=f.needsContext?n(e,this).index(i)>=0:n.find(e,this,null,[i]).length),d[e]&&d.push(f);d.length&&g.push({elem:i,handlers:d})}return h]*)\/>/gi,bb=/<([\w:]+)/,cb=/<|&#?\w+;/,db=/<(?:script|style|link)/i,eb=/checked\s*(?:[^=]|=\s*.checked.)/i,fb=/^$|\/(?:java|ecma)script/i,gb=/^true\/(.*)/,hb=/^\s*\s*$/g,ib={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};ib.optgroup=ib.option,ib.tbody=ib.tfoot=ib.colgroup=ib.caption=ib.thead,ib.th=ib.td;function jb(a,b){return n.nodeName(a,"table")&&n.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function kb(a){return a.type=(null!==a.getAttribute("type"))+"/"+a.type,a}function lb(a){var b=gb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function mb(a,b){for(var c=0,d=a.length;d>c;c++)L.set(a[c],"globalEval",!b||L.get(b[c],"globalEval"))}function nb(a,b){var c,d,e,f,g,h,i,j;if(1===b.nodeType){if(L.hasData(a)&&(f=L.access(a),g=L.set(b,f),j=f.events)){delete g.handle,g.events={};for(e in j)for(c=0,d=j[e].length;d>c;c++)n.event.add(b,e,j[e][c])}M.hasData(a)&&(h=M.access(a),i=n.extend({},h),M.set(b,i))}}function ob(a,b){var c=a.getElementsByTagName?a.getElementsByTagName(b||"*"):a.querySelectorAll?a.querySelectorAll(b||"*"):[];return void 0===b||b&&n.nodeName(a,b)?n.merge([a],c):c}function pb(a,b){var c=b.nodeName.toLowerCase();"input"===c&&T.test(a.type)?b.checked=a.checked:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}n.extend({clone:function(a,b,c){var d,e,f,g,h=a.cloneNode(!0),i=n.contains(a.ownerDocument,a);if(!(k.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||n.isXMLDoc(a)))for(g=ob(h),f=ob(a),d=0,e=f.length;e>d;d++)pb(f[d],g[d]);if(b)if(c)for(f=f||ob(a),g=g||ob(h),d=0,e=f.length;e>d;d++)nb(f[d],g[d]);else nb(a,h);return g=ob(h,"script"),g.length>0&&mb(g,!i&&ob(a,"script")),h},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,k=b.createDocumentFragment(),l=[],m=0,o=a.length;o>m;m++)if(e=a[m],e||0===e)if("object"===n.type(e))n.merge(l,e.nodeType?[e]:e);else if(cb.test(e)){f=f||k.appendChild(b.createElement("div")),g=(bb.exec(e)||["",""])[1].toLowerCase(),h=ib[g]||ib._default,f.innerHTML=h[1]+e.replace(ab,"<$1>")+h[2],j=h[0];while(j--)f=f.lastChild;n.merge(l,f.childNodes),f=k.firstChild,f.textContent=""}else l.push(b.createTextNode(e));k.textContent="",m=0;while(e=l[m++])if((!d||-1===n.inArray(e,d))&&(i=n.contains(e.ownerDocument,e),f=ob(k.appendChild(e),"script"),i&&mb(f),c)){j=0;while(e=f[j++])fb.test(e.type||"")&&c.push(e)}return k},cleanData:function(a){for(var b,c,d,e,f=n.event.special,g=0;void 0!==(c=a[g]);g++){if(n.acceptData(c)&&(e=c[L.expando],e&&(b=L.cache[e]))){if(b.events)for(d in b.events)f[d]?n.event.remove(c,d):n.removeEvent(c,d,b.handle);L.cache[e]&&delete L.cache[e]}delete M.cache[c[M.expando]]}}}),n.fn.extend({text:function(a){return J(this,function(a){return void 0===a?n.text(this):this.empty().each(function(){(1===this.nodeType||11===this.nodeType||9===this.nodeType)&&(this.textContent=a)})},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=jb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=jb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?n.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||n.cleanData(ob(c)),c.parentNode&&(b&&n.contains(c.ownerDocument,c)&&mb(ob(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++)1===a.nodeType&&(n.cleanData(ob(a,!1)),a.textContent="");return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return n.clone(this,a,b)})},html:function(a){return J(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a&&1===b.nodeType)return b.innerHTML;if("string"==typeof a&&!db.test(a)&&!ib[(bb.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(ab,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(n.cleanData(ob(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,n.cleanData(ob(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,l=this.length,m=this,o=l-1,p=a[0],q=n.isFunction(p);if(q||l>1&&"string"==typeof p&&!k.checkClone&&eb.test(p))return this.each(function(c){var d=m.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(l&&(c=n.buildFragment(a,this[0].ownerDocument,!1,this),d=c.firstChild,1===c.childNodes.length&&(c=d),d)){for(f=n.map(ob(c,"script"),kb),g=f.length;l>j;j++)h=c,j!==o&&(h=n.clone(h,!0,!0),g&&n.merge(f,ob(h,"script"))),b.call(this[j],h,j);if(g)for(i=f[f.length-1].ownerDocument,n.map(f,lb),j=0;g>j;j++)h=f[j],fb.test(h.type||"")&&!L.access(h,"globalEval")&&n.contains(i,h)&&(h.src?n._evalUrl&&n._evalUrl(h.src):n.globalEval(h.textContent.replace(hb,"")))}return this}}),n.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){n.fn[a]=function(a){for(var c,d=[],e=n(a),g=e.length-1,h=0;g>=h;h++)c=h===g?this:this.clone(!0),n(e[h])[b](c),f.apply(d,c.get());return this.pushStack(d)}});var qb,rb={};function sb(b,c){var d,e=n(c.createElement(b)).appendTo(c.body),f=a.getDefaultComputedStyle&&(d=a.getDefaultComputedStyle(e[0]))?d.display:n.css(e[0],"display");return e.detach(),f}function tb(a){var b=l,c=rb[a];return c||(c=sb(a,b),"none"!==c&&c||(qb=(qb||n("