From 07d765209dea12229e70a09784e647611acabcda Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 19 Apr 2016 14:24:36 +0100 Subject: [PATCH 001/134] First bits of emailpusher Mostly logic of when to send an email --- synapse/push/emailpusher.py | 214 ++++++++++++++++++ synapse/push/pusher.py | 4 +- synapse/storage/event_push_actions.py | 57 ++++- synapse/storage/events.py | 2 + synapse/storage/pusher.py | 27 +++ synapse/storage/schema/delta/31/events.sql | 16 ++ .../schema/delta/31/pusher_throttle.sql | 23 ++ 7 files changed, 335 insertions(+), 8 deletions(-) create mode 100644 synapse/push/emailpusher.py create mode 100644 synapse/storage/schema/delta/31/events.sql create mode 100644 synapse/storage/schema/delta/31/pusher_throttle.sql diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py new file mode 100644 index 000000000..f9954df39 --- /dev/null +++ b/synapse/push/emailpusher.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer, reactor + +import logging + +from synapse.util.metrics import Measure +from synapse.util.async import run_on_reactor + +logger = logging.getLogger(__name__) + +# The amount of time we always wait before ever emailing about a notification +# (to give the user a chance to respond to other push or notice the window) +DELAY_BEFORE_MAIL_MS = 2 * 60 * 1000 + +THROTTLE_START_MS = 2 * 60 * 1000 +THROTTLE_MAX_MS = (2 * 60 * 1000) * (2**11) # ~3 days + +# If no event triggers a notification for this long after the previous, +# the throttle is released. +THROTTLE_RESET_AFTER_MS = (2 * 60 * 1000) * (2**11) # ~3 days + + +class EmailPusher(object): + """ + A pusher that sends email notifications about events (approximately) + when they happen. + This shares quite a bit of code with httpusher: it would be good to + factor out the common parts + """ + def __init__(self, hs, pusherdict): + self.hs = hs + self.store = self.hs.get_datastore() + self.clock = self.hs.get_clock() + self.pusher_id = pusherdict['id'] + self.user_id = pusherdict['user_name'] + self.app_id = pusherdict['app_id'] + self.email = pusherdict['pushkey'] + self.last_stream_ordering = pusherdict['last_stream_ordering'] + self.timed_call = None + self.throttle_params = None + + # See httppusher + self.max_stream_ordering = None + + @defer.inlineCallbacks + def on_started(self): + self.throttle_params = yield self.store.get_throttle_params_by_room( + self.pusher_id + ) + yield self._process() + + @defer.inlineCallbacks + def on_new_notifications(self, min_stream_ordering, max_stream_ordering): + with Measure(self.clock, "push.on_new_notifications"): + self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering) + yield self._process() + + @defer.inlineCallbacks + def on_timer(self): + self.timed_call = None + with Measure(self.clock, "push.on_timer"): + yield self._process() + + @defer.inlineCallbacks + def _process(self): + last_notifs = yield self.store.get_time_of_latest_push_action_by_room_for_user( + self.user_id + ) + + unprocessed = yield self.store.get_unread_push_actions_for_user_in_range( + self.user_id, self.last_stream_ordering, self.max_stream_ordering + ) + + soonest_due_at = None + + for push_action in unprocessed: + received_at = push_action['received_ts'] + if received_at is None: + received_at = 0 + notif_ready_at = received_at + DELAY_BEFORE_MAIL_MS + + room_ready_at = self.room_ready_to_notify_at( + push_action['room_id'], self.get_room_last_notif_ts( + last_notifs, push_action['room_id'] + ) + ) + + should_notify_at = max(notif_ready_at, room_ready_at) + + if should_notify_at < self.clock.time_msec(): + # one of our notifications is ready for sending, so we send + # *one* email updating the user on their notifications, + # we then consider all previously outstanding notifications + # to be delivered. + yield self.send_notification(push_action) + + yield self.save_last_stream_ordering_and_success(max([ + ea['stream_ordering'] for ea in unprocessed + ])) + yield self.sent_notif_update_throttle( + push_action['room_id'], push_action + ) + else: + if soonest_due_at is None or should_notify_at < soonest_due_at: + soonest_due_at = should_notify_at + + if self.timed_call is not None: + self.timed_call.cancel() + self.timed_call = None + + if soonest_due_at is not None: + self.timed_call = reactor.callLater( + self.seconds_until(soonest_due_at), self.on_timer + ) + + @defer.inlineCallbacks + def save_last_stream_ordering_and_success(self, last_stream_ordering): + self.last_stream_ordering = last_stream_ordering + yield self.store.update_pusher_last_stream_ordering_and_success( + self.app_id, self.email, self.user_id, + last_stream_ordering, self.clock.time_msec() + ) + + def seconds_until(self, ts_msec): + return (ts_msec - self.clock.time_msec()) / 1000 + + def get_room_last_notif_ts(self, last_notif_by_room, room_id): + if room_id in last_notif_by_room: + return last_notif_by_room[room_id] + else: + return 0 + + def get_room_throttle_ms(self, room_id): + if room_id in self.throttle_params: + return self.throttle_params[room_id]["throttle_ms"] + else: + return 0 + + def get_room_last_sent_ts(self, room_id): + if room_id in self.throttle_params: + return self.throttle_params[room_id]["last_sent_ts"] + else: + return 0 + + def room_ready_to_notify_at(self, room_id, last_notif_time): + """ + Determines whether throttling should prevent us from sending an email + for the given room + Returns: True if we should send, False if we should not + """ + last_sent_ts = self.get_room_last_sent_ts(room_id) + throttle_ms = self.get_room_throttle_ms(room_id) + + may_send_at = last_sent_ts + throttle_ms + return may_send_at + + @defer.inlineCallbacks + def sent_notif_update_throttle(self, room_id, notified_push_action): + # We have sent a notification, so update the throttle accordingly. + # If the event that triggered the notif happened more than + # THROTTLE_RESET_AFTER_MS after the previous one that triggered a + # notif, we release the throttle. Otherwise, the throttle is increased. + time_of_previous_notifs = yield self.store.get_time_of_last_push_action_before( + notified_push_action['stream_ordering'] + ) + + time_of_this_notifs = notified_push_action['received_ts'] + + if time_of_previous_notifs is not None and time_of_this_notifs is not None: + gap = time_of_this_notifs - time_of_previous_notifs + else: + # if we don't know the arrival time of one of the notifs (it was not + # stored prior to email notification code) then assume a gap of + # zero which will just not reset the throttle + gap = 0 + + current_throttle_ms = self.get_room_throttle_ms(room_id) + + if gap > THROTTLE_RESET_AFTER_MS: + new_throttle_ms = THROTTLE_START_MS + else: + if current_throttle_ms == 0: + new_throttle_ms = THROTTLE_START_MS + else: + new_throttle_ms = min( + current_throttle_ms * 2, + THROTTLE_MAX_MS + ) + self.throttle_params[room_id] = { + "last_sent_ts": self.clock.time_msec(), + "throttle_ms": new_throttle_ms + } + yield self.store.set_throttle_params( + self.pusher_id, room_id, self.throttle_params[room_id] + ) + + @defer.inlineCallbacks + def send_notification(self, push_action): + yield run_on_reactor() + logger.error("sending notif email for user %r", self.user_id) \ No newline at end of file diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index 496083750..f7c3021fc 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -1,7 +1,9 @@ from httppusher import HttpPusher +from emailpusher import EmailPusher PUSHER_TYPES = { - 'http': HttpPusher + 'http': HttpPusher, + 'email': EmailPusher, } diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 86a98b6f1..ad512b2f0 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -118,15 +118,17 @@ class EventPushActionsStore(SQLBaseStore): max_stream_ordering=None): def get_after_receipt(txn): sql = ( - "SELECT ep.event_id, ep.stream_ordering, ep.actions " + "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, " + "e.received_ts " "FROM event_push_actions AS ep, (" - " SELECT room_id, user_id," - " max(topological_ordering) as topological_ordering," - " max(stream_ordering) as stream_ordering" + " SELECT room_id, user_id, " + " max(topological_ordering) as topological_ordering, " + " max(stream_ordering) as stream_ordering " " FROM events" " NATURAL JOIN receipts_linearized WHERE receipt_type = 'm.read'" " GROUP BY room_id, user_id" ") AS rl " + "NATURAL JOIN events e " "WHERE" " ep.room_id = rl.room_id" " AND (" @@ -153,8 +155,10 @@ class EventPushActionsStore(SQLBaseStore): def get_no_receipt(txn): sql = ( - "SELECT ep.event_id, ep.stream_ordering, ep.actions " + "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, " + "e.received_ts " "FROM event_push_actions AS ep " + "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id " "WHERE ep.room_id not in (" " SELECT room_id FROM events NATURAL JOIN receipts_linearized" " WHERE receipt_type = 'm.read' AND user_id = ? " @@ -175,11 +179,30 @@ class EventPushActionsStore(SQLBaseStore): defer.returnValue([ { "event_id": row[0], - "stream_ordering": row[1], - "actions": json.loads(row[2]), + "room_id": row[1], + "stream_ordering": row[2], + "actions": json.loads(row[3]), + "received_ts": row[4], } for row in after_read_receipt + no_read_receipt ]) + @defer.inlineCallbacks + def get_time_of_last_push_action_before(self, stream_ordering): + def f(txn): + sql = ( + "SELECT e.received_ts " + "FROM event_push_actions AS ep " + "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id " + "WHERE ep.stream_ordering > ? " + "ORDER BY ep.stream_ordering ASC " + "LIMIT 1" + ) + txn.execute(sql, (stream_ordering,)) + return txn.fetchone() + result = yield self.runInteraction("get_time_of_last_push_action_before", f) + defer.returnValue(result[0] if result is not None else None) + + @defer.inlineCallbacks def get_latest_push_action_stream_ordering(self): def f(txn): @@ -190,6 +213,26 @@ class EventPushActionsStore(SQLBaseStore): ) defer.returnValue(result[0] or 0) + @defer.inlineCallbacks + def get_time_of_latest_push_action_by_room_for_user(self, user_id): + """ + Returns only the received_ts of the last notification in each of the + user's rooms, in a dict by room_id + """ + def f(txn): + txn.execute( + "SELECT ep.room_id, MAX(e.received_ts) " + "FROM event_push_actions AS ep " + "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id " + "GROUP BY ep.room_id" + ) + return txn.fetchall() + result = yield self.runInteraction( + "get_time_of_latest_push_action_by_room_for_user", f + ) + + defer.returnValue({row[0]: row[1] for row in result}) + def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id): # Sad that we have to blow away the cache for the whole room here txn.call_after( diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 21487724e..dd58e001d 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -55,6 +55,7 @@ class EventsStore(SQLBaseStore): def __init__(self, hs): super(EventsStore, self).__init__(hs) + self._clock = hs.get_clock() self.register_background_update_handler( self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts ) @@ -427,6 +428,7 @@ class EventsStore(SQLBaseStore): "outlier": event.internal_metadata.is_outlier(), "content": encode_json(event.content).decode("UTF-8"), "origin_server_ts": int(event.origin_server_ts), + "received_ts": self._clock.time_msec(), } for event, _ in events_and_contexts ], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index e5755c0ae..caef9b59a 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -230,3 +230,30 @@ class PusherStore(SQLBaseStore): {'failing_since': failing_since}, desc="update_pusher_failing_since", ) + + @defer.inlineCallbacks + def get_throttle_params_by_room(self, pusher_id): + res = yield self._simple_select_list( + "pusher_throttle", + {"pusher": pusher_id}, + ["room_id", "last_sent_ts", "throttle_ms"], + desc="get_throttle_params_by_room" + ) + + params_by_room = {} + for row in res: + params_by_room[row["room_id"]] = { + "last_sent_ts": row["last_sent_ts"], + "throttle_ms": row["throttle_ms"] + } + + defer.returnValue(params_by_room) + + @defer.inlineCallbacks + def set_throttle_params(self, pusher_id, room_id, params): + yield self._simple_upsert( + "pusher_throttle", + {"pusher": pusher_id, "room_id": room_id}, + params, + desc="set_throttle_params" + ) \ No newline at end of file diff --git a/synapse/storage/schema/delta/31/events.sql b/synapse/storage/schema/delta/31/events.sql new file mode 100644 index 000000000..1dd0f9e17 --- /dev/null +++ b/synapse/storage/schema/delta/31/events.sql @@ -0,0 +1,16 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE events ADD COLUMN received_ts BIGINT; diff --git a/synapse/storage/schema/delta/31/pusher_throttle.sql b/synapse/storage/schema/delta/31/pusher_throttle.sql new file mode 100644 index 000000000..d86d30c13 --- /dev/null +++ b/synapse/storage/schema/delta/31/pusher_throttle.sql @@ -0,0 +1,23 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE TABLE pusher_throttle( + pusher BIGINT NOT NULL, + room_id TEXT NOT NULL, + last_sent_ts BIGINT, + throttle_ms BIGINT, + PRIMARY KEY (pusher, room_id) +); From e2a01455af8dbab26b4a005d847f468a51fea6c3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 19 Apr 2016 14:52:58 +0100 Subject: [PATCH 002/134] Add single instance & logging stuff Copy the stuff over from http pusher that prevents multiple instances of process running at once and sets up logging and measure blocks. --- synapse/push/emailpusher.py | 47 ++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index f9954df39..74e3a7056 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -19,6 +19,7 @@ import logging from synapse.util.metrics import Measure from synapse.util.async import run_on_reactor +from synapse.util.logcontext import LoggingContext logger = logging.getLogger(__name__) @@ -56,6 +57,8 @@ class EmailPusher(object): # See httppusher self.max_stream_ordering = None + self.processing = False + @defer.inlineCallbacks def on_started(self): self.throttle_params = yield self.store.get_throttle_params_by_room( @@ -63,20 +66,48 @@ class EmailPusher(object): ) yield self._process() + def on_stop(self): + if self.timed_call: + self.timed_call.cancel() + @defer.inlineCallbacks def on_new_notifications(self, min_stream_ordering, max_stream_ordering): - with Measure(self.clock, "push.on_new_notifications"): - self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering) - yield self._process() + self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering) + yield self._process() @defer.inlineCallbacks def on_timer(self): self.timed_call = None - with Measure(self.clock, "push.on_timer"): - yield self._process() + yield self._process() @defer.inlineCallbacks def _process(self): + if self.processing: + return + + with LoggingContext("emailpush._process"): + with Measure(self.clock, "emailpush._process"): + try: + self.processing = True + # if the max ordering changes while we're running _unsafe_process, + # call it again, and so on until we've caught up. + while True: + starting_max_ordering = self.max_stream_ordering + try: + yield self._unsafe_process() + except: + logger.exception("Exception processing notifs") + if self.max_stream_ordering == starting_max_ordering: + break + finally: + self.processing = False + + def _unsafe_process(self): + """ + Main logic of the push loop without the wrapper function that sets + up logging, measures and guards against multiple instances of it + being run. + """ last_notifs = yield self.store.get_time_of_latest_push_action_by_room_for_user( self.user_id ) @@ -118,9 +149,9 @@ class EmailPusher(object): if soonest_due_at is None or should_notify_at < soonest_due_at: soonest_due_at = should_notify_at - if self.timed_call is not None: - self.timed_call.cancel() - self.timed_call = None + if self.timed_call is not None: + self.timed_call.cancel() + self.timed_call = None if soonest_due_at is not None: self.timed_call = reactor.callLater( From f63bd4ff4704c9f7b6e23c76720dbd955a60c058 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Apr 2016 13:02:01 +0100 Subject: [PATCH 003/134] Send a rather basic email notif Also pep8 fixes --- synapse/config/emailconfig.py | 62 +++++++++++++++++++++++++++ synapse/config/homeserver.py | 3 +- synapse/push/emailpusher.py | 32 ++++++++++---- synapse/push/mailer.py | 48 +++++++++++++++++++++ synapse/storage/event_push_actions.py | 1 - synapse/storage/pusher.py | 2 +- 6 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 synapse/config/emailconfig.py create mode 100644 synapse/push/mailer.py diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py new file mode 100644 index 000000000..978826627 --- /dev/null +++ b/synapse/config/emailconfig.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright 2015, 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file can't be called email.py because if it is, we cannot: +import email.utils + +from ._base import Config + + +class EmailConfig(Config): + """ + Email Configuration + """ + + def read_config(self, config): + email_config = config.get("email", None) + if email_config: + self.email_enable_notifs = email_config.get("enable_notifs", True) + if ( + "smtp_host" not in email_config or + "smtp_port" not in email_config or + "notif_from" not in email_config + ): + raise RuntimeError( + "You must set smtp_host, smtp_port and notif_from " + "to send email notifications" + ) + + self.email_smtp_host = email_config["smtp_host"] + self.email_smtp_port = email_config["smtp_port"] + self.email_notif_from = email_config["notif_from"] + + # make sure it's valid + parsed = email.utils.parseaddr(self.email_notif_from) + if parsed[1] == '': + raise RuntimeError("Invalid notif_from address") + else: + self.email_enable_notifs = False + self.email_smtp_host = None + self.email_smtp_port = None + self.email_notif_from = None + + def default_config(self, config_dir_path, server_name, **kwargs): + return """ + # Enable sending emails for notification events + #email_config: + # enable_notifs: false + # smtp_host: "localhost" + # smtp_port: 25 + """ diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 9a80ac39e..fc2445484 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -31,13 +31,14 @@ from .cas import CasConfig from .password import PasswordConfig from .jwt import JWTConfig from .ldap import LDAPConfig +from .emailconfig import EmailConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig, AppServiceConfig, KeyConfig, SAML2Config, CasConfig, - JWTConfig, LDAPConfig, PasswordConfig,): + JWTConfig, LDAPConfig, PasswordConfig, EmailConfig,): pass diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 74e3a7056..820c8f846 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -18,9 +18,10 @@ from twisted.internet import defer, reactor import logging from synapse.util.metrics import Measure -from synapse.util.async import run_on_reactor from synapse.util.logcontext import LoggingContext +from mailer import Mailer + logger = logging.getLogger(__name__) # The amount of time we always wait before ever emailing about a notification @@ -28,11 +29,11 @@ logger = logging.getLogger(__name__) DELAY_BEFORE_MAIL_MS = 2 * 60 * 1000 THROTTLE_START_MS = 2 * 60 * 1000 -THROTTLE_MAX_MS = (2 * 60 * 1000) * (2**11) # ~3 days +THROTTLE_MAX_MS = (2 * 60 * 1000) * (2 ** 11) # ~3 days # If no event triggers a notification for this long after the previous, # the throttle is released. -THROTTLE_RESET_AFTER_MS = (2 * 60 * 1000) * (2**11) # ~3 days +THROTTLE_RESET_AFTER_MS = (2 * 60 * 1000) * (2 ** 11) # ~3 days class EmailPusher(object): @@ -59,12 +60,22 @@ class EmailPusher(object): self.processing = False + if self.hs.config.email_enable_notifs: + self.mailer = Mailer( + self.store, + self.hs.config.email_smtp_host, self.hs.config.email_smtp_port, + self.hs.config.email_notif_from, + ) + else: + self.mailer = None + @defer.inlineCallbacks def on_started(self): - self.throttle_params = yield self.store.get_throttle_params_by_room( - self.pusher_id - ) - yield self._process() + if self.mailer is not None: + self.throttle_params = yield self.store.get_throttle_params_by_room( + self.pusher_id + ) + yield self._process() def on_stop(self): if self.timed_call: @@ -102,6 +113,7 @@ class EmailPusher(object): finally: self.processing = False + @defer.inlineCallbacks def _unsafe_process(self): """ Main logic of the push loop without the wrapper function that sets @@ -241,5 +253,7 @@ class EmailPusher(object): @defer.inlineCallbacks def send_notification(self, push_action): - yield run_on_reactor() - logger.error("sending notif email for user %r", self.user_id) \ No newline at end of file + logger.info("Sending notif email for user %r", self.user_id) + yield self.mailer.send_notification_mail( + self.user_id, self.email, push_action + ) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py new file mode 100644 index 000000000..93d3866ec --- /dev/null +++ b/synapse/push/mailer.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +import smtplib +import email.utils +import email.mime.multipart +from email.mime.text import MIMEText + + +class Mailer(object): + def __init__(self, store, smtp_host, smtp_port, notif_from): + self.store = store + self.smtp_host = smtp_host + self.smtp_port = smtp_port + self.notif_from = notif_from + + @defer.inlineCallbacks + def send_notification_mail(self, user_id, email_address, push_action): + raw_from = email.utils.parseaddr(self.notif_from)[1] + raw_to = email.utils.parseaddr(email_address)[1] + + if raw_to == '': + raise RuntimeError("Invalid 'to' address") + + plainText = "yo dawg, you got notifications!" + + text_part = MIMEText(plainText, "plain") + text_part['Subject'] = "New Matrix Notifications" + text_part['From'] = self.notif_from + text_part['To'] = email_address + + smtp = smtplib.SMTP(self.smtp_host, self.smtp_port) + smtp.sendmail(raw_from, raw_to, text_part.as_string()) + smtp.quit() \ No newline at end of file diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index ad512b2f0..f2af8bdb3 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -202,7 +202,6 @@ class EventPushActionsStore(SQLBaseStore): result = yield self.runInteraction("get_time_of_last_push_action_before", f) defer.returnValue(result[0] if result is not None else None) - @defer.inlineCallbacks def get_latest_push_action_stream_ordering(self): def f(txn): diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index caef9b59a..5fb47d418 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -256,4 +256,4 @@ class PusherStore(SQLBaseStore): {"pusher": pusher_id, "room_id": room_id}, params, desc="set_throttle_params" - ) \ No newline at end of file + ) From 05adc6c2de7def8058d97e9644dddca639886322 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Apr 2016 13:02:45 +0100 Subject: [PATCH 004/134] more pep8 --- synapse/push/mailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 93d3866ec..97cba2ec2 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -45,4 +45,4 @@ class Mailer(object): smtp = smtplib.SMTP(self.smtp_host, self.smtp_port) smtp.sendmail(raw_from, raw_to, text_part.as_string()) - smtp.quit() \ No newline at end of file + smtp.quit() From 2ed0adb075b745e6586ca88ce7cf6b169460a7d7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Apr 2016 18:35:29 +0100 Subject: [PATCH 005/134] Generate mails from a template --- synapse/config/emailconfig.py | 35 ++++++++++++++++++++++++---------- synapse/push/emailpusher.py | 12 ++++-------- synapse/push/mailer.py | 30 ++++++++++++++++------------- synapse/python_dependencies.py | 3 +++ 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 978826627..68fb4d806 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -28,19 +28,32 @@ class EmailConfig(Config): email_config = config.get("email", None) if email_config: self.email_enable_notifs = email_config.get("enable_notifs", True) - if ( - "smtp_host" not in email_config or - "smtp_port" not in email_config or - "notif_from" not in email_config - ): + + required = [ + "smtp_host", + "smtp_port", + "notif_from", + "template_dir", + "notif_template_html", + + ] + + missing = [] + for k in required: + if k not in email_config: + missing.append(k) + + if (len(missing) > 0): raise RuntimeError( - "You must set smtp_host, smtp_port and notif_from " - "to send email notifications" + "email.enable_notifs is True but required keys are missing: %s" % + (", ".join(["email."+k for k in missing]),) ) self.email_smtp_host = email_config["smtp_host"] self.email_smtp_port = email_config["smtp_port"] self.email_notif_from = email_config["notif_from"] + self.email_template_dir = email_config["template_dir"] + self.email_notif_template_html = email_config["notif_template_html"] # make sure it's valid parsed = email.utils.parseaddr(self.email_notif_from) @@ -48,9 +61,8 @@ class EmailConfig(Config): raise RuntimeError("Invalid notif_from address") else: self.email_enable_notifs = False - self.email_smtp_host = None - self.email_smtp_port = None - self.email_notif_from = None + # Not much point setting defaults for the rest: it would be an + # error for them to be used. def default_config(self, config_dir_path, server_name, **kwargs): return """ @@ -59,4 +71,7 @@ class EmailConfig(Config): # enable_notifs: false # smtp_host: "localhost" # smtp_port: 25 + # notif_from: Your Friendly Matrix Home Server + # template_dir: res/templates + # notif_template_html: notif.html """ diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 820c8f846..4e21221fb 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -61,11 +61,7 @@ class EmailPusher(object): self.processing = False if self.hs.config.email_enable_notifs: - self.mailer = Mailer( - self.store, - self.hs.config.email_smtp_host, self.hs.config.email_smtp_port, - self.hs.config.email_notif_from, - ) + self.mailer = Mailer(self.hs) else: self.mailer = None @@ -149,7 +145,7 @@ class EmailPusher(object): # *one* email updating the user on their notifications, # we then consider all previously outstanding notifications # to be delivered. - yield self.send_notification(push_action) + yield self.send_notification(unprocessed) yield self.save_last_stream_ordering_and_success(max([ ea['stream_ordering'] for ea in unprocessed @@ -252,8 +248,8 @@ class EmailPusher(object): ) @defer.inlineCallbacks - def send_notification(self, push_action): + def send_notification(self, push_actions): logger.info("Sending notif email for user %r", self.user_id) yield self.mailer.send_notification_mail( - self.user_id, self.email, push_action + self.user_id, self.email, push_actions ) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 97cba2ec2..0f20d43f7 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -15,34 +15,38 @@ from twisted.internet import defer -import smtplib +from twisted.mail.smtp import sendmail import email.utils import email.mime.multipart from email.mime.text import MIMEText +import jinja2 + class Mailer(object): - def __init__(self, store, smtp_host, smtp_port, notif_from): - self.store = store - self.smtp_host = smtp_host - self.smtp_port = smtp_port - self.notif_from = notif_from + def __init__(self, hs): + self.hs = hs + loader = jinja2.FileSystemLoader(self.hs.config.email_template_dir) + env = jinja2.Environment(loader=loader) + self.notif_template = env.get_template(self.hs.config.email_notif_template_html) @defer.inlineCallbacks - def send_notification_mail(self, user_id, email_address, push_action): - raw_from = email.utils.parseaddr(self.notif_from)[1] + def send_notification_mail(self, user_id, email_address, push_actions): + raw_from = email.utils.parseaddr(self.hs.config.email_notif_from)[1] raw_to = email.utils.parseaddr(email_address)[1] if raw_to == '': raise RuntimeError("Invalid 'to' address") - plainText = "yo dawg, you got notifications!" + plainText = self.notif_template.render() text_part = MIMEText(plainText, "plain") text_part['Subject'] = "New Matrix Notifications" - text_part['From'] = self.notif_from + text_part['From'] = self.hs.config.email_notif_from text_part['To'] = email_address - smtp = smtplib.SMTP(self.smtp_host, self.smtp_port) - smtp.sendmail(raw_from, raw_to, text_part.as_string()) - smtp.quit() + yield sendmail( + self.hs.config.email_smtp_host, + raw_from, raw_to, text_part.as_string(), + port=self.hs.config.email_smtp_port + ) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index b25b73649..a065c78b4 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -45,6 +45,9 @@ CONDITIONAL_REQUIREMENTS = { "preview_url": { "netaddr>=0.7.18": ["netaddr"], }, + "email.enable_notifs": { + "Jinja2": ["Jinja2"], + }, } From c10ed26c303741fe0e43f11e2fbeeb148f466b17 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 21 Apr 2016 19:19:07 +0100 Subject: [PATCH 006/134] Flesh out email templating Mostly WIP porting the room name calculation logic from the web client so our room names in the email mirror the clients. --- synapse/push/emailpusher.py | 7 ++ synapse/push/mailer.py | 61 +++++++++++++- synapse/python_dependencies.py | 2 +- synapse/util/room_name.py | 142 +++++++++++++++++++++++++++++++++ 4 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 synapse/util/room_name.py diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 4e21221fb..7c810029f 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -82,6 +82,13 @@ class EmailPusher(object): self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering) yield self._process() + @defer.inlineCallbacks + def on_new_receipts(self, min_stream_id, max_stream_id): + # We could wake up and cancel the timer but there tend to be quite a + # lot of read receipts so it's probably less work to just let the + # timer fire + return defer.succeed(None) + @defer.inlineCallbacks def on_timer(self): self.timed_call = None diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 0f20d43f7..e68d701ff 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -14,18 +14,23 @@ # limitations under the License. from twisted.internet import defer - from twisted.mail.smtp import sendmail + import email.utils import email.mime.multipart from email.mime.text import MIMEText +from synapse.util.async import concurrently_execute +from synapse.util.room_name import calculate_room_name + import jinja2 class Mailer(object): def __init__(self, hs): self.hs = hs + self.store = self.hs.get_datastore() + self.state_handler = self.hs.get_state_handler() loader = jinja2.FileSystemLoader(self.hs.config.email_template_dir) env = jinja2.Environment(loader=loader) self.notif_template = env.get_template(self.hs.config.email_notif_template_html) @@ -38,9 +43,41 @@ class Mailer(object): if raw_to == '': raise RuntimeError("Invalid 'to' address") - plainText = self.notif_template.render() + rooms_in_order = deduped_ordered_list( + [pa['room_id'] for pa in push_actions] + ) - text_part = MIMEText(plainText, "plain") + notifs_by_room = {} + for pa in push_actions: + notifs_by_room.setdefault(pa["room_id"], []).append(pa) + + # collect the current state for all the rooms in which we have + # notifications + state_by_room = {} + + @defer.inlineCallbacks + def _fetch_room_state(room_id): + room_state = yield self.state_handler.get_current_state(room_id) + state_by_room[room_id] = room_state + + # Run at most 3 of these at once: sync does 10 at a time but email + # notifs are much realtime than sync so we can afford to wait a bit. + yield concurrently_execute(_fetch_room_state, rooms_in_order, 3) + + rooms = [ + self.get_room_vars( + r, user_id, notifs_by_room[r], state_by_room[r] + ) for r in rooms_in_order + ] + + template_vars = { + "unsubscribe_link": self.make_unsubscribe_link(), + "rooms": rooms, + } + + plainText = self.notif_template.render(**template_vars) + + text_part = MIMEText(plainText, "html") text_part['Subject'] = "New Matrix Notifications" text_part['From'] = self.hs.config.email_notif_from text_part['To'] = email_address @@ -50,3 +87,21 @@ class Mailer(object): raw_from, raw_to, text_part.as_string(), port=self.hs.config.email_smtp_port ) + + def get_room_vars(self, room_id, user_id, notifs, room_state): + room_vars = {} + room_vars['title'] = calculate_room_name(room_state, user_id) + return room_vars + + def make_unsubscribe_link(self): + return "https://vector.im/#/settings" # XXX: matrix.to + + +def deduped_ordered_list(l): + seen = set() + ret = [] + for item in l: + if item not in seen: + seen.add(item) + ret.append(item) + return ret \ No newline at end of file diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a065c78b4..16524dbdc 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -46,7 +46,7 @@ CONDITIONAL_REQUIREMENTS = { "netaddr>=0.7.18": ["netaddr"], }, "email.enable_notifs": { - "Jinja2": ["Jinja2"], + "Jinja2>=2.8": ["Jinja2>=2.8"], }, } diff --git a/synapse/util/room_name.py b/synapse/util/room_name.py new file mode 100644 index 000000000..7e49b92bb --- /dev/null +++ b/synapse/util/room_name.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +# intentionally looser than what aliases we allow to be registered since +# other HSes may allow aliases that we would not +ALIAS_RE = re.compile(r"^#.*:.+$") + +ALL_ALONE = "Empty Room" + + +def calculate_room_name(room_state, user_id): + # does it have a name? + if ("m.room.name", "") in room_state: + m_room_name = room_state[("m.room.name", "")] + if m_room_name.content and m_room_name.content["name"]: + return m_room_name.content["name"] + + # does it have a caononical alias? + if ("m.room.canonical_alias", "") in room_state: + canon_alias = room_state[("m.room.canonical_alias", "")] + if ( + canon_alias.content and canon_alias.content["alias"] and + looks_like_an_alias(canon_alias.content["alias"]) + ): + return canon_alias.content["alias"] + + # at this point we're going to need to search the state by all state keys + # for an event type, so rearrange the data structure + room_state_bytype = state_as_two_level_dict(room_state) + + # right then, any aliases at all? + if "m.room.aliases" in room_state_bytype: + m_room_aliases = room_state_bytype["m.room.aliases"] + if len(m_room_aliases.values()) > 0: + first_alias_event = m_room_aliases.values()[0] + if first_alias_event.content and first_alias_event.content["aliases"]: + the_aliases = first_alias_event.content["aliases"] + if len(the_aliases) > 0 and looks_like_an_alias(the_aliases[0]): + return the_aliases[0] + + my_member_event = None + if ("m.room.member", user_id) in room_state: + my_member_event = room_state[("m.room.member", user_id)] + + if ( + my_member_event is not None and + my_member_event.content['membership'] == "invite" + ): + if ("m.room.member", my_member_event.sender) in room_state: + inviter_member_event = room_state[("m.room.member", my_member_event.sender)] + return "Invite from %s" % (name_from_member_event(inviter_member_event),) + else: + return "Room Invite" + + # we're going to have to generate a name based on who's in the room, + # so find out who is in the room that isn't the user. + if "m.room.member" in room_state_bytype: + all_members = [ + ev for ev in room_state_bytype["m.room.member"].values() + if ev.membership == "join" or ev.membership == "invite" + ] + other_members = [m for m in all_members if m.sender != user_id] + else: + other_members = [] + all_members = [] + + if len(other_members) == 0: + if len(all_members) == 1: + # self-chat, peeked room with 1 participant, + # or inbound invite, or outbound 3PID invite. + if all_members[0].sender == user_id: + if "m.room.third_party_invite" in room_state_bytype: + third_party_invites = room_state_bytype["m.room.third_party_invite"] + if len(third_party_invites) > 0: + # technically third party invite events are not member + # events, but they are close enough + return "Inviting %s" ( + descriptor_from_member_events(third_party_invites) + ) + else: + return ALL_ALONE + else: + return name_from_member_event(all_members[0]) + else: + return ALL_ALONE + else: + return descriptor_from_member_events(other_members) + + +def state_as_two_level_dict(state): + ret = {} + for k, v in state.items(): + ret.setdefault(k[0], {})[k[1]] = v + return ret + + +def looks_like_an_alias(string): + return ALIAS_RE.match(string) is not None + + +def descriptor_from_member_events(member_events): + # else if (otherMembers.length === 1) { + # return otherMembers[0].name; + # } + # else if (otherMembers.length === 2) { + # return ( + # otherMembers[0].name + " and " + otherMembers[1].name + # ); + # } + # else { + # return ( + # otherMembers[0].name + " and " + (otherMembers.length - 1) + " others" + # ); + # } + if len(member_events) == 0: + return "nobody" + elif len(member_events) == 1: + return name_from_member_event(member_events[0]) + return "all the people, so many people. They all go hand in hand, hand in hand in their park life." + + +def name_from_member_event(member_event): + if ( + member_event.content and "displayname" in member_event.content and + member_event.content["displayname"] + ): + return member_event.content["displayname"] + return member_event.sender \ No newline at end of file From 19d6b6cd7aa32f844362f77ca7f0403af37fc396 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 21 Apr 2016 19:22:53 +0100 Subject: [PATCH 007/134] Add WIP email template files --- res/templates/notif.html | 14 ++++++++++++++ res/templates/room.html | 6 ++++++ 2 files changed, 20 insertions(+) create mode 100644 res/templates/notif.html create mode 100644 res/templates/room.html diff --git a/res/templates/notif.html b/res/templates/notif.html new file mode 100644 index 000000000..648ff034b --- /dev/null +++ b/res/templates/notif.html @@ -0,0 +1,14 @@ + + + +

{{ summaryText }}

+
+ {% for room in rooms %} + {% include 'room.html' with context %} + {% endfor %} +
+
+ Unsubscribe +
+ + diff --git a/res/templates/room.html b/res/templates/room.html new file mode 100644 index 000000000..ef36b4ee5 --- /dev/null +++ b/res/templates/room.html @@ -0,0 +1,6 @@ +
+

{{ room.title }}

+
+ Things have happened in this room +
+
From c553797c4f0b772cfa9c9370d0789bf32d82e6c5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Apr 2016 17:27:54 +0100 Subject: [PATCH 008/134] No inlineCallbacks necessary on this --- synapse/push/emailpusher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 7c810029f..dcbee4c3f 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -82,7 +82,6 @@ class EmailPusher(object): self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering) yield self._process() - @defer.inlineCallbacks def on_new_receipts(self, min_stream_id, max_stream_id): # We could wake up and cancel the timer but there tend to be quite a # lot of read receipts so it's probably less work to just let the From e8701e64b9ce52a377dba7091017e5d2e116ecdf Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Apr 2016 17:28:42 +0100 Subject: [PATCH 009/134] Implement group-of-people names --- synapse/util/room_name.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/synapse/util/room_name.py b/synapse/util/room_name.py index 7e49b92bb..30b729136 100644 --- a/synapse/util/room_name.py +++ b/synapse/util/room_name.py @@ -71,9 +71,9 @@ def calculate_room_name(room_state, user_id): if "m.room.member" in room_state_bytype: all_members = [ ev for ev in room_state_bytype["m.room.member"].values() - if ev.membership == "join" or ev.membership == "invite" + if ev.content['membership'] == "join" or ev.content['membership'] == "invite" ] - other_members = [m for m in all_members if m.sender != user_id] + other_members = [m for m in all_members if m.state_key != user_id] else: other_members = [] all_members = [] @@ -113,30 +113,27 @@ def looks_like_an_alias(string): def descriptor_from_member_events(member_events): - # else if (otherMembers.length === 1) { - # return otherMembers[0].name; - # } - # else if (otherMembers.length === 2) { - # return ( - # otherMembers[0].name + " and " + otherMembers[1].name - # ); - # } - # else { - # return ( - # otherMembers[0].name + " and " + (otherMembers.length - 1) + " others" - # ); - # } if len(member_events) == 0: return "nobody" elif len(member_events) == 1: return name_from_member_event(member_events[0]) - return "all the people, so many people. They all go hand in hand, hand in hand in their park life." + elif len(member_events) == 2: + return "%s and %s" % ( + name_from_member_event(member_events[0]), + name_from_member_event(member_events[1]), + ) + else: + return "%s and %d others" % ( + name_from_member_event(member_events[0]), + len(member_events) - 1, + ) def name_from_member_event(member_event): + # XXX: Need to look in invite state for invite display names. if ( member_event.content and "displayname" in member_event.content and member_event.content["displayname"] ): return member_event.content["displayname"] - return member_event.sender \ No newline at end of file + return member_event.state_key \ No newline at end of file From 83bf65297a624a205ce13e27f0f7c887b7db48af Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Apr 2016 18:31:47 +0100 Subject: [PATCH 010/134] Mime part is binary so encode it first. Doesn't get character enocind right yet but makes it not error. --- synapse/push/mailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index e68d701ff..f679718b0 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -77,7 +77,7 @@ class Mailer(object): plainText = self.notif_template.render(**template_vars) - text_part = MIMEText(plainText, "html") + text_part = MIMEText(plainText.encode('utf8'), "html") text_part['Subject'] = "New Matrix Notifications" text_part['From'] = self.hs.config.email_notif_from text_part['To'] = email_address From c5b3c6e1010ce55eda27b35f008819867c21bc51 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Apr 2016 18:33:36 +0100 Subject: [PATCH 011/134] Sort member events So names of people in a room are given in order --- synapse/util/room_name.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/util/room_name.py b/synapse/util/room_name.py index 30b729136..30ef77b9f 100644 --- a/synapse/util/room_name.py +++ b/synapse/util/room_name.py @@ -73,6 +73,10 @@ def calculate_room_name(room_state, user_id): ev for ev in room_state_bytype["m.room.member"].values() if ev.content['membership'] == "join" or ev.content['membership'] == "invite" ] + # Sort the member events oldest-first so the we name people in the + # order the joined (it should at least be deterministic rather than + # dictionary iteration order) + all_members.sort(key=lambda e: e.origin_server_ts) other_members = [m for m in all_members if m.state_key != user_id] else: other_members = [] From bd0f9c2065a0f766786493649893489b8e0e3d38 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Apr 2016 18:42:00 +0100 Subject: [PATCH 012/134] Actually do UTF8 correctly --- synapse/push/mailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index f679718b0..9212d36b8 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -77,7 +77,7 @@ class Mailer(object): plainText = self.notif_template.render(**template_vars) - text_part = MIMEText(plainText.encode('utf8'), "html") + text_part = MIMEText(plainText, "html", "utf8") text_part['Subject'] = "New Matrix Notifications" text_part['From'] = self.hs.config.email_notif_from text_part['To'] = email_address From 05e49ffbdf83ec3f910e1f8dbbe23aa4da297986 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Apr 2016 18:44:17 +0100 Subject: [PATCH 013/134] No we don't: it's just the display name --- synapse/util/room_name.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/util/room_name.py b/synapse/util/room_name.py index 30ef77b9f..d85ccaea5 100644 --- a/synapse/util/room_name.py +++ b/synapse/util/room_name.py @@ -134,7 +134,6 @@ def descriptor_from_member_events(member_events): def name_from_member_event(member_event): - # XXX: Need to look in invite state for invite display names. if ( member_event.content and "displayname" in member_event.content and member_event.content["displayname"] From 290f125a13c3c9f5cb772909248d4e482dcfb871 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 25 Apr 2016 14:42:59 +0100 Subject: [PATCH 014/134] Typo --- synapse/util/room_name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/room_name.py b/synapse/util/room_name.py index d85ccaea5..f55ef293b 100644 --- a/synapse/util/room_name.py +++ b/synapse/util/room_name.py @@ -29,7 +29,7 @@ def calculate_room_name(room_state, user_id): if m_room_name.content and m_room_name.content["name"]: return m_room_name.content["name"] - # does it have a caononical alias? + # does it have a canonical alias? if ("m.room.canonical_alias", "") in room_state: canon_alias = room_state[("m.room.canonical_alias", "")] if ( From 7b4715bad704231b51c6d0462cfd19ed32df5e0b Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 25 Apr 2016 18:27:04 +0100 Subject: [PATCH 015/134] More variable calculation for email notifs Include name of the person we're sending to and add summary text at the top giving an overview of what's happened. --- res/templates/notif.html | 3 +- synapse/push/mailer.py | 57 ++++++++++++++++++- .../{room_name.py => presentable_names.py} | 35 ++++++------ 3 files changed, 76 insertions(+), 19 deletions(-) rename synapse/util/{room_name.py => presentable_names.py} (91%) diff --git a/res/templates/notif.html b/res/templates/notif.html index 648ff034b..aee52ec8c 100644 --- a/res/templates/notif.html +++ b/res/templates/notif.html @@ -1,7 +1,8 @@ -

{{ summaryText }}

+
Hi {{ user_display_name }},
+
{{ summary_text }}
{% for room in rooms %} {% include 'room.html' with context %} diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 9212d36b8..9e2297a03 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -21,11 +21,19 @@ import email.mime.multipart from email.mime.text import MIMEText from synapse.util.async import concurrently_execute -from synapse.util.room_name import calculate_room_name +from synapse.util.presentable_names import calculate_room_name, name_from_member_event +from synapse.types import UserID +from synapse.api.errors import StoreError import jinja2 +MESSAGE_FROM_PERSON_IN_ROOM = "You have a message from %s in the %s room" +MESSAGE_FROM_PERSON = "You have a message from %s" +MESSAGES_IN_ROOM = "There are some messages for you in the %s room" +MESSAGES_IN_ROOMS = "Here are some messages you may have missed" + + class Mailer(object): def __init__(self, hs): self.hs = hs @@ -55,6 +63,13 @@ class Mailer(object): # notifications state_by_room = {} + try: + user_display_name = yield self.store.get_profile_displayname( + UserID.from_string(user_id).localpart + ) + except StoreError: + user_display_name = user_id + @defer.inlineCallbacks def _fetch_room_state(room_id): room_state = yield self.state_handler.get_current_state(room_id) @@ -70,8 +85,14 @@ class Mailer(object): ) for r in rooms_in_order ] + summary_text = yield self.make_summary_text( + notifs_by_room, state_by_room, user_id + ) + template_vars = { + "user_display_name": user_display_name, "unsubscribe_link": self.make_unsubscribe_link(), + "summary_text": summary_text, "rooms": rooms, } @@ -93,6 +114,38 @@ class Mailer(object): room_vars['title'] = calculate_room_name(room_state, user_id) return room_vars + @defer.inlineCallbacks + def make_summary_text(self, notifs_by_room, state_by_room, user_id): + if len(notifs_by_room) == 1: + room_id = notifs_by_room.keys()[0] + sender_name = None + if len(notifs_by_room[room_id]) == 1: + # If the room has some kind of name, use it, but we don't + # want the generated-from-names one here otherwise we'll + # end up with, "new message from Bob in the Bob room" + room_name = calculate_room_name( + state_by_room[room_id], user_id, fallback_to_members=False + ) + event = yield self.store.get_event( + notifs_by_room[room_id][0]["event_id"] + ) + if ("m.room.member", event.sender) in state_by_room[room_id]: + state_event = state_by_room[room_id][("m.room.member", event.sender)] + sender_name = name_from_member_event(state_event) + if sender_name is not None and room_name is not None: + defer.returnValue( + MESSAGE_FROM_PERSON_IN_ROOM % (sender_name, room_name) + ) + elif sender_name is not None: + defer.returnValue(MESSAGE_FROM_PERSON % (sender_name,)) + else: + room_name = calculate_room_name(state_by_room[room_id], user_id) + defer.returnValue(MESSAGES_IN_ROOM % (room_name,)) + else: + defer.returnValue(MESSAGES_IN_ROOMS) + + defer.returnValue("Some thing have occurred in some rooms") + def make_unsubscribe_link(self): return "https://vector.im/#/settings" # XXX: matrix.to @@ -104,4 +157,4 @@ def deduped_ordered_list(l): if item not in seen: seen.add(item) ret.append(item) - return ret \ No newline at end of file + return ret diff --git a/synapse/util/room_name.py b/synapse/util/presentable_names.py similarity index 91% rename from synapse/util/room_name.py rename to synapse/util/presentable_names.py index f55ef293b..2ae01e453 100644 --- a/synapse/util/room_name.py +++ b/synapse/util/presentable_names.py @@ -22,7 +22,7 @@ ALIAS_RE = re.compile(r"^#.*:.+$") ALL_ALONE = "Empty Room" -def calculate_room_name(room_state, user_id): +def calculate_room_name(room_state, user_id, fallback_to_members=True): # does it have a name? if ("m.room.name", "") in room_state: m_room_name = room_state[("m.room.name", "")] @@ -34,13 +34,13 @@ def calculate_room_name(room_state, user_id): canon_alias = room_state[("m.room.canonical_alias", "")] if ( canon_alias.content and canon_alias.content["alias"] and - looks_like_an_alias(canon_alias.content["alias"]) + _looks_like_an_alias(canon_alias.content["alias"]) ): return canon_alias.content["alias"] # at this point we're going to need to search the state by all state keys # for an event type, so rearrange the data structure - room_state_bytype = state_as_two_level_dict(room_state) + room_state_bytype = _state_as_two_level_dict(room_state) # right then, any aliases at all? if "m.room.aliases" in room_state_bytype: @@ -49,7 +49,7 @@ def calculate_room_name(room_state, user_id): first_alias_event = m_room_aliases.values()[0] if first_alias_event.content and first_alias_event.content["aliases"]: the_aliases = first_alias_event.content["aliases"] - if len(the_aliases) > 0 and looks_like_an_alias(the_aliases[0]): + if len(the_aliases) > 0 and _looks_like_an_alias(the_aliases[0]): return the_aliases[0] my_member_event = None @@ -66,6 +66,9 @@ def calculate_room_name(room_state, user_id): else: return "Room Invite" + if not fallback_to_members: + return None + # we're going to have to generate a name based on who's in the room, # so find out who is in the room that isn't the user. if "m.room.member" in room_state_bytype: @@ -105,17 +108,6 @@ def calculate_room_name(room_state, user_id): return descriptor_from_member_events(other_members) -def state_as_two_level_dict(state): - ret = {} - for k, v in state.items(): - ret.setdefault(k[0], {})[k[1]] = v - return ret - - -def looks_like_an_alias(string): - return ALIAS_RE.match(string) is not None - - def descriptor_from_member_events(member_events): if len(member_events) == 0: return "nobody" @@ -139,4 +131,15 @@ def name_from_member_event(member_event): member_event.content["displayname"] ): return member_event.content["displayname"] - return member_event.state_key \ No newline at end of file + return member_event.state_key + + +def _state_as_two_level_dict(state): + ret = {} + for k, v in state.items(): + ret.setdefault(k[0], {})[k[1]] = v + return ret + + +def _looks_like_an_alias(string): + return ALIAS_RE.match(string) is not None \ No newline at end of file From fa12209c1b297a1710f487744a8a143d6cb6a2d1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 27 Apr 2016 15:09:55 +0100 Subject: [PATCH 016/134] Hopefully all remaining bits for email notifs Add public facing base url to the server so synapse knows what URL to use when converting mxc to http urls for use in emails --- res/templates/notif.html | 15 --- res/templates/notif_mail.html | 15 +++ res/templates/room.html | 23 ++++- synapse/config/emailconfig.py | 9 +- synapse/config/server.py | 8 ++ synapse/push/mailer.py | 166 ++++++++++++++++++++++++++++----- synapse/python_dependencies.py | 1 + 7 files changed, 195 insertions(+), 42 deletions(-) delete mode 100644 res/templates/notif.html create mode 100644 res/templates/notif_mail.html diff --git a/res/templates/notif.html b/res/templates/notif.html deleted file mode 100644 index aee52ec8c..000000000 --- a/res/templates/notif.html +++ /dev/null @@ -1,15 +0,0 @@ - - - -
Hi {{ user_display_name }},
-
{{ summary_text }}
-
- {% for room in rooms %} - {% include 'room.html' with context %} - {% endfor %} -
-
- Unsubscribe -
- - diff --git a/res/templates/notif_mail.html b/res/templates/notif_mail.html new file mode 100644 index 000000000..fbfb0a767 --- /dev/null +++ b/res/templates/notif_mail.html @@ -0,0 +1,15 @@ + + + +
Hi {{ user_display_name }},
+
{{ summary_text }}
+
+ {% for room in rooms %} + {% include 'room.html' with context %} + {% endfor %} +
+
+ Unsubscribe +
+ + diff --git a/res/templates/room.html b/res/templates/room.html index ef36b4ee5..f369575b9 100644 --- a/res/templates/room.html +++ b/res/templates/room.html @@ -1,6 +1,21 @@
-

{{ room.title }}

-
- Things have happened in this room -
+

{{ room.title }}

+
+ {% if room.avatar_url %} + + {% else %} + {% if room.hash % 3 == 0 %} + + {% elif room.hash % 3 == 1 %} + + {% else %} + + {% endif %} + {% endif %} +
+
+ {% for notif in room.notifs %} + {% include 'notif.html' with context %} + {% endfor %} +
diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 68fb4d806..893034e2e 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -25,17 +25,19 @@ class EmailConfig(Config): """ def read_config(self, config): + self.email_enable_notifs = False + email_config = config.get("email", None) if email_config: self.email_enable_notifs = email_config.get("enable_notifs", True) + if self.email_enable_notifs: required = [ "smtp_host", "smtp_port", "notif_from", "template_dir", "notif_template_html", - ] missing = [] @@ -49,6 +51,11 @@ class EmailConfig(Config): (", ".join(["email."+k for k in missing]),) ) + if config.get("public_baseurl") is None: + raise RuntimeError( + "email.enable_notifs is True but no public_baseurl is set" + ) + self.email_smtp_host = email_config["smtp_host"] self.email_smtp_port = email_config["smtp_port"] self.email_notif_from = email_config["notif_from"] diff --git a/synapse/config/server.py b/synapse/config/server.py index df4707e1d..19af39da7 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -28,6 +28,11 @@ class ServerConfig(Config): self.print_pidfile = config.get("print_pidfile") self.user_agent_suffix = config.get("user_agent_suffix") self.use_frozen_dicts = config.get("use_frozen_dicts", True) + self.public_baseurl = config.get("public_baseurl") + + if self.public_baseurl is not None: + if self.public_baseurl[-1] != '/': + self.public_baseurl += '/' self.listeners = config.get("listeners", []) @@ -142,6 +147,9 @@ class ServerConfig(Config): # Whether to serve a web client from the HTTP/HTTPS root resource. web_client: True + # The server's public-facing base URL + # https://example.com:8448/ + # Set the soft limit on the number of file descriptors synapse can use # Zero is used to indicate synapse should set the soft limit to the # hard limit. diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 9e2297a03..e78c26ede 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -26,6 +26,10 @@ from synapse.types import UserID from synapse.api.errors import StoreError import jinja2 +import bleach + +import time +import urllib MESSAGE_FROM_PERSON_IN_ROOM = "You have a message from %s in the %s room" @@ -33,6 +37,27 @@ MESSAGE_FROM_PERSON = "You have a message from %s" MESSAGES_IN_ROOM = "There are some messages for you in the %s room" MESSAGES_IN_ROOMS = "Here are some messages you may have missed" +CONTEXT_BEFORE = 1 + +# From https://github.com/matrix-org/matrix-react-sdk/blob/master/src/HtmlUtils.js +ALLOWED_TAGS = [ + 'font', # custom to matrix for IRC-style font coloring + 'del', # for markdown + # deliberately no h1/h2 to stop people shouting. + 'h3', 'h4', 'h5', 'h6', 'blockquote', 'p', 'a', 'ul', 'ol', + 'nl', 'li', 'b', 'i', 'u', 'strong', 'em', 'strike', 'code', 'hr', 'br', 'div', + 'table', 'thead', 'caption', 'tbody', 'tr', 'th', 'td', 'pre' +] +ALLOWED_ATTRS = { + # custom ones first: + "font": ["color"], # custom to matrix + "a": ["href", "name", "target"], # remote target: custom to matrix + # We don't currently allow img itself by default, but this + # would make sense if we did + "img": ["src"], +} +ALLOWED_SCHEMES = ["http", "https", "ftp", "mailto"] + class Mailer(object): def __init__(self, hs): @@ -41,6 +66,8 @@ class Mailer(object): self.state_handler = self.hs.get_state_handler() loader = jinja2.FileSystemLoader(self.hs.config.email_template_dir) env = jinja2.Environment(loader=loader) + env.filters["format_ts"] = format_ts_filter + env.filters["mxc_to_http"] = self.mxc_to_http_filter self.notif_template = env.get_template(self.hs.config.email_notif_template_html) @defer.inlineCallbacks @@ -55,6 +82,10 @@ class Mailer(object): [pa['room_id'] for pa in push_actions] ) + notif_events = yield self.store.get_events( + [pa['event_id'] for pa in push_actions] + ) + notifs_by_room = {} for pa in push_actions: notifs_by_room.setdefault(pa["room_id"], []).append(pa) @@ -79,14 +110,16 @@ class Mailer(object): # notifs are much realtime than sync so we can afford to wait a bit. yield concurrently_execute(_fetch_room_state, rooms_in_order, 3) - rooms = [ - self.get_room_vars( - r, user_id, notifs_by_room[r], state_by_room[r] - ) for r in rooms_in_order - ] + rooms = [] - summary_text = yield self.make_summary_text( - notifs_by_room, state_by_room, user_id + for r in rooms_in_order: + vars = yield self.get_room_vars( + r, user_id, notifs_by_room[r], notif_events, state_by_room[r] + ) + rooms.append(vars) + + summary_text = self.make_summary_text( + notifs_by_room, state_by_room, notif_events, user_id ) template_vars = { @@ -109,13 +142,72 @@ class Mailer(object): port=self.hs.config.email_smtp_port ) - def get_room_vars(self, room_id, user_id, notifs, room_state): - room_vars = {} - room_vars['title'] = calculate_room_name(room_state, user_id) - return room_vars + @defer.inlineCallbacks + def get_room_vars(self, room_id, user_id, notifs, notif_events, room_state): + room_vars = { + "title": calculate_room_name(room_state, user_id), + "hash": string_ordinal_total(room_id), # See sender avatar hash + "notifs": [], + } + + for n in notifs: + vars = yield self.get_notif_vars(n, notif_events[n['event_id']], room_state) + room_vars['notifs'].append(vars) + + defer.returnValue(room_vars) @defer.inlineCallbacks - def make_summary_text(self, notifs_by_room, state_by_room, user_id): + def get_notif_vars(self, notif, notif_event, room_state): + results = yield self.store.get_events_around( + notif['room_id'], notif['event_id'], + before_limit=CONTEXT_BEFORE, after_limit=0 + ) + + ret = { + "link": self.make_notif_link(notif), + "ts": notif['received_ts'], + "messages": [], + } + + for event in results['events_before']: + vars = self.get_message_vars(notif, event, room_state) + if vars is not None: + ret['messages'].append(vars) + + vars = self.get_message_vars(notif, notif_event, room_state) + if vars is not None: + ret['messages'].append(vars) + + defer.returnValue(ret) + + def get_message_vars(self, notif, event, room_state): + msgtype = event.content["msgtype"] + + sender_state_event = room_state[("m.room.member", event.sender)] + sender_name = name_from_member_event(sender_state_event) + sender_avatar_url = sender_state_event.content["avatar_url"] + + # 'hash' for deterministically picking default images: use + # sender_hash % the number of default images to choose from + sender_hash = string_ordinal_total(event.sender) + + ret = { + "msgtype": msgtype, + "is_historical": event.event_id != notif['event_id'], + "ts": event.origin_server_ts, + "sender_name": sender_name, + "sender_avatar_url": sender_avatar_url, + "sender_hash": sender_hash, + } + + if msgtype == "m.text": + ret["body_text_plain"] = event.content["body"] + elif msgtype == "org.matrix.custom.html": + ret["body_text_html"] = safe_markup(event.content["formatted_body"]) + + return ret + + def make_summary_text(self, notifs_by_room, state_by_room, notif_events, user_id): if len(notifs_by_room) == 1: room_id = notifs_by_room.keys()[0] sender_name = None @@ -126,29 +218,50 @@ class Mailer(object): room_name = calculate_room_name( state_by_room[room_id], user_id, fallback_to_members=False ) - event = yield self.store.get_event( - notifs_by_room[room_id][0]["event_id"] - ) + event = notif_events[notifs_by_room[room_id][0]["event_id"]] if ("m.room.member", event.sender) in state_by_room[room_id]: state_event = state_by_room[room_id][("m.room.member", event.sender)] sender_name = name_from_member_event(state_event) if sender_name is not None and room_name is not None: - defer.returnValue( - MESSAGE_FROM_PERSON_IN_ROOM % (sender_name, room_name) - ) + return MESSAGE_FROM_PERSON_IN_ROOM % (sender_name, room_name) elif sender_name is not None: - defer.returnValue(MESSAGE_FROM_PERSON % (sender_name,)) + return MESSAGE_FROM_PERSON % (sender_name,) else: room_name = calculate_room_name(state_by_room[room_id], user_id) - defer.returnValue(MESSAGES_IN_ROOM % (room_name,)) + return MESSAGES_IN_ROOM % (room_name,) else: - defer.returnValue(MESSAGES_IN_ROOMS) + return MESSAGES_IN_ROOMS - defer.returnValue("Some thing have occurred in some rooms") + def make_notif_link(self, notif): + return "https://matrix.to/%s/%s" % ( + notif['room_id'], notif['event_id'] + ) def make_unsubscribe_link(self): return "https://vector.im/#/settings" # XXX: matrix.to + def mxc_to_http_filter(self, value, width, height, resizeMethod="crop"): + if value[0:6] != "mxc://": + return "" + serverAndMediaId = value[6:] + params = { + "width": width, + "height": height, + "method": resizeMethod, + } + return "%s_matrix/media/v1/thumbnail/%s?%s" % ( + self.hs.config.public_baseurl, + serverAndMediaId, + urllib.urlencode(params) + ) + + +def safe_markup(self, raw_html): + return jinja2.Markup(bleach.linkify(bleach.clean( + raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRS, + protocols=ALLOWED_SCHEMES, strip=True + ))) + def deduped_ordered_list(l): seen = set() @@ -158,3 +271,12 @@ def deduped_ordered_list(l): seen.add(item) ret.append(item) return ret + +def string_ordinal_total(s): + tot = 0 + for c in s: + tot += ord(c) + return tot + +def format_ts_filter(value, format): + return time.strftime(format, time.localtime(value / 1000)) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 16524dbdc..618f3c43a 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -47,6 +47,7 @@ CONDITIONAL_REQUIREMENTS = { }, "email.enable_notifs": { "Jinja2>=2.8": ["Jinja2>=2.8"], + "bleach>=1.4.2": ["bleach>=1.4.2"], }, } From 8781083960325e7aae16b1745c8e90561fde3d35 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 27 Apr 2016 15:30:41 +0100 Subject: [PATCH 017/134] Better grammar for multiple messages in a room Say who the messages are from if there's no room name, otherwise it's a bit nonsensical --- synapse/push/emailpusher.py | 2 +- synapse/push/mailer.py | 41 +++++++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index dcbee4c3f..6ae16e986 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) # The amount of time we always wait before ever emailing about a notification # (to give the user a chance to respond to other push or notice the window) -DELAY_BEFORE_MAIL_MS = 2 * 60 * 1000 +DELAY_BEFORE_MAIL_MS = 2000#2 * 60 * 1000 THROTTLE_START_MS = 2 * 60 * 1000 THROTTLE_MAX_MS = (2 * 60 * 1000) * (2 ** 11) # ~3 days diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index e78c26ede..272e372be 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -21,7 +21,9 @@ import email.mime.multipart from email.mime.text import MIMEText from synapse.util.async import concurrently_execute -from synapse.util.presentable_names import calculate_room_name, name_from_member_event +from synapse.util.presentable_names import ( + calculate_room_name, name_from_member_event, descriptor_from_member_events +) from synapse.types import UserID from synapse.api.errors import StoreError @@ -34,6 +36,7 @@ import urllib MESSAGE_FROM_PERSON_IN_ROOM = "You have a message from %s in the %s room" MESSAGE_FROM_PERSON = "You have a message from %s" +MESSAGES_FROM_PERSON = "You have messages from %s" MESSAGES_IN_ROOM = "There are some messages for you in the %s room" MESSAGES_IN_ROOMS = "Here are some messages you may have missed" @@ -209,15 +212,19 @@ class Mailer(object): def make_summary_text(self, notifs_by_room, state_by_room, notif_events, user_id): if len(notifs_by_room) == 1: + # Only one room has new stuff room_id = notifs_by_room.keys()[0] + + # If the room has some kind of name, use it, but we don't + # want the generated-from-names one here otherwise we'll + # end up with, "new message from Bob in the Bob room" + room_name = calculate_room_name( + state_by_room[room_id], user_id, fallback_to_members=False + ) + sender_name = None if len(notifs_by_room[room_id]) == 1: - # If the room has some kind of name, use it, but we don't - # want the generated-from-names one here otherwise we'll - # end up with, "new message from Bob in the Bob room" - room_name = calculate_room_name( - state_by_room[room_id], user_id, fallback_to_members=False - ) + # There is just the one notification, so give some detail event = notif_events[notifs_by_room[room_id][0]["event_id"]] if ("m.room.member", event.sender) in state_by_room[room_id]: state_event = state_by_room[room_id][("m.room.member", event.sender)] @@ -227,9 +234,25 @@ class Mailer(object): elif sender_name is not None: return MESSAGE_FROM_PERSON % (sender_name,) else: - room_name = calculate_room_name(state_by_room[room_id], user_id) - return MESSAGES_IN_ROOM % (room_name,) + # There's more than one notification for this room, so just + # say there are several + if room_name is not None: + return MESSAGES_IN_ROOM % (room_name,) + else: + # If the room doesn't have a name, say who the messages + # are from explicitly to avoid, "messages in the Bob room" + sender_ids = list(set([ + notif_events[n['event_id']].sender + for n in notifs_by_room[room_id] + ])) + + return MESSAGES_FROM_PERSON % ( + descriptor_from_member_events([ + state_by_room[room_id][("m.room.member", s)] for s in sender_ids + ]) + ) else: + # Stuff's happened in multiple different rooms return MESSAGES_IN_ROOMS def make_notif_link(self, notif): From 4ed1e45869cf59b517df38831a373a1a770e5917 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 27 Apr 2016 17:18:51 +0100 Subject: [PATCH 018/134] Make html messages work --- synapse/push/mailer.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 272e372be..a4a0891e0 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -59,7 +59,8 @@ ALLOWED_ATTRS = { # would make sense if we did "img": ["src"], } -ALLOWED_SCHEMES = ["http", "https", "ftp", "mailto"] +# When bleach release a version with this option, we can specify schemes +#ALLOWED_SCHEMES = ["http", "https", "ftp", "mailto"] class Mailer(object): @@ -184,7 +185,15 @@ class Mailer(object): defer.returnValue(ret) def get_message_vars(self, notif, event, room_state): - msgtype = event.content["msgtype"] + if event.type != "m.room.message": + return None + if event.content["msgtype"] != "m.text": + return None + + if "format" in event.content: + msgformat = event.content["format"] + else: + msgformat = None sender_state_event = room_state[("m.room.member", event.sender)] sender_name = name_from_member_event(sender_state_event) @@ -195,7 +204,7 @@ class Mailer(object): sender_hash = string_ordinal_total(event.sender) ret = { - "msgtype": msgtype, + "format": msgformat, "is_historical": event.event_id != notif['event_id'], "ts": event.origin_server_ts, "sender_name": sender_name, @@ -203,10 +212,10 @@ class Mailer(object): "sender_hash": sender_hash, } - if msgtype == "m.text": - ret["body_text_plain"] = event.content["body"] - elif msgtype == "org.matrix.custom.html": + if msgformat == "org.matrix.custom.html": ret["body_text_html"] = safe_markup(event.content["formatted_body"]) + else: + ret["body_text_plain"] = event.content["body"] return ret @@ -263,14 +272,14 @@ class Mailer(object): def make_unsubscribe_link(self): return "https://vector.im/#/settings" # XXX: matrix.to - def mxc_to_http_filter(self, value, width, height, resizeMethod="crop"): + def mxc_to_http_filter(self, value, width, height, resize_method="crop"): if value[0:6] != "mxc://": return "" serverAndMediaId = value[6:] params = { "width": width, "height": height, - "method": resizeMethod, + "method": resize_method, } return "%s_matrix/media/v1/thumbnail/%s?%s" % ( self.hs.config.public_baseurl, @@ -279,10 +288,12 @@ class Mailer(object): ) -def safe_markup(self, raw_html): +def safe_markup(raw_html): return jinja2.Markup(bleach.linkify(bleach.clean( raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRS, - protocols=ALLOWED_SCHEMES, strip=True + # bleach master has this, but it isn't released yet + # protocols=ALLOWED_SCHEMES, + strip=True ))) From 424a7f48f8d5cdb97ec3567d6841cecbf65ffda2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 27 Apr 2016 17:50:49 +0100 Subject: [PATCH 019/134] Run filter_events_for_client so we don't accidentally mail out events people shouldn't see --- synapse/handlers/_base.py | 2 +- synapse/handlers/message.py | 8 ++++---- synapse/handlers/room.py | 2 +- synapse/handlers/search.py | 8 ++++---- synapse/handlers/sync.py | 4 ++-- synapse/notifier.py | 2 +- synapse/push/mailer.py | 18 +++++++++++------- 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 13a675b20..134729069 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -192,7 +192,7 @@ class BaseHandler(object): }) @defer.inlineCallbacks - def _filter_events_for_client(self, user_id, events, is_peeking=False): + def filter_events_for_client(self, user_id, events, is_peeking=False): """ Check which events a user is allowed to see diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index f51feda2f..7d9e3cf36 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -123,7 +123,7 @@ class MessageHandler(BaseHandler): "end": next_token.to_string(), }) - events = yield self._filter_events_for_client( + events = yield self.filter_events_for_client( user_id, events, is_peeking=(member_event_id is None), @@ -483,7 +483,7 @@ class MessageHandler(BaseHandler): ] ).addErrback(unwrapFirstError) - messages = yield self._filter_events_for_client( + messages = yield self.filter_events_for_client( user_id, messages ) @@ -619,7 +619,7 @@ class MessageHandler(BaseHandler): end_token=stream_token ) - messages = yield self._filter_events_for_client( + messages = yield self.filter_events_for_client( user_id, messages, is_peeking=is_peeking ) @@ -700,7 +700,7 @@ class MessageHandler(BaseHandler): consumeErrors=True, ).addErrback(unwrapFirstError) - messages = yield self._filter_events_for_client( + messages = yield self.filter_events_for_client( user_id, messages, is_peeking=is_peeking, ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index dd9c18df8..fdebc9c43 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -449,7 +449,7 @@ class RoomContextHandler(BaseHandler): now_token = yield self.hs.get_event_sources().get_current_token() def filter_evts(events): - return self._filter_events_for_client( + return self.filter_events_for_client( user.to_string(), events, is_peeking=is_guest) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 9937d8dd7..a937e8740 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -172,7 +172,7 @@ class SearchHandler(BaseHandler): filtered_events = search_filter.filter([r["event"] for r in results]) - events = yield self._filter_events_for_client( + events = yield self.filter_events_for_client( user.to_string(), filtered_events ) @@ -223,7 +223,7 @@ class SearchHandler(BaseHandler): r["event"] for r in results ]) - events = yield self._filter_events_for_client( + events = yield self.filter_events_for_client( user.to_string(), filtered_events ) @@ -281,11 +281,11 @@ class SearchHandler(BaseHandler): event.room_id, event.event_id, before_limit, after_limit ) - res["events_before"] = yield self._filter_events_for_client( + res["events_before"] = yield self.filter_events_for_client( user.to_string(), res["events_before"] ) - res["events_after"] = yield self._filter_events_for_client( + res["events_after"] = yield self.filter_events_for_client( user.to_string(), res["events_after"] ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 231140b65..b51bb651e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -681,7 +681,7 @@ class SyncHandler(BaseHandler): if recents is not None: recents = sync_config.filter_collection.filter_room_timeline(recents) - recents = yield self._filter_events_for_client( + recents = yield self.filter_events_for_client( sync_config.user.to_string(), recents, ) @@ -702,7 +702,7 @@ class SyncHandler(BaseHandler): loaded_recents = sync_config.filter_collection.filter_room_timeline( events ) - loaded_recents = yield self._filter_events_for_client( + loaded_recents = yield self.filter_events_for_client( sync_config.user.to_string(), loaded_recents, ) diff --git a/synapse/notifier.py b/synapse/notifier.py index 6af7a8f42..cb58dfffd 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -399,7 +399,7 @@ class Notifier(object): if name == "room": room_member_handler = self.hs.get_handlers().room_member_handler - new_events = yield room_member_handler._filter_events_for_client( + new_events = yield room_member_handler.filter_events_for_client( user.to_string(), new_events, is_peeking=is_peeking, diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index a4a0891e0..afdf43966 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -155,13 +155,15 @@ class Mailer(object): } for n in notifs: - vars = yield self.get_notif_vars(n, notif_events[n['event_id']], room_state) + vars = yield self.get_notif_vars( + n, user_id, notif_events[n['event_id']], room_state + ) room_vars['notifs'].append(vars) defer.returnValue(room_vars) @defer.inlineCallbacks - def get_notif_vars(self, notif, notif_event, room_state): + def get_notif_vars(self, notif, user_id, notif_event, room_state): results = yield self.store.get_events_around( notif['room_id'], notif['event_id'], before_limit=CONTEXT_BEFORE, after_limit=0 @@ -173,15 +175,17 @@ class Mailer(object): "messages": [], } - for event in results['events_before']: + handler = self.hs.get_handlers().message_handler + the_events = yield handler.filter_events_for_client( + user_id, results["events_before"] + ) + the_events.append(notif_event) + + for event in the_events: vars = self.get_message_vars(notif, event, room_state) if vars is not None: ret['messages'].append(vars) - vars = self.get_message_vars(notif, notif_event, room_state) - if vars is not None: - ret['messages'].append(vars) - defer.returnValue(ret) def get_message_vars(self, notif, event, room_state): From 9dba1b668ce1d2ea4db1fb6bfae3df319e7c76d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 10:55:08 +0100 Subject: [PATCH 020/134] Linkify plain text messages too --- synapse/push/mailer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index afdf43966..7ef64f8f6 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -219,7 +219,7 @@ class Mailer(object): if msgformat == "org.matrix.custom.html": ret["body_text_html"] = safe_markup(event.content["formatted_body"]) else: - ret["body_text_plain"] = event.content["body"] + ret["body_text_html"] = safe_text(event.content["body"]) return ret @@ -301,6 +301,17 @@ def safe_markup(raw_html): ))) +def safe_text(raw_text): + """ + Process text: treat it as HTML but escape any tags (ie. just escape the + HTML) then linkify it. + """ + return jinja2.Markup(bleach.linkify(bleach.clean( + raw_text, tags=[], attributes={}, + strip=False + ))) + + def deduped_ordered_list(l): seen = set() ret = [] From 5367708236934a2b191991f1e861ec0d86c02be4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 10:55:32 +0100 Subject: [PATCH 021/134] Add the jinja template for individual notifs --- res/templates/notif.html | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 res/templates/notif.html diff --git a/res/templates/notif.html b/res/templates/notif.html new file mode 100644 index 000000000..aa6ed1e06 --- /dev/null +++ b/res/templates/notif.html @@ -0,0 +1,32 @@ +
+
+ {% for message in notif.messages %} +
+ {% if message.sender_avatar_url %} + + {% else %} + {% if message.sender_hash % 3 == 0 %} + + {% elif message.sender_hash % 3 == 1 %} + + {% else %} + + {% endif %} + + {% endif %} +
{{ message.sender_name }}
+
{{ message.ts|format_ts("%H:%M") }}
+
+ {% if message.format == "org.matrix.custom.html" %} + {{ message.body_text_html }} + {% else %} + {{ message.body_text_plain }} + {% endif %} +
+
+ {% endfor %} +
+
+ View in Vector +
+
From ebbabc4986371c83d1d2659d10b27caad9b47951 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 11:49:36 +0100 Subject: [PATCH 022/134] Handle room invites in email notifs --- res/templates/room.html | 14 ++++++++----- synapse/push/mailer.py | 35 ++++++++++++++++++++++++++----- synapse/util/presentable_names.py | 6 +++--- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/res/templates/room.html b/res/templates/room.html index f369575b9..6c68ee1fd 100644 --- a/res/templates/room.html +++ b/res/templates/room.html @@ -13,9 +13,13 @@ {% endif %} {% endif %}
-
- {% for notif in room.notifs %} - {% include 'notif.html' with context %} - {% endfor %} -
+ {% if room.invite %} + Join the conversation. + {% else %} +
+ {% for notif in room.notifs %} + {% include 'notif.html' with context %} + {% endfor %} +
+ {% endif %} diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 7ef64f8f6..d2cf24765 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -39,6 +39,8 @@ MESSAGE_FROM_PERSON = "You have a message from %s" MESSAGES_FROM_PERSON = "You have messages from %s" MESSAGES_IN_ROOM = "There are some messages for you in the %s room" MESSAGES_IN_ROOMS = "Here are some messages you may have missed" +INVITE_FROM_PERSON_TO_ROOM = "%s has invited you to join the %s room" +INVITE_FROM_PERSON = "%s has invited you to chat" CONTEXT_BEFORE = 1 @@ -148,17 +150,24 @@ class Mailer(object): @defer.inlineCallbacks def get_room_vars(self, room_id, user_id, notifs, notif_events, room_state): + my_member_event = room_state[("m.room.member", user_id)] + is_invite = my_member_event.content["membership"] == "invite" + room_vars = { "title": calculate_room_name(room_state, user_id), "hash": string_ordinal_total(room_id), # See sender avatar hash "notifs": [], + "invite": is_invite } - for n in notifs: - vars = yield self.get_notif_vars( - n, user_id, notif_events[n['event_id']], room_state - ) - room_vars['notifs'].append(vars) + if is_invite: + room_vars["link"] = self.make_room_link(room_id) + else: + for n in notifs: + vars = yield self.get_notif_vars( + n, user_id, notif_events[n['event_id']], room_state + ) + room_vars['notifs'].append(vars) defer.returnValue(room_vars) @@ -235,6 +244,18 @@ class Mailer(object): state_by_room[room_id], user_id, fallback_to_members=False ) + my_member_event = state_by_room[room_id][("m.room.member", user_id)] + if my_member_event.content["membership"] == "invite": + inviter_member_event = state_by_room[room_id][ + ("m.room.member", my_member_event.sender) + ] + inviter_name = name_from_member_event(inviter_member_event) + + if room_name is None: + return INVITE_FROM_PERSON % (inviter_name,) + else: + return INVITE_FROM_PERSON_TO_ROOM % (inviter_name, room_name) + sender_name = None if len(notifs_by_room[room_id]) == 1: # There is just the one notification, so give some detail @@ -242,6 +263,7 @@ class Mailer(object): if ("m.room.member", event.sender) in state_by_room[room_id]: state_event = state_by_room[room_id][("m.room.member", event.sender)] sender_name = name_from_member_event(state_event) + if sender_name is not None and room_name is not None: return MESSAGE_FROM_PERSON_IN_ROOM % (sender_name, room_name) elif sender_name is not None: @@ -268,6 +290,9 @@ class Mailer(object): # Stuff's happened in multiple different rooms return MESSAGES_IN_ROOMS + def make_room_link(self, room_id): + return "https://matrix.to/%s" % (room_id,) + def make_notif_link(self, notif): return "https://matrix.to/%s/%s" % ( notif['room_id'], notif['event_id'] diff --git a/synapse/util/presentable_names.py b/synapse/util/presentable_names.py index 2ae01e453..f80a7fe58 100644 --- a/synapse/util/presentable_names.py +++ b/synapse/util/presentable_names.py @@ -52,6 +52,9 @@ def calculate_room_name(room_state, user_id, fallback_to_members=True): if len(the_aliases) > 0 and _looks_like_an_alias(the_aliases[0]): return the_aliases[0] + if not fallback_to_members: + return None + my_member_event = None if ("m.room.member", user_id) in room_state: my_member_event = room_state[("m.room.member", user_id)] @@ -66,9 +69,6 @@ def calculate_room_name(room_state, user_id, fallback_to_members=True): else: return "Room Invite" - if not fallback_to_members: - return None - # we're going to have to generate a name based on who's in the room, # so find out who is in the room that isn't the user. if "m.room.member" in room_state_bytype: From 937c407eef78648f1c4c1afe56fdfd8598315b19 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 15:12:14 +0100 Subject: [PATCH 023/134] Only import email pusher if email notifs are on --- synapse/push/pusher.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index f7c3021fc..25a45af77 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -1,12 +1,14 @@ from httppusher import HttpPusher -from emailpusher import EmailPusher - -PUSHER_TYPES = { - 'http': HttpPusher, - 'email': EmailPusher, -} def create_pusher(hs, pusherdict): + PUSHER_TYPES = { + "http": HttpPusher, + } + + if hs.config.email_enable_notifs: + from emailpusher import EmailPusher + PUSHER_TYPES["email"] = EmailPusher + if pusherdict['kind'] in PUSHER_TYPES: return PUSHER_TYPES[pusherdict['kind']](hs, pusherdict) From 60f86fc876bcb106842d804baac9aff4860ece3b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 15:16:30 +0100 Subject: [PATCH 024/134] pep8 --- synapse/config/emailconfig.py | 2 +- synapse/push/mailer.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 893034e2e..06b076e3f 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -48,7 +48,7 @@ class EmailConfig(Config): if (len(missing) > 0): raise RuntimeError( "email.enable_notifs is True but required keys are missing: %s" % - (", ".join(["email."+k for k in missing]),) + (", ".join(["email." + k for k in missing]),) ) if config.get("public_baseurl") is None: diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index d2cf24765..ae3e41b8c 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -62,7 +62,7 @@ ALLOWED_ATTRS = { "img": ["src"], } # When bleach release a version with this option, we can specify schemes -#ALLOWED_SCHEMES = ["http", "https", "ftp", "mailto"] +# ALLOWED_SCHEMES = ["http", "https", "ftp", "mailto"] class Mailer(object): @@ -283,7 +283,8 @@ class Mailer(object): return MESSAGES_FROM_PERSON % ( descriptor_from_member_events([ - state_by_room[room_id][("m.room.member", s)] for s in sender_ids + state_by_room[room_id][("m.room.member", s)] + for s in sender_ids ]) ) else: @@ -346,11 +347,13 @@ def deduped_ordered_list(l): ret.append(item) return ret + def string_ordinal_total(s): tot = 0 for c in s: tot += ord(c) return tot + def format_ts_filter(value, format): return time.strftime(format, time.localtime(value / 1000)) From 4845c7359de6c1ad1e7132653939e44ee8ff9156 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 15:55:53 +0100 Subject: [PATCH 025/134] Support image notifs --- res/templates/notif.html | 12 ++++++++---- synapse/push/mailer.py | 36 +++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/res/templates/notif.html b/res/templates/notif.html index aa6ed1e06..bdff2786f 100644 --- a/res/templates/notif.html +++ b/res/templates/notif.html @@ -17,10 +17,14 @@
{{ message.sender_name }}
{{ message.ts|format_ts("%H:%M") }}
- {% if message.format == "org.matrix.custom.html" %} - {{ message.body_text_html }} - {% else %} - {{ message.body_text_plain }} + {% if message.msgtype == "m.text" %} + {% if message.format == "org.matrix.custom.html" %} + {{ message.body_text_html }} + {% else %} + {{ message.body_text_plain }} + {% endif %} + {% elif message.msgtype == "m.image" %} + {% endif %}
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index ae3e41b8c..e6554dc7f 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -200,13 +200,6 @@ class Mailer(object): def get_message_vars(self, notif, event, room_state): if event.type != "m.room.message": return None - if event.content["msgtype"] != "m.text": - return None - - if "format" in event.content: - msgformat = event.content["format"] - else: - msgformat = None sender_state_event = room_state[("m.room.member", event.sender)] sender_name = name_from_member_event(sender_state_event) @@ -217,7 +210,7 @@ class Mailer(object): sender_hash = string_ordinal_total(event.sender) ret = { - "format": msgformat, + "msgtype": event.content["msgtype"], "is_historical": event.event_id != notif['event_id'], "ts": event.origin_server_ts, "sender_name": sender_name, @@ -225,13 +218,34 @@ class Mailer(object): "sender_hash": sender_hash, } - if msgformat == "org.matrix.custom.html": - ret["body_text_html"] = safe_markup(event.content["formatted_body"]) + if event.content["msgtype"] == "m.text": + self.add_text_message_vars(ret, event) + elif event.content["msgtype"] == "m.image": + self.add_image_message_vars(ret, event) else: - ret["body_text_html"] = safe_text(event.content["body"]) + return None return ret + def add_text_message_vars(self, vars, event): + if "format" in event.content: + msgformat = event.content["format"] + else: + msgformat = None + vars["format"] = msgformat + + if msgformat == "org.matrix.custom.html": + vars["body_text_html"] = safe_markup(event.content["formatted_body"]) + else: + vars["body_text_html"] = safe_text(event.content["body"]) + + return vars + + def add_image_message_vars(self, vars, event): + vars["image_url"] = event.content["url"] + + return vars + def make_summary_text(self, notifs_by_room, state_by_room, notif_events, user_id): if len(notifs_by_room) == 1: # Only one room has new stuff From 68f8fc2f143d44b560b07e7521b28aae332e243d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 16:59:57 +0100 Subject: [PATCH 026/134] Support file messages & fix plain text --- res/templates/notif.html | 8 +++----- synapse/push/mailer.py | 5 +++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/res/templates/notif.html b/res/templates/notif.html index bdff2786f..70f565535 100644 --- a/res/templates/notif.html +++ b/res/templates/notif.html @@ -18,13 +18,11 @@
{{ message.ts|format_ts("%H:%M") }}
{% if message.msgtype == "m.text" %} - {% if message.format == "org.matrix.custom.html" %} - {{ message.body_text_html }} - {% else %} - {{ message.body_text_plain }} - {% endif %} + {{ message.body_text_html }} {% elif message.msgtype == "m.image" %} + {% elif message.msgtype == "m.file" %} + {{ message.body_text_plain }} {% endif %}
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index e6554dc7f..60a4878a3 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -222,8 +222,9 @@ class Mailer(object): self.add_text_message_vars(ret, event) elif event.content["msgtype"] == "m.image": self.add_image_message_vars(ret, event) - else: - return None + + if "body" in event.content: + ret["body_text_plain"] = event.content["body"] return ret From cc0874cf71b2d2eef713155900b4ac5384ba1a1a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 17:00:40 +0100 Subject: [PATCH 027/134] Put back real delay before mailing --- synapse/push/emailpusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 6ae16e986..dcbee4c3f 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) # The amount of time we always wait before ever emailing about a notification # (to give the user a chance to respond to other push or notice the window) -DELAY_BEFORE_MAIL_MS = 2000#2 * 60 * 1000 +DELAY_BEFORE_MAIL_MS = 2 * 60 * 1000 THROTTLE_START_MS = 2 * 60 * 1000 THROTTLE_MAX_MS = (2 * 60 * 1000) * (2 ** 11) # ~3 days From e800ee2f636a6aa63a9ac35388b9f982b01cd1c0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 17:28:27 +0100 Subject: [PATCH 028/134] May as well always include room link --- synapse/push/mailer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 60a4878a3..c53ae9a54 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -157,12 +157,11 @@ class Mailer(object): "title": calculate_room_name(room_state, user_id), "hash": string_ordinal_total(room_id), # See sender avatar hash "notifs": [], - "invite": is_invite + "invite": is_invite, + "link": self.make_room_link(room_id), } - if is_invite: - room_vars["link"] = self.make_room_link(room_id) - else: + if not is_invite: for n in notifs: vars = yield self.get_notif_vars( n, user_id, notif_events[n['event_id']], room_state From 36bb5c23831444f35c018923be8d8c0b0562b331 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 17:28:48 +0100 Subject: [PATCH 029/134] Fix notification link --- res/templates/notif.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res/templates/notif.html b/res/templates/notif.html index 70f565535..3112df970 100644 --- a/res/templates/notif.html +++ b/res/templates/notif.html @@ -29,6 +29,6 @@ {% endfor %}
- View in Vector + View
From 46b5547a4227ec29e0b71e2138a2ebade39724ca Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 17:29:39 +0100 Subject: [PATCH 030/134] Some basic css to make it halfway legible --- res/templates/mail.css | 35 +++++++++++++++++++++++++++++++++++ res/templates/notif_mail.html | 25 ++++++++++++++++--------- res/templates/room.html | 14 +++++++------- 3 files changed, 58 insertions(+), 16 deletions(-) create mode 100644 res/templates/mail.css diff --git a/res/templates/mail.css b/res/templates/mail.css new file mode 100644 index 000000000..be63b8c33 --- /dev/null +++ b/res/templates/mail.css @@ -0,0 +1,35 @@ +body { + font-family: "Open Sans", Helvetica, Arial, sans-serif; + font-size: 12pt; + margin: 1em; +} + +#page { + margin: auto; + width: 640px; + padding: 1em; + border: 1px solid #76cfa6; + border-radius: 5px; +} + +.room_avatar { + width: 48px; + height: 48px; + float: left; +} + +.room_content { + clear: left; +} + +.historical { + color: #888; +} + +.sender_name { + display: inline; +} + +.message_time { + display: inline; +} diff --git a/res/templates/notif_mail.html b/res/templates/notif_mail.html index fbfb0a767..cc3573e65 100644 --- a/res/templates/notif_mail.html +++ b/res/templates/notif_mail.html @@ -1,15 +1,22 @@ + + + -
Hi {{ user_display_name }},
-
{{ summary_text }}
-
- {% for room in rooms %} - {% include 'room.html' with context %} - {% endfor %} -
-
- Unsubscribe +
+
Hi {{ user_display_name }},
+
{{ summary_text }}
+
+ {% for room in rooms %} + {% include 'room.html' with context %} + {% endfor %} +
+
+ Unsubscribe +
diff --git a/res/templates/room.html b/res/templates/room.html index 6c68ee1fd..3c0a4607b 100644 --- a/res/templates/room.html +++ b/res/templates/room.html @@ -1,5 +1,4 @@
-

{{ room.title }}

{% if room.avatar_url %} @@ -13,13 +12,14 @@ {% endif %} {% endif %}
- {% if room.invite %} - Join the conversation. - {% else %} -
+

{{ room.title }}

+
+ {% if room.invite %} + Join the conversation. + {% else %} {% for notif in room.notifs %} {% include 'notif.html' with context %} {% endfor %} -
- {% endif %} + {% endif %} +
From 69e519052bacdcadaeb825346c8e101aea932769 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Apr 2016 17:34:32 +0100 Subject: [PATCH 031/134] Remove vector specific style --- res/templates/mail.css | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/res/templates/mail.css b/res/templates/mail.css index be63b8c33..61953ba51 100644 --- a/res/templates/mail.css +++ b/res/templates/mail.css @@ -1,17 +1,3 @@ -body { - font-family: "Open Sans", Helvetica, Arial, sans-serif; - font-size: 12pt; - margin: 1em; -} - -#page { - margin: auto; - width: 640px; - padding: 1em; - border: 1px solid #76cfa6; - border-radius: 5px; -} - .room_avatar { width: 48px; height: 48px; From ec9cbe847d0d489c602309312fb947daa0e6a129 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 10:07:30 +0100 Subject: [PATCH 032/134] pep8 newline --- synapse/util/presentable_names.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/presentable_names.py b/synapse/util/presentable_names.py index f80a7fe58..13def9177 100644 --- a/synapse/util/presentable_names.py +++ b/synapse/util/presentable_names.py @@ -142,4 +142,4 @@ def _state_as_two_level_dict(state): def _looks_like_an_alias(string): - return ALIAS_RE.match(string) is not None \ No newline at end of file + return ALIAS_RE.match(string) is not None From b2c04da8dc98ca09620dc207c95f68b2e8a52e62 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 11:43:57 +0100 Subject: [PATCH 033/134] Add an email pusher for new users If they registered with an email address and email notifs are enabled on the HS --- synapse/push/pusherpool.py | 1 + synapse/rest/client/v2_alpha/register.py | 26 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 6ef48d63f..7fef2fb6f 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -50,6 +50,7 @@ class PusherPool: # recreated, added and started: this means we have only one # code path adding pushers. pusher.create_pusher(self.hs, { + "id": None, "user_name": user_id, "kind": kind, "app_id": app_id, diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index ff8f69ddb..883b1c129 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -48,6 +48,7 @@ class RegisterRestServlet(RestServlet): super(RegisterRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.store = hs.get_datastore() self.auth_handler = hs.get_handlers().auth_handler self.registration_handler = hs.get_handlers().registration_handler self.identity_handler = hs.get_handlers().identity_handler @@ -214,6 +215,31 @@ class RegisterRestServlet(RestServlet): threepid['validated_at'], ) + # And we add an email pusher for them by default, but only + # if email notifications are enabled (so people don't start + # getting mail spam where they weren't before if email + # notifs are set up on a home server) + if self.hs.config.email_enable_notifs: + # Pull the ID of the access token back out of the db + # It would really make more sense for this to be passed + # up when the access token is saved, but that's quite an + # invasive change I'd rather do separately. + user_tuple = yield self.store.get_user_by_access_token( + token + ) + + yield self.hs.get_pusherpool().add_pusher( + user_id=user_id, + access_token=user_tuple["token_id"], + kind="email", + app_id="m.email", + app_display_name="Email Notifications", + device_display_name=threepid["address"], + pushkey=threepid["address"], + lang=None, # We don't know a user's language here + data={}, + ) + if 'bind_email' in params and params['bind_email']: logger.info("bind_email specified: binding") From 62b51b84520243d25b82a303ff44ff2f329c8d82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Apr 2016 12:00:51 +0100 Subject: [PATCH 034/134] Fix typo in event_auth servlet path --- synapse/federation/transport/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index d65a7893d..3e552b6c4 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -323,7 +323,7 @@ class FederationSendLeaveServlet(BaseFederationServlet): class FederationEventAuthServlet(BaseFederationServlet): - PATH = "/event_auth(?P[^/]*)/(?P[^/]*)" + PATH = "/event_auth/(?P[^/]*)/(?P[^/]*)" def on_GET(self, origin, content, query, context, event_id): return self.handler.on_event_auth(origin, context, event_id) From 40d40e470d9ff047f4255a93e40af8a9870b43cc Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 13:56:21 +0100 Subject: [PATCH 035/134] Send mail notifs with a plaintext part too --- res/templates/notif.txt | 12 ++++++++++++ res/templates/notif_mail.txt | 10 ++++++++++ res/templates/room.txt | 6 ++++++ synapse/config/emailconfig.py | 2 ++ synapse/push/mailer.py | 27 ++++++++++++++++++++------- 5 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 res/templates/notif.txt create mode 100644 res/templates/notif_mail.txt create mode 100644 res/templates/room.txt diff --git a/res/templates/notif.txt b/res/templates/notif.txt new file mode 100644 index 000000000..b515f394c --- /dev/null +++ b/res/templates/notif.txt @@ -0,0 +1,12 @@ +{% for message in notif.messages %} +{{ message.sender_name }} ({{ message.ts|format_ts("%H:%M") }}) +{% if message.msgtype == "m.text" %} +{{ message.body_text_plain }} +{% elif message.msgtype == "m.image" %} +{{ message.body_text_plain }} +{% elif message.msgtype == "m.file" %} +{{ message.body_text_plain }} +{% endif %} +{% endfor %} + +View at {{ notif.link }} diff --git a/res/templates/notif_mail.txt b/res/templates/notif_mail.txt new file mode 100644 index 000000000..24843042a --- /dev/null +++ b/res/templates/notif_mail.txt @@ -0,0 +1,10 @@ +Hi {{ user_display_name }}, + +{{ summary_text }} + +{% for room in rooms %} +{% include 'room.txt' with context %} +{% endfor %} + +You can disable these notifications at {{ unsubscribe_link }} + diff --git a/res/templates/room.txt b/res/templates/room.txt new file mode 100644 index 000000000..999d0ae60 --- /dev/null +++ b/res/templates/room.txt @@ -0,0 +1,6 @@ +{{ room.title }} +You've been invited, join at {{ room.link }} + +{% for notif in room.notifs %} +{% include 'notif.txt' with context %} +{% endfor %} diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 06b076e3f..b7be67f17 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -38,6 +38,7 @@ class EmailConfig(Config): "notif_from", "template_dir", "notif_template_html", + "notif_template_text", ] missing = [] @@ -61,6 +62,7 @@ class EmailConfig(Config): self.email_notif_from = email_config["notif_from"] self.email_template_dir = email_config["template_dir"] self.email_notif_template_html = email_config["notif_template_html"] + self.email_notif_template_text = email_config["notif_template_text"] # make sure it's valid parsed = email.utils.parseaddr(self.email_notif_from) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index c53ae9a54..4fd89b3e9 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -19,6 +19,7 @@ from twisted.mail.smtp import sendmail import email.utils import email.mime.multipart from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart from synapse.util.async import concurrently_execute from synapse.util.presentable_names import ( @@ -74,7 +75,12 @@ class Mailer(object): env = jinja2.Environment(loader=loader) env.filters["format_ts"] = format_ts_filter env.filters["mxc_to_http"] = self.mxc_to_http_filter - self.notif_template = env.get_template(self.hs.config.email_notif_template_html) + self.notif_template_html = env.get_template( + self.hs.config.email_notif_template_html + ) + self.notif_template_text = env.get_template( + self.hs.config.email_notif_template_text + ) @defer.inlineCallbacks def send_notification_mail(self, user_id, email_address, push_actions): @@ -135,16 +141,23 @@ class Mailer(object): "rooms": rooms, } - plainText = self.notif_template.render(**template_vars) + html_text = self.notif_template_html.render(**template_vars) + html_part = MIMEText(html_text, "html", "utf8") + + plain_text = self.notif_template_text.render(**template_vars) + text_part = MIMEText(plain_text, "plain", "utf8") + + multipart_msg = MIMEMultipart('alternative') + multipart_msg['Subject'] = "New Matrix Notifications" + multipart_msg['From'] = self.hs.config.email_notif_from + multipart_msg['To'] = email_address + multipart_msg.attach(text_part) + multipart_msg.attach(html_part) - text_part = MIMEText(plainText, "html", "utf8") - text_part['Subject'] = "New Matrix Notifications" - text_part['From'] = self.hs.config.email_notif_from - text_part['To'] = email_address yield sendmail( self.hs.config.email_smtp_host, - raw_from, raw_to, text_part.as_string(), + raw_from, raw_to, multipart_msg.as_string(), port=self.hs.config.email_smtp_port ) From 18ce88bd2dc15cad08ca6bd3c835ca14fb3478d9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:24:25 +0100 Subject: [PATCH 036/134] Correct default template and add text template --- synapse/config/emailconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index b7be67f17..6db01fa14 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -82,5 +82,6 @@ class EmailConfig(Config): # smtp_port: 25 # notif_from: Your Friendly Matrix Home Server # template_dir: res/templates - # notif_template_html: notif.html + # notif_template_html: notif_email.html + # notif_template_text: notif_email.txt """ From 6c8957be7f2b27b753d75df2d67989e09a6d6286 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:25:28 +0100 Subject: [PATCH 037/134] Remove redundant docstring --- synapse/config/emailconfig.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 6db01fa14..761ac3311 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -20,10 +20,6 @@ from ._base import Config class EmailConfig(Config): - """ - Email Configuration - """ - def read_config(self, config): self.email_enable_notifs = False From 50484559658d06203565df5af73bb20e279faa1c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:27:40 +0100 Subject: [PATCH 038/134] Nicer get() shorthand --- synapse/config/emailconfig.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 761ac3311..4318c6f13 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -23,9 +23,8 @@ class EmailConfig(Config): def read_config(self, config): self.email_enable_notifs = False - email_config = config.get("email", None) - if email_config: - self.email_enable_notifs = email_config.get("enable_notifs", True) + email_config = config.get("email", {}) + self.email_enable_notifs = email_config.get("enable_notifs", True) if self.email_enable_notifs: required = [ From 4b0c3a327007b86346745fc96dafc4fd8ada2f19 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:30:15 +0100 Subject: [PATCH 039/134] Correct public_baseurl default --- synapse/config/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 04b922190..0b5f462e4 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -148,8 +148,8 @@ class ServerConfig(Config): # Whether to serve a web client from the HTTP/HTTPS root resource. web_client: True - # The server's public-facing base URL - # https://example.com:8448/ + # The public-facing base URL for the client API (not including _matrix/...) + # public_baseurl: https://example.com:8448/ # Set the soft limit on the number of file descriptors synapse can use # Zero is used to indicate synapse should set the soft limit to the From 4364ea1272a2201fc434f4cf6f0c3d13f5cbc37e Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:31:27 +0100 Subject: [PATCH 040/134] Stop processing notifs once we've sent a mail --- synapse/push/emailpusher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index dcbee4c3f..68f59a3fa 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -159,6 +159,7 @@ class EmailPusher(object): yield self.sent_notif_update_throttle( push_action['room_id'], push_action ) + break else: if soonest_due_at is None or should_notify_at < soonest_due_at: soonest_due_at = should_notify_at From 3facde2536f392477d59c2e03be2f2fb07e383dc Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:36:45 +0100 Subject: [PATCH 041/134] Remove rather pointless get function --- synapse/push/emailpusher.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 68f59a3fa..62e1e4032 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -139,9 +139,7 @@ class EmailPusher(object): notif_ready_at = received_at + DELAY_BEFORE_MAIL_MS room_ready_at = self.room_ready_to_notify_at( - push_action['room_id'], self.get_room_last_notif_ts( - last_notifs, push_action['room_id'] - ) + push_action['room_id'], last_notifs.get(push_action['room_id'], 0) ) should_notify_at = max(notif_ready_at, room_ready_at) @@ -184,12 +182,6 @@ class EmailPusher(object): def seconds_until(self, ts_msec): return (ts_msec - self.clock.time_msec()) / 1000 - def get_room_last_notif_ts(self, last_notif_by_room, room_id): - if room_id in last_notif_by_room: - return last_notif_by_room[room_id] - else: - return 0 - def get_room_throttle_ms(self, room_id): if room_id in self.throttle_params: return self.throttle_params[room_id]["throttle_ms"] From 311b5ce051cf979f30f6aa53051b1349326b2235 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:37:30 +0100 Subject: [PATCH 042/134] pep8 --- synapse/push/mailer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 4fd89b3e9..0ab5168e2 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -154,7 +154,6 @@ class Mailer(object): multipart_msg.attach(text_part) multipart_msg.attach(html_part) - yield sendmail( self.hs.config.email_smtp_host, raw_from, raw_to, multipart_msg.as_string(), From ff5e5423e587231232dd36dce5ab62c4a9a386ee Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:43:48 +0100 Subject: [PATCH 043/134] Include res in the manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 211bde2fc..dfb7c9d28 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,6 +11,7 @@ recursive-include synapse/storage/schema *.sql recursive-include synapse/storage/schema *.py recursive-include docs * +recursive-include res * recursive-include scripts * recursive-include scripts-dev * recursive-include tests *.py From 765f2b8446b99fd9d31311ec0f965e6e57fc9c44 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 14:46:18 +0100 Subject: [PATCH 044/134] Default enable email notifs to False --- synapse/config/emailconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 4318c6f13..1fa2bab3a 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -24,7 +24,7 @@ class EmailConfig(Config): self.email_enable_notifs = False email_config = config.get("email", {}) - self.email_enable_notifs = email_config.get("enable_notifs", True) + self.email_enable_notifs = email_config.get("enable_notifs", False) if self.email_enable_notifs: required = [ From d3da5294e8acaba34c59bdfc62f7010363b17a18 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:04:40 +0100 Subject: [PATCH 045/134] Use named parameter format --- synapse/push/mailer.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 0ab5168e2..b193bb84b 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -35,13 +35,13 @@ import time import urllib -MESSAGE_FROM_PERSON_IN_ROOM = "You have a message from %s in the %s room" -MESSAGE_FROM_PERSON = "You have a message from %s" -MESSAGES_FROM_PERSON = "You have messages from %s" -MESSAGES_IN_ROOM = "There are some messages for you in the %s room" +MESSAGE_FROM_PERSON_IN_ROOM = "You have a message from %(person)s in the %s room" +MESSAGE_FROM_PERSON = "You have a message from %(person)s" +MESSAGES_FROM_PERSON = "You have messages from %(person)s" +MESSAGES_IN_ROOM = "There are some messages for you in the %(room)s room" MESSAGES_IN_ROOMS = "Here are some messages you may have missed" -INVITE_FROM_PERSON_TO_ROOM = "%s has invited you to join the %s room" -INVITE_FROM_PERSON = "%s has invited you to chat" +INVITE_FROM_PERSON_TO_ROOM = "%(person)s has invited you to join the %(room)s room" +INVITE_FROM_PERSON = "%(person)s has invited you to chat" CONTEXT_BEFORE = 1 @@ -278,9 +278,11 @@ class Mailer(object): inviter_name = name_from_member_event(inviter_member_event) if room_name is None: - return INVITE_FROM_PERSON % (inviter_name,) + return INVITE_FROM_PERSON % {"person": inviter_name} else: - return INVITE_FROM_PERSON_TO_ROOM % (inviter_name, room_name) + return INVITE_FROM_PERSON_TO_ROOM % { + "person": inviter_name, "room": room_name + } sender_name = None if len(notifs_by_room[room_id]) == 1: @@ -291,14 +293,20 @@ class Mailer(object): sender_name = name_from_member_event(state_event) if sender_name is not None and room_name is not None: - return MESSAGE_FROM_PERSON_IN_ROOM % (sender_name, room_name) + return MESSAGE_FROM_PERSON_IN_ROOM % { + "person": sender_name, "room": room_name + } elif sender_name is not None: - return MESSAGE_FROM_PERSON % (sender_name,) + return MESSAGE_FROM_PERSON % { + "person": sender_name + } else: # There's more than one notification for this room, so just # say there are several if room_name is not None: - return MESSAGES_IN_ROOM % (room_name,) + return MESSAGES_IN_ROOM % { + "room": room_name + } else: # If the room doesn't have a name, say who the messages # are from explicitly to avoid, "messages in the Bob room" @@ -307,12 +315,12 @@ class Mailer(object): for n in notifs_by_room[room_id] ])) - return MESSAGES_FROM_PERSON % ( - descriptor_from_member_events([ + return MESSAGES_FROM_PERSON % { + "person": descriptor_from_member_events([ state_by_room[room_id][("m.room.member", s)] for s in sender_ids ]) - ) + } else: # Stuff's happened in multiple different rooms return MESSAGES_IN_ROOMS From 29c8cf8db81b89b1571b0a042d2bfed3ed278890 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:09:28 +0100 Subject: [PATCH 046/134] Avoid `vars` builtin --- synapse/push/mailer.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index b193bb84b..95f872ccf 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -125,10 +125,10 @@ class Mailer(object): rooms = [] for r in rooms_in_order: - vars = yield self.get_room_vars( + roomvars = yield self.get_room_vars( r, user_id, notifs_by_room[r], notif_events, state_by_room[r] ) - rooms.append(vars) + rooms.append(roomvars) summary_text = self.make_summary_text( notifs_by_room, state_by_room, notif_events, user_id @@ -175,10 +175,10 @@ class Mailer(object): if not is_invite: for n in notifs: - vars = yield self.get_notif_vars( + notifvars = yield self.get_notif_vars( n, user_id, notif_events[n['event_id']], room_state ) - room_vars['notifs'].append(vars) + room_vars['notifs'].append(notifvars) defer.returnValue(room_vars) @@ -202,9 +202,9 @@ class Mailer(object): the_events.append(notif_event) for event in the_events: - vars = self.get_message_vars(notif, event, room_state) - if vars is not None: - ret['messages'].append(vars) + messagevars = self.get_message_vars(notif, event, room_state) + if messagevars is not None: + ret['messages'].append(messagevars) defer.returnValue(ret) @@ -239,24 +239,24 @@ class Mailer(object): return ret - def add_text_message_vars(self, vars, event): + def add_text_message_vars(self, messagevars, event): if "format" in event.content: msgformat = event.content["format"] else: msgformat = None - vars["format"] = msgformat + messagevars["format"] = msgformat if msgformat == "org.matrix.custom.html": - vars["body_text_html"] = safe_markup(event.content["formatted_body"]) + messagevars["body_text_html"] = safe_markup(event.content["formatted_body"]) else: - vars["body_text_html"] = safe_text(event.content["body"]) + messagevars["body_text_html"] = safe_text(event.content["body"]) - return vars + return messagevars - def add_image_message_vars(self, vars, event): - vars["image_url"] = event.content["url"] + def add_image_message_vars(self, messagevars, event): + messagevars["image_url"] = event.content["url"] - return vars + return messagevars def make_summary_text(self, notifs_by_room, state_by_room, notif_events, user_id): if len(notifs_by_room) == 1: From e7a76b512302ee77e3afc6c17dfb2efc3b58b156 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:10:45 +0100 Subject: [PATCH 047/134] Use the constant --- synapse/push/mailer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 95f872ccf..1e6654e2b 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -27,6 +27,7 @@ from synapse.util.presentable_names import ( ) from synapse.types import UserID from synapse.api.errors import StoreError +from synapse.api.constants import EventTypes import jinja2 import bleach @@ -209,7 +210,7 @@ class Mailer(object): defer.returnValue(ret) def get_message_vars(self, notif, event, room_state): - if event.type != "m.room.message": + if event.type != EventTypes.Message: return None sender_state_event = room_state[("m.room.member", event.sender)] From 83618d719a437ac40f1d004689da4570eb436e9c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:13:52 +0100 Subject: [PATCH 048/134] Try imports in config --- synapse/config/emailconfig.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 1fa2bab3a..e4d86255f 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -27,6 +27,13 @@ class EmailConfig(Config): self.email_enable_notifs = email_config.get("enable_notifs", False) if self.email_enable_notifs: + # make sure we can import the required deps + import jinja2 + import bleach + # prevent unused warnings + jinja2 + bleach + required = [ "smtp_host", "smtp_port", From 50ad8005e4b3e8054f990f34d2d7735b09dc8c19 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:16:15 +0100 Subject: [PATCH 049/134] Put spaces at start of line --- synapse/storage/event_push_actions.py | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index f2af8bdb3..5b9a4ca60 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -155,13 +155,13 @@ class EventPushActionsStore(SQLBaseStore): def get_no_receipt(txn): sql = ( - "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, " - "e.received_ts " - "FROM event_push_actions AS ep " - "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id " - "WHERE ep.room_id not in (" + "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," + " e.received_ts" + " FROM event_push_actions AS ep" + " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id" + " WHERE ep.room_id not in (" " SELECT room_id FROM events NATURAL JOIN receipts_linearized" - " WHERE receipt_type = 'm.read' AND user_id = ? " + " WHERE receipt_type = 'm.read' AND user_id = ?" " GROUP BY room_id" ") AND ep.user_id = ? AND ep.stream_ordering > ?" ) @@ -190,12 +190,12 @@ class EventPushActionsStore(SQLBaseStore): def get_time_of_last_push_action_before(self, stream_ordering): def f(txn): sql = ( - "SELECT e.received_ts " - "FROM event_push_actions AS ep " - "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id " - "WHERE ep.stream_ordering > ? " - "ORDER BY ep.stream_ordering ASC " - "LIMIT 1" + "SELECT e.received_ts" + " FROM event_push_actions AS ep" + " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id" + " WHERE ep.stream_ordering > ?" + " ORDER BY ep.stream_ordering ASC" + " LIMIT 1" ) txn.execute(sql, (stream_ordering,)) return txn.fetchone() @@ -220,10 +220,10 @@ class EventPushActionsStore(SQLBaseStore): """ def f(txn): txn.execute( - "SELECT ep.room_id, MAX(e.received_ts) " - "FROM event_push_actions AS ep " - "JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id " - "GROUP BY ep.room_id" + "SELECT ep.room_id, MAX(e.received_ts)" + " FROM event_push_actions AS ep" + " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id" + " GROUP BY ep.room_id" ) return txn.fetchall() result = yield self.runInteraction( From 60f44c098d14754d0159c2ccc2c888e7ca970427 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:17:10 +0100 Subject: [PATCH 050/134] Remove unnecessary if --- synapse/storage/event_push_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 5b9a4ca60..c6625a7b0 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -200,7 +200,7 @@ class EventPushActionsStore(SQLBaseStore): txn.execute(sql, (stream_ordering,)) return txn.fetchone() result = yield self.runInteraction("get_time_of_last_push_action_before", f) - defer.returnValue(result[0] if result is not None else None) + defer.returnValue(result[0] if result else None) @defer.inlineCallbacks def get_latest_push_action_stream_ordering(self): From 8f99cd5996a11bb1cc50dce35542b2591426fdbc Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:27:03 +0100 Subject: [PATCH 051/134] Oops, actually specify the user id --- synapse/storage/event_push_actions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index c6625a7b0..974ceb338 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -223,7 +223,9 @@ class EventPushActionsStore(SQLBaseStore): "SELECT ep.room_id, MAX(e.received_ts)" " FROM event_push_actions AS ep" " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id" - " GROUP BY ep.room_id" + " WHERE ep.user_id = ?" + " GROUP BY ep.room_id", + (user_id,) ) return txn.fetchall() result = yield self.runInteraction( From b0a1036d93eaec50839c95bf0dc621826342ea62 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:28:56 +0100 Subject: [PATCH 052/134] Use explicit join --- synapse/storage/event_push_actions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 974ceb338..b710101b1 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -127,9 +127,9 @@ class EventPushActionsStore(SQLBaseStore): " FROM events" " NATURAL JOIN receipts_linearized WHERE receipt_type = 'm.read'" " GROUP BY room_id, user_id" - ") AS rl " - "NATURAL JOIN events e " - "WHERE" + ") AS rl" + " INNER JOIN events AS e USING (room_id, event_id)" + " WHERE" " ep.room_id = rl.room_id" " AND (" " ep.topological_ordering > rl.topological_ordering" From c7c75e87dc6b619d07d224bd4c6464313cc96840 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 19:47:35 +0100 Subject: [PATCH 053/134] Docstring --- synapse/util/presentable_names.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/util/presentable_names.py b/synapse/util/presentable_names.py index 13def9177..3efa8a820 100644 --- a/synapse/util/presentable_names.py +++ b/synapse/util/presentable_names.py @@ -23,6 +23,20 @@ ALL_ALONE = "Empty Room" def calculate_room_name(room_state, user_id, fallback_to_members=True): + """ + Works out a user-facing name for the given room as per Matrix + spec recommendations. + Does not yet support internationalisation. + Args: + room_state: Dictionary of the room's state + user_id: The ID of the user to whom the room name is being presented + fallback_to_members: If False, return None instead of generating a name + based on the room's members if the room has no + title or aliases. + + Returns: + (string or None) A human readable name for the room. + """ # does it have a name? if ("m.room.name", "") in room_state: m_room_name = room_state[("m.room.name", "")] From 6b9b6a91691d543fe420e82366f641beb760cf11 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 20:02:52 +0100 Subject: [PATCH 054/134] Remove unused arg --- synapse/push/emailpusher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 62e1e4032..8b105b85c 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -139,7 +139,7 @@ class EmailPusher(object): notif_ready_at = received_at + DELAY_BEFORE_MAIL_MS room_ready_at = self.room_ready_to_notify_at( - push_action['room_id'], last_notifs.get(push_action['room_id'], 0) + push_action['room_id'] ) should_notify_at = max(notif_ready_at, room_ready_at) @@ -194,7 +194,7 @@ class EmailPusher(object): else: return 0 - def room_ready_to_notify_at(self, room_id, last_notif_time): + def room_ready_to_notify_at(self, room_id): """ Determines whether throttling should prevent us from sending an email for the given room From 35b7b8e4bccd8cfbe44b2ffee97cdee0a48701ba Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 20:10:34 +0100 Subject: [PATCH 055/134] Remove unused function --- synapse/push/emailpusher.py | 4 ---- synapse/storage/event_push_actions.py | 22 ---------------------- 2 files changed, 26 deletions(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 8b105b85c..c10deded0 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -122,10 +122,6 @@ class EmailPusher(object): up logging, measures and guards against multiple instances of it being run. """ - last_notifs = yield self.store.get_time_of_latest_push_action_by_room_for_user( - self.user_id - ) - unprocessed = yield self.store.get_unread_push_actions_for_user_in_range( self.user_id, self.last_stream_ordering, self.max_stream_ordering ) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index b710101b1..85290d2a9 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -212,28 +212,6 @@ class EventPushActionsStore(SQLBaseStore): ) defer.returnValue(result[0] or 0) - @defer.inlineCallbacks - def get_time_of_latest_push_action_by_room_for_user(self, user_id): - """ - Returns only the received_ts of the last notification in each of the - user's rooms, in a dict by room_id - """ - def f(txn): - txn.execute( - "SELECT ep.room_id, MAX(e.received_ts)" - " FROM event_push_actions AS ep" - " JOIN events e ON ep.room_id = e.room_id AND ep.event_id = e.event_id" - " WHERE ep.user_id = ?" - " GROUP BY ep.room_id", - (user_id,) - ) - return txn.fetchall() - result = yield self.runInteraction( - "get_time_of_latest_push_action_by_room_for_user", f - ) - - defer.returnValue({row[0]: row[1] for row in result}) - def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id): # Sad that we have to blow away the cache for the whole room here txn.call_after( From b084e4d963f2df377dfe98b6d9645fa9183a368b Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 29 Apr 2016 20:14:55 +0100 Subject: [PATCH 056/134] Add constant for throttle multiplier --- synapse/push/emailpusher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index c10deded0..3a13c7485 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -30,6 +30,7 @@ DELAY_BEFORE_MAIL_MS = 2 * 60 * 1000 THROTTLE_START_MS = 2 * 60 * 1000 THROTTLE_MAX_MS = (2 * 60 * 1000) * (2 ** 11) # ~3 days +THROTTLE_MULTIPLIER = 2 # If no event triggers a notification for this long after the previous, # the throttle is released. @@ -231,7 +232,7 @@ class EmailPusher(object): new_throttle_ms = THROTTLE_START_MS else: new_throttle_ms = min( - current_throttle_ms * 2, + current_throttle_ms * THROTTLE_MULTIPLIER, THROTTLE_MAX_MS ) self.throttle_params[room_id] = { From 183f23f10d32ece4503f18e70296c88a1625ecff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 May 2016 14:22:33 +0100 Subject: [PATCH 057/134] Delete old pushers --- synapse/storage/event_push_actions.py | 12 ++++++++++++ synapse/storage/receipts.py | 26 +++++++++++++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 86a98b6f1..312c0071f 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -201,6 +201,18 @@ class EventPushActionsStore(SQLBaseStore): (room_id, event_id) ) + def _remove_push_actions_before_txn(self, txn, room_id, user_id, + topological_ordering): + txn.call_after( + self.get_unread_event_push_actions_by_room_for_user.invalidate_many, + (room_id, user_id, ) + ) + txn.execute( + "DELETE FROM event_push_actions" + " WHERE room_id = ? AND user_id = ? AND topological_ordering < ?", + (room_id, user_id, topological_ordering,) + ) + def _action_has_highlight(actions): for action in actions: diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 935fc503d..cd1b611a0 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -244,6 +244,15 @@ class ReceiptsStore(SQLBaseStore): (user_id, room_id, receipt_type) ) + res = self._simple_select_one_txn( + txn, + table="events", + retcols=["topological_ordering", "stream_ordering"], + keyvalues={"event_id": event_id}, + ) + topological_ordering = int(res["topological_ordering"]) + stream_ordering = int(res["stream_ordering"]) + # We don't want to clobber receipts for more recent events, so we # have to compare orderings of existing receipts sql = ( @@ -256,15 +265,6 @@ class ReceiptsStore(SQLBaseStore): results = txn.fetchall() if results: - res = self._simple_select_one_txn( - txn, - table="events", - retcols=["topological_ordering", "stream_ordering"], - keyvalues={"event_id": event_id}, - ) - topological_ordering = int(res["topological_ordering"]) - stream_ordering = int(res["stream_ordering"]) - for to, so, _ in results: if int(to) > topological_ordering: return False @@ -294,6 +294,14 @@ class ReceiptsStore(SQLBaseStore): } ) + if receipt_type == "m.read": + self._remove_push_actions_before_txn( + txn, + room_id=room_id, + user_id=user_id, + topological_ordering=topological_ordering, + ) + return True @defer.inlineCallbacks From a438a6d2bca7fae3a4f1fa38b72b821c742cdc8a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 May 2016 16:01:24 +0100 Subject: [PATCH 058/134] Implement basic ignore user --- synapse/api/filtering.py | 8 +++++--- synapse/handlers/_base.py | 26 +++++++++++++++++++++++--- synapse/handlers/sync.py | 22 +++++++++++++++++++--- synapse/storage/account_data.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index cd699ef27..4f5a4281f 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -15,6 +15,8 @@ from synapse.api.errors import SynapseError from synapse.types import UserID, RoomID +from twisted.internet import defer + import ujson as json @@ -24,10 +26,10 @@ class Filtering(object): super(Filtering, self).__init__() self.store = hs.get_datastore() + @defer.inlineCallbacks def get_user_filter(self, user_localpart, filter_id): - result = self.store.get_user_filter(user_localpart, filter_id) - result.addCallback(FilterCollection) - return result + result = yield self.store.get_user_filter(user_localpart, filter_id) + defer.returnValue(FilterCollection(result)) def add_user_filter(self, user_localpart, user_filter): self.check_valid_filter(user_filter) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 13a675b20..0912274e1 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -84,7 +84,7 @@ class BaseHandler(object): events ([synapse.events.EventBase]): list of events to filter """ forgotten = yield defer.gatherResults([ - self.store.who_forgot_in_room( + preserve_fn(self.store.who_forgot_in_room)( room_id, ) for room_id in frozenset(e.room_id for e in events) @@ -95,13 +95,33 @@ class BaseHandler(object): row["event_id"] for rows in forgotten for row in rows ) - def allowed(event, user_id, is_peeking): + # Maps user_id -> account data content + ignore_dict_content = yield defer.gatherResults([ + preserve_fn(self.store.get_global_account_data_by_type_for_user)( + user_id, "m.ignored_user_list" + ).addCallback(lambda d, u: (u, d), user_id) + for user_id, is_peeking in user_tuples + ]).addCallback(dict) + + # FIXME: This will explode if people upload something incorrect. + ignore_dict = { + user_id: frozenset( + content.get("ignored_users", {}).keys() if content else [] + ) + for user_id, content in ignore_dict_content.items() + } + + def allowed(event, user_id, is_peeking, ignore_list): """ Args: event (synapse.events.EventBase): event to check user_id (str) is_peeking (bool) + ignore_list (list): list of users to ignore """ + if not event.is_state() and event.sender in ignore_list: + return False + state = event_id_to_state[event.event_id] # get the room_visibility at the time of the event. @@ -186,7 +206,7 @@ class BaseHandler(object): user_id: [ event for event in events - if allowed(event, user_id, is_peeking) + if allowed(event, user_id, is_peeking, ignore_dict.get(user_id, [])) ] for user_id, is_peeking in user_tuples }) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 231140b65..27c3c1b52 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -247,6 +247,10 @@ class SyncHandler(BaseHandler): sync_config.user.to_string() ) + ignored_users = account_data.get( + "m.ignored_user_list", {} + ).get("ignored_users", {}).keys() + joined = [] invited = [] archived = [] @@ -267,6 +271,8 @@ class SyncHandler(BaseHandler): ) joined.append(room_result) elif event.membership == Membership.INVITE: + if event.sender in ignored_users: + return invite = yield self.store.get_event(event.event_id) invited.append(InvitedSyncResult( room_id=event.room_id, @@ -515,6 +521,15 @@ class SyncHandler(BaseHandler): sync_config.user ) + ignored_account_data = yield self.store.get_global_account_data_by_type_for_user( + user_id, "m.ignored_user_list" + ) + + if ignored_account_data: + ignored_users = ignored_account_data.get("ignored_users", {}).keys() + else: + ignored_users = frozenset() + # Get a list of membership change events that have happened. rooms_changed = yield self.store.get_membership_changes_for_user( user_id, since_token.room_key, now_token.room_key @@ -549,9 +564,10 @@ class SyncHandler(BaseHandler): # Only bother if we're still currently invited should_invite = non_joins[-1].membership == Membership.INVITE if should_invite: - room_sync = InvitedSyncResult(room_id, invite=non_joins[-1]) - if room_sync: - invited.append(room_sync) + if event.sender not in ignored_users: + room_sync = InvitedSyncResult(room_id, invite=non_joins[-1]) + if room_sync: + invited.append(room_sync) # Always include leave/ban events. Just take the last one. # TODO: How do we handle ban -> leave in same batch? diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 7a7fbf1e5..cc0b92bc8 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -16,6 +16,8 @@ from ._base import SQLBaseStore from twisted.internet import defer +from synapse.util.caches.descriptors import cached, cachedInlineCallbacks + import ujson as json import logging @@ -24,6 +26,7 @@ logger = logging.getLogger(__name__) class AccountDataStore(SQLBaseStore): + @cached() def get_account_data_for_user(self, user_id): """Get all the client account_data for a user. @@ -60,6 +63,28 @@ class AccountDataStore(SQLBaseStore): "get_account_data_for_user", get_account_data_for_user_txn ) + @cachedInlineCallbacks(num_args=2) + def get_global_account_data_by_type_for_user(self, user_id, data_type): + """ + Returns: + Deferred: A dict + """ + result = yield self._simple_select_one_onecol( + table="account_data", + keyvalues={ + "user_id": user_id, + "account_data_type": data_type, + }, + retcol="content", + desc="get_global_account_data_by_type_for_user", + allow_none=True, + ) + + if result: + defer.returnValue(json.loads(result)) + else: + defer.returnValue(None) + def get_account_data_for_room(self, user_id, room_id): """Get all the client account_data for a user for a room. @@ -193,6 +218,7 @@ class AccountDataStore(SQLBaseStore): self._account_data_stream_cache.entity_has_changed, user_id, next_id, ) + txn.call_after(self.get_account_data_for_user.invalidate, (user_id,)) self._update_max_stream_id(txn, next_id) with self._account_data_id_gen.get_next() as next_id: @@ -232,6 +258,11 @@ class AccountDataStore(SQLBaseStore): self._account_data_stream_cache.entity_has_changed, user_id, next_id, ) + txn.call_after(self.get_account_data_for_user.invalidate, (user_id,)) + txn.call_after( + self.get_global_account_data_by_type_for_user.invalidate, + (user_id, account_data_type,) + ) self._update_max_stream_id(txn, next_id) with self._account_data_id_gen.get_next() as next_id: From 92f0f3d21d52ae14a3d4d4536a84055b92d228ae Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 May 2016 11:24:07 +0100 Subject: [PATCH 059/134] Catch all exceptions when creating a pusher --- synapse/push/pusherpool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 7fef2fb6f..66eafb69d 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -186,8 +186,8 @@ class PusherPool: for pusherdict in pushers: try: p = pusher.create_pusher(self.hs, pusherdict) - except PusherConfigException: - logger.exception("Couldn't start a pusher: caught PusherConfigException") + except: + logger.exception("Couldn't start a pusher: caught Exception") continue if p: appid_pushkey = "%s:%s" % ( From e6bffa4475e1a37643317d709b2003cdf99c149f Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 May 2016 11:26:58 +0100 Subject: [PATCH 060/134] Unused import --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 66eafb69d..5853ec36a 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -17,7 +17,6 @@ from twisted.internet import defer import pusher -from synapse.push import PusherConfigException from synapse.util.logcontext import preserve_fn from synapse.util.async import run_on_reactor From 984d4a2c0f59039a623b6a6f1945ff697f004c27 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 May 2016 11:28:10 +0100 Subject: [PATCH 061/134] Add /report endpoint --- synapse/rest/__init__.py | 2 + synapse/rest/client/v2_alpha/report_event.py | 59 ++++++++++++++++++++ synapse/storage/prepare_database.py | 2 +- synapse/storage/room.py | 14 +++++ synapse/storage/schema/delta/32/reports.sql | 23 ++++++++ 5 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/report_event.py create mode 100644 synapse/storage/schema/delta/32/reports.sql diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 6688fa8fa..e805cb911 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -44,6 +44,7 @@ from synapse.rest.client.v2_alpha import ( tokenrefresh, tags, account_data, + report_event, ) from synapse.http.server import JsonResource @@ -86,3 +87,4 @@ class ClientRestResource(JsonResource): tokenrefresh.register_servlets(hs, client_resource) tags.register_servlets(hs, client_resource) account_data.register_servlets(hs, client_resource) + report_event.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/v2_alpha/report_event.py new file mode 100644 index 000000000..412e5b190 --- /dev/null +++ b/synapse/rest/client/v2_alpha/report_event.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.http.servlet import RestServlet, parse_json_object_from_request +from ._base import client_v2_patterns + +import logging + + +logger = logging.getLogger(__name__) + + +class ReportEventRestServlet(RestServlet): + PATTERNS = client_v2_patterns( + "/rooms/(?P[^/]*)/report$" + ) + + def __init__(self, hs): + super(ReportEventRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.store = hs.get_datastore() + + @defer.inlineCallbacks + def on_POST(self, request, room_id): + requester = yield self.auth.get_user_by_req(request) + user_id = requester.user.to_string() + + body = parse_json_object_from_request(request) + + event_id = body["event_id"] + + yield self.store.add_event_report( + room_id=room_id, + event_id=event_id, + user_id=user_id, + reason=body.get("reason"), + content=body, + ) + + defer.returnValue((200, {})) + + +def register_servlets(hs, http_server): + ReportEventRestServlet(hs).register(http_server) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 57f14fd12..c8487c883 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 31 +SCHEMA_VERSION = 32 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 70aa64fb3..ceced7d51 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -23,6 +23,7 @@ from .engines import PostgresEngine, Sqlite3Engine import collections import logging +import ujson as json logger = logging.getLogger(__name__) @@ -221,3 +222,16 @@ class RoomStore(SQLBaseStore): aliases.extend(e.content['aliases']) defer.returnValue((name, aliases)) + + def add_event_report(self, room_id, event_id, user_id, reason, content): + return self._simple_insert( + table="event_reports", + values={ + "room_id": room_id, + "event_id": event_id, + "user_id": user_id, + "reason": reason, + "content": json.dumps(content), + }, + desc="add_event_report" + ) diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql new file mode 100644 index 000000000..06bf0d9b5 --- /dev/null +++ b/synapse/storage/schema/delta/32/reports.sql @@ -0,0 +1,23 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE TABLE event_reports( + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + reason TEXT, + content TEXT +); From 17cbf773b971e4122a913233a8656099465b3306 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 4 May 2016 11:37:21 +0100 Subject: [PATCH 062/134] fix assorted typos in default config --- synapse/config/emailconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index e4d86255f..7a3868043 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -78,12 +78,12 @@ class EmailConfig(Config): def default_config(self, config_dir_path, server_name, **kwargs): return """ # Enable sending emails for notification events - #email_config: + #email: # enable_notifs: false # smtp_host: "localhost" # smtp_port: 25 # notif_from: Your Friendly Matrix Home Server # template_dir: res/templates - # notif_template_html: notif_email.html - # notif_template_text: notif_email.txt + # notif_template_html: notif_mail.html + # notif_template_text: notif_mail.txt """ From f1026418ea0f2dba6cc4075b737ac6fa4ffac89d Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 4 May 2016 11:37:42 +0100 Subject: [PATCH 063/134] copyright --- synapse/push/pusher.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index 25a45af77..d1eaa236b 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -1,14 +1,36 @@ +# -*- coding: utf-8 -*- +# Copyright 2014-2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from httppusher import HttpPusher +import logging +logger = logging.getLogger(__name__) def create_pusher(hs, pusherdict): + logger.info("trying to create_pusher for %r", pusherdict) + PUSHER_TYPES = { "http": HttpPusher, } + logger.info("email enable notifs: %r", hs.config.email_enable_notifs) if hs.config.email_enable_notifs: from emailpusher import EmailPusher PUSHER_TYPES["email"] = EmailPusher + logger.info("defined email pusher type") if pusherdict['kind'] in PUSHER_TYPES: + logger.info("found pusher") return PUSHER_TYPES[pusherdict['kind']](hs, pusherdict) From de22001ab55d7fb89e4fd1527a4b419f5c195458 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 May 2016 11:41:35 +0100 Subject: [PATCH 064/134] pep8 --- synapse/push/pusher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index d1eaa236b..db5c1f1aa 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -18,6 +18,7 @@ from httppusher import HttpPusher import logging logger = logging.getLogger(__name__) + def create_pusher(hs, pusherdict): logger.info("trying to create_pusher for %r", pusherdict) From 8cc82aad873d8c6c933a20d8c7552d061b4e6316 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 May 2016 11:47:59 +0100 Subject: [PATCH 065/134] Add db functions used for email to the pusher app --- synapse/app/pusher.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index b5339f030..997690898 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -110,6 +110,18 @@ class PusherSlaveStore( DataStore.update_pusher_last_stream_ordering.__func__ ) + get_throttle_params_by_room = ( + DataStore.get_throttle_params_by_room.__func__ + ) + + set_throttle_params = ( + DataStore.set_throttle_params.__func__ + ) + + get_time_of_last_push_action_before = ( + DataStore.get_time_of_last_push_action_before.__func__ + ) + class PusherServer(HomeServer): From 8a04412fa1e837d733302038c64854ba0766efc0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 4 May 2016 12:19:04 +0100 Subject: [PATCH 066/134] starting point for doc on how log contexts are supposed to work --- docs/log_contexts.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/log_contexts.rst diff --git a/docs/log_contexts.rst b/docs/log_contexts.rst new file mode 100644 index 000000000..0046e171b --- /dev/null +++ b/docs/log_contexts.rst @@ -0,0 +1,10 @@ +What do I do about "Unexpected logging context" debug log-lines everywhere? + + The logging context lives in thread local storage + Sometimes it gets out of sync with what it should actually be, usually because something scheduled something to run on the reactor without preserving the logging context. + what is the impact of it getting out of sync? and how and when should we preserve log context? + The impact is that some of the CPU and database metrics will be under-reported, and some log lines will be mis-attributed. + It should happen auto-magically in all the APIs that do IO or otherwise defer to the reactor. + Mjark: the other place is if we branch, e.g. using defer.gatherResults + +Unanswered: how and when should we preserve log context? \ No newline at end of file From 5650e38e7de4cf89074ff84f4ecfbfcd81fa810d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 May 2016 13:19:39 +0100 Subject: [PATCH 067/134] Move event_id to path --- synapse/rest/client/v2_alpha/report_event.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/v2_alpha/report_event.py index 412e5b190..9c1c9662c 100644 --- a/synapse/rest/client/v2_alpha/report_event.py +++ b/synapse/rest/client/v2_alpha/report_event.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class ReportEventRestServlet(RestServlet): PATTERNS = client_v2_patterns( - "/rooms/(?P[^/]*)/report$" + "/rooms/(?P[^/]*)/report/(?P[^/]*)$" ) def __init__(self, hs): @@ -36,14 +36,12 @@ class ReportEventRestServlet(RestServlet): self.store = hs.get_datastore() @defer.inlineCallbacks - def on_POST(self, request, room_id): + def on_POST(self, request, room_id, event_id): requester = yield self.auth.get_user_by_req(request) user_id = requester.user.to_string() body = parse_json_object_from_request(request) - event_id = body["event_id"] - yield self.store.add_event_report( room_id=room_id, event_id=event_id, From 80be39646467c46a52530cd0839746810ad32b62 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 May 2016 13:19:59 +0100 Subject: [PATCH 068/134] Correct SQL statement for postgres In standard sql, join binds tighter than comma, so we were joining on the wrong table. Postgres follows the standard (apparently). --- synapse/storage/event_push_actions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 85290d2a9..6f316f7d2 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -120,14 +120,15 @@ class EventPushActionsStore(SQLBaseStore): sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions, " "e.received_ts " - "FROM event_push_actions AS ep, (" + "FROM (" " SELECT room_id, user_id, " " max(topological_ordering) as topological_ordering, " " max(stream_ordering) as stream_ordering " " FROM events" " NATURAL JOIN receipts_linearized WHERE receipt_type = 'm.read'" " GROUP BY room_id, user_id" - ") AS rl" + ") AS rl," + " event_push_actions AS ep" " INNER JOIN events AS e USING (room_id, event_id)" " WHERE" " ep.room_id = rl.room_id" From 9ef05a12c380a6a5a45226d6086d70512fd4f073 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 May 2016 14:52:10 +0100 Subject: [PATCH 069/134] Add date header & message id --- synapse/app/pusher.py | 3 ++- synapse/push/mailer.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 997690898..89c8d5c7c 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -20,6 +20,7 @@ from synapse.server import HomeServer from synapse.config._base import ConfigError from synapse.config.database import DatabaseConfig from synapse.config.logger import LoggingConfig +from synapse.config.emailconfig import EmailConfig from synapse.http.site import SynapseSite from synapse.metrics.resource import MetricsResource, METRICS_PREFIX from synapse.replication.slave.storage.events import SlavedEventStore @@ -91,7 +92,7 @@ class SlaveConfig(DatabaseConfig): """ % locals() -class PusherSlaveConfig(SlaveConfig, LoggingConfig): +class PusherSlaveConfig(SlaveConfig, LoggingConfig, EmailConfig): pass diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 1e6654e2b..037bc990d 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -152,6 +152,8 @@ class Mailer(object): multipart_msg['Subject'] = "New Matrix Notifications" multipart_msg['From'] = self.hs.config.email_notif_from multipart_msg['To'] = email_address + multipart_msg['Date'] = email.utils.formatdate() + multipart_msg['Message-ID'] = email.utils.make_msgid() multipart_msg.attach(text_part) multipart_msg.attach(html_part) From 39d0a99972cc58e8d2c7f7997d13aa18d547a2d6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 4 May 2016 14:52:49 +0100 Subject: [PATCH 070/134] Include no context until we can de-dup between the context and other notifs --- synapse/push/mailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 037bc990d..ad44ba985 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -44,7 +44,7 @@ MESSAGES_IN_ROOMS = "Here are some messages you may have missed" INVITE_FROM_PERSON_TO_ROOM = "%(person)s has invited you to join the %(room)s room" INVITE_FROM_PERSON = "%(person)s has invited you to chat" -CONTEXT_BEFORE = 1 +CONTEXT_BEFORE = 0 # From https://github.com/matrix-org/matrix-react-sdk/blob/master/src/HtmlUtils.js ALLOWED_TAGS = [ From 8e6a163f2762b3f62ae9b350c5050bc2318ec268 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 May 2016 15:19:12 +0100 Subject: [PATCH 071/134] Add timestamp and auto incrementing ID --- synapse/rest/client/v2_alpha/report_event.py | 2 ++ synapse/storage/__init__.py | 1 + synapse/storage/room.py | 6 +++++- synapse/storage/schema/delta/32/reports.sql | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/v2_alpha/report_event.py index 9c1c9662c..8903e1240 100644 --- a/synapse/rest/client/v2_alpha/report_event.py +++ b/synapse/rest/client/v2_alpha/report_event.py @@ -33,6 +33,7 @@ class ReportEventRestServlet(RestServlet): super(ReportEventRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.clock = hs.get_clock() self.store = hs.get_datastore() @defer.inlineCallbacks @@ -48,6 +49,7 @@ class ReportEventRestServlet(RestServlet): user_id=user_id, reason=body.get("reason"), content=body, + received_ts=self.clock.time_msec(), ) defer.returnValue((200, {})) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 045ae6c03..7122b0cbb 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -114,6 +114,7 @@ class DataStore(RoomMemberStore, RoomStore, self._state_groups_id_gen = StreamIdGenerator(db_conn, "state_groups", "id") self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id") self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id") + self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id") self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") self._push_rules_stream_id_gen = ChainedIdGenerator( diff --git a/synapse/storage/room.py b/synapse/storage/room.py index ceced7d51..26933e593 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -223,10 +223,14 @@ class RoomStore(SQLBaseStore): defer.returnValue((name, aliases)) - def add_event_report(self, room_id, event_id, user_id, reason, content): + def add_event_report(self, room_id, event_id, user_id, reason, content, + received_ts): + next_id = self._event_reports_id_gen.get_next() return self._simple_insert( table="event_reports", values={ + "id": next_id, + "received_ts": received_ts, "room_id": room_id, "event_id": event_id, "user_id": user_id, diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql index 06bf0d9b5..3f2502745 100644 --- a/synapse/storage/schema/delta/32/reports.sql +++ b/synapse/storage/schema/delta/32/reports.sql @@ -15,6 +15,8 @@ CREATE TABLE event_reports( + id BIGINT NOT NULL, + received_ts BIGINT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL, user_id TEXT NOT NULL, From fcd1eb642dc8b73e372b78bb788e3a17a5e40ace Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 May 2016 16:51:51 +0100 Subject: [PATCH 072/134] Add primary key --- synapse/storage/schema/delta/32/reports.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql index 3f2502745..d13609776 100644 --- a/synapse/storage/schema/delta/32/reports.sql +++ b/synapse/storage/schema/delta/32/reports.sql @@ -15,7 +15,7 @@ CREATE TABLE event_reports( - id BIGINT NOT NULL, + id BIGINT NOT NULL PRIMARY KEY, received_ts BIGINT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL, From 1cf5c379cb101c89897750d79e4548c172581f48 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 May 2016 01:54:12 +0100 Subject: [PATCH 073/134] spell out emailpusher full path --- synapse/push/pusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index db5c1f1aa..e6c080641 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -28,7 +28,7 @@ def create_pusher(hs, pusherdict): logger.info("email enable notifs: %r", hs.config.email_enable_notifs) if hs.config.email_enable_notifs: - from emailpusher import EmailPusher + from synapse.push.emailpusher import EmailPusher PUSHER_TYPES["email"] = EmailPusher logger.info("defined email pusher type") From ce81ccb063c46bf1bc3adca0d5dcfbc786624641 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 May 2016 01:56:43 +0100 Subject: [PATCH 074/134] handle fragments correctly on mxc URLs. switch to vector.im permalinks as matrix.to isn't ready yet. merge overlapping notifications together. give one message of context after a notification (in the unlikely event it exists, but it's possible thanks to throttling). include name of app in mail templates --- synapse/push/mailer.py | 59 ++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index ad44ba985..5951c14ce 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -35,16 +35,20 @@ import bleach import time import urllib +import logging +logger = logging.getLogger(__name__) -MESSAGE_FROM_PERSON_IN_ROOM = "You have a message from %(person)s in the %s room" -MESSAGE_FROM_PERSON = "You have a message from %(person)s" -MESSAGES_FROM_PERSON = "You have messages from %(person)s" -MESSAGES_IN_ROOM = "There are some messages for you in the %(room)s room" -MESSAGES_IN_ROOMS = "Here are some messages you may have missed" -INVITE_FROM_PERSON_TO_ROOM = "%(person)s has invited you to join the %(room)s room" -INVITE_FROM_PERSON = "%(person)s has invited you to chat" -CONTEXT_BEFORE = 0 +MESSAGE_FROM_PERSON_IN_ROOM = "You have a message on %%app%% from %(person)s in the %s room..." +MESSAGE_FROM_PERSON = "You have a message on %%app%% from %(person)s..." +MESSAGES_FROM_PERSON = "You have messages on %%app%% from %(person)s..." +MESSAGES_IN_ROOM = "There are some messages on %%app%% for you in the %(room)s room..." +MESSAGES_IN_ROOMS = "Here are some messages on %%app%% you may have missed..." +INVITE_FROM_PERSON_TO_ROOM = "%(person)s has invited you to join the %(room)s room on %%app%%..." +INVITE_FROM_PERSON = "%(person)s has invited you to chat on %%app%%..." + +CONTEXT_BEFORE = 1 +CONTEXT_AFTER = 1 # From https://github.com/matrix-org/matrix-react-sdk/blob/master/src/HtmlUtils.js ALLOWED_TAGS = [ @@ -181,7 +185,25 @@ class Mailer(object): notifvars = yield self.get_notif_vars( n, user_id, notif_events[n['event_id']], room_state ) - room_vars['notifs'].append(notifvars) + + # merge overlapping notifs together. + # relies on the notifs being in chronological order. + merge = False + if room_vars['notifs'] and 'messages' in room_vars['notifs'][-1]: + prev_messages = room_vars['notifs'][-1]['messages'] + for message in notifvars['messages']: + pm = filter(lambda pm: pm['id'] == message['id'], prev_messages) + if pm: + if not message["is_historical"]: + pm[0]["is_historical"] = False + merge = True + elif merge: + # we're merging, so append any remaining messages + # in this notif to the previous one + prev_messages.append(message) + + if not merge: + room_vars['notifs'].append(notifvars) defer.returnValue(room_vars) @@ -189,7 +211,7 @@ class Mailer(object): def get_notif_vars(self, notif, user_id, notif_event, room_state): results = yield self.store.get_events_around( notif['room_id'], notif['event_id'], - before_limit=CONTEXT_BEFORE, after_limit=0 + before_limit=CONTEXT_BEFORE, after_limit=CONTEXT_AFTER ) ret = { @@ -226,6 +248,7 @@ class Mailer(object): ret = { "msgtype": event.content["msgtype"], "is_historical": event.event_id != notif['event_id'], + "id": event.event_id, "ts": event.origin_server_ts, "sender_name": sender_name, "sender_avatar_url": sender_avatar_url, @@ -329,10 +352,12 @@ class Mailer(object): return MESSAGES_IN_ROOMS def make_room_link(self, room_id): - return "https://matrix.to/%s" % (room_id,) + # XXX: matrix.to + return "https://vector.im/#/room/%s" % (room_id,) def make_notif_link(self, notif): - return "https://matrix.to/%s/%s" % ( + # XXX: matrix.to + return "https://vector.im/#/room/%s/%s" % ( notif['room_id'], notif['event_id'] ) @@ -342,16 +367,22 @@ class Mailer(object): def mxc_to_http_filter(self, value, width, height, resize_method="crop"): if value[0:6] != "mxc://": return "" + serverAndMediaId = value[6:] + if '#' in serverAndMediaId: + (serverAndMediaId, fragment) = serverAndMediaId.split('#', 1) + fragment = "#" + fragment + params = { "width": width, "height": height, "method": resize_method, } - return "%s_matrix/media/v1/thumbnail/%s?%s" % ( + return "%s_matrix/media/v1/thumbnail/%s?%s%s" % ( self.hs.config.public_baseurl, serverAndMediaId, - urllib.urlencode(params) + urllib.urlencode(params), + fragment or "", ) From 3c39fa89027452cc4472ceb85d5d91a254cb2f50 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 May 2016 01:58:58 +0100 Subject: [PATCH 075/134] First cut at Vector-branded mail templates --- res/templates-vector/mail.css | 112 +++++++++++++++++++++++++++ res/templates-vector/notif.html | 34 ++++++++ res/templates-vector/notif.txt | 12 +++ res/templates-vector/notif_mail.html | 25 ++++++ res/templates-vector/notif_mail.txt | 10 +++ res/templates-vector/room.html | 27 +++++++ res/templates-vector/room.txt | 9 +++ 7 files changed, 229 insertions(+) create mode 100644 res/templates-vector/mail.css create mode 100644 res/templates-vector/notif.html create mode 100644 res/templates-vector/notif.txt create mode 100644 res/templates-vector/notif_mail.html create mode 100644 res/templates-vector/notif_mail.txt create mode 100644 res/templates-vector/room.html create mode 100644 res/templates-vector/room.txt diff --git a/res/templates-vector/mail.css b/res/templates-vector/mail.css new file mode 100644 index 000000000..642948fdc --- /dev/null +++ b/res/templates-vector/mail.css @@ -0,0 +1,112 @@ +body { + margin: 0px; +} + +#page { + font-family: 'Open Sans', Helvetica, Arial, Sans-Serif; + font-size: 12pt; + + margin: auto; + max-width: 640px; + padding: 20px; +} + +.header { + height: 87px; + border-bottom: 4px solid #e4f7ed; +} + +.logo { + float: right; + margin-left: 20px; +} + +.salutation { + padding-top: 10px; + font-weight: bold; +} + +.summarytext { +} + +.room_header { + padding-top: 38px; + padding-bottom: 10px; + border-bottom: 1px solid #e5e5e5; +} + +.room_header h2 { + margin-top: 0px; + margin-left: 75px; + font-size: 20px; +} + +.room_avatar { + float: left; + margin-top: -8px; +} + +.room_avatar img { + width: 48px; + height: 48px; + object-fit: cover; + border-radius: 24px; + margin-left: 7px; +} + +.room_content { + clear: left; +} + +.notif { + border-bottom: 1px solid #e5e5e5; + margin-top: 16px; + padding-bottom: 16px; +} + +.historical { + opacity: 0.3; +} + +.message { + position: relative; + margin-bottom: 10px; +} + +.sender_avatar { + width: 32px; + height: 32px; + border-radius: 16px; + position: absolute; + margin-left: 14px; + margin-top: -2px; +} + +.sender_name { + margin-left: 75px; + display: inline; + font-weight: bold; +} + +.message_time { + float: right; +} + +.message_body { + margin-left: 75px; +} + +.notif_link { + margin-left: 75px; + font-weight: bold; +} + +.notif_link a, .footer a { + color: #76CFA6; + text-decoration: none; +} + +.footer { + margin-top: 20px; + text-align: center; +} diff --git a/res/templates-vector/notif.html b/res/templates-vector/notif.html new file mode 100644 index 000000000..97ea42501 --- /dev/null +++ b/res/templates-vector/notif.html @@ -0,0 +1,34 @@ +
+
+ {% for message in notif.messages %} +
+ {% if message.sender_avatar_url %} + + {% else %} + {% if message.sender_hash % 3 == 0 %} + + {% elif message.sender_hash % 3 == 1 %} + + {% else %} + + {% endif %} + + {% endif %} +
{{ message.sender_name }}
+
{{ message.ts|format_ts("%H:%M") }}
+
+ {% if message.msgtype == "m.text" %} + {{ message.body_text_html }} + {% elif message.msgtype == "m.image" %} + + {% elif message.msgtype == "m.file" %} + {{ message.body_text_plain }} + {% endif %} +
+
+ {% endfor %} +
+
+ View {{ room.title }} +
+
diff --git a/res/templates-vector/notif.txt b/res/templates-vector/notif.txt new file mode 100644 index 000000000..b515f394c --- /dev/null +++ b/res/templates-vector/notif.txt @@ -0,0 +1,12 @@ +{% for message in notif.messages %} +{{ message.sender_name }} ({{ message.ts|format_ts("%H:%M") }}) +{% if message.msgtype == "m.text" %} +{{ message.body_text_plain }} +{% elif message.msgtype == "m.image" %} +{{ message.body_text_plain }} +{% elif message.msgtype == "m.file" %} +{{ message.body_text_plain }} +{% endif %} +{% endfor %} + +View at {{ notif.link }} diff --git a/res/templates-vector/notif_mail.html b/res/templates-vector/notif_mail.html new file mode 100644 index 000000000..86e7b6e86 --- /dev/null +++ b/res/templates-vector/notif_mail.html @@ -0,0 +1,25 @@ + + + + + + +
+
+ +
Hi {{ user_display_name }},
+
{{ summary_text|replace("%app%", "Vector") }}
+
+
+ {% for room in rooms %} + {% include 'room.html' with context %} + {% endfor %} +
+
+ Unsubscribe +
+
+ + diff --git a/res/templates-vector/notif_mail.txt b/res/templates-vector/notif_mail.txt new file mode 100644 index 000000000..24843042a --- /dev/null +++ b/res/templates-vector/notif_mail.txt @@ -0,0 +1,10 @@ +Hi {{ user_display_name }}, + +{{ summary_text }} + +{% for room in rooms %} +{% include 'room.txt' with context %} +{% endfor %} + +You can disable these notifications at {{ unsubscribe_link }} + diff --git a/res/templates-vector/room.html b/res/templates-vector/room.html new file mode 100644 index 000000000..ab1149257 --- /dev/null +++ b/res/templates-vector/room.html @@ -0,0 +1,27 @@ +
+
+
+ {% if room.avatar_url %} + + {% else %} + {% if room.hash % 3 == 0 %} + + {% elif room.hash % 3 == 1 %} + + {% else %} + + {% endif %} + {% endif %} +
+

{{ room.title }}

+
+
+ {% if room.invite %} + Join the conversation. + {% else %} + {% for notif in room.notifs %} + {% include 'notif.html' with context %} + {% endfor %} + {% endif %} +
+
diff --git a/res/templates-vector/room.txt b/res/templates-vector/room.txt new file mode 100644 index 000000000..84648c710 --- /dev/null +++ b/res/templates-vector/room.txt @@ -0,0 +1,9 @@ +{{ room.title }} + +{% if room.invite %} + You've been invited, join at {{ room.link }} +{% else %} + {% for notif in room.notifs %} + {% include 'notif.txt' with context %} + {% endfor %} +{% endif %} From c64d5fc66cdadaecd98de8ce58318c022fa3e2b2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 May 2016 01:59:20 +0100 Subject: [PATCH 076/134] fix room.txt --- res/templates/room.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/res/templates/room.txt b/res/templates/room.txt index 999d0ae60..84648c710 100644 --- a/res/templates/room.txt +++ b/res/templates/room.txt @@ -1,6 +1,9 @@ {{ room.title }} -You've been invited, join at {{ room.link }} -{% for notif in room.notifs %} -{% include 'notif.txt' with context %} -{% endfor %} +{% if room.invite %} + You've been invited, join at {{ room.link }} +{% else %} + {% for notif in room.notifs %} + {% include 'notif.txt' with context %} + {% endfor %} +{% endif %} From 634efb65f1ca8d3485c75a828a98161cfd617bb6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 May 2016 02:10:57 +0100 Subject: [PATCH 077/134] pep8 --- synapse/push/mailer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 5951c14ce..aab638712 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -39,12 +39,14 @@ import logging logger = logging.getLogger(__name__) -MESSAGE_FROM_PERSON_IN_ROOM = "You have a message on %%app%% from %(person)s in the %s room..." +MESSAGE_FROM_PERSON_IN_ROOM = "You have a message on %%app%% from %(person)s " \ + "in the %s room..." MESSAGE_FROM_PERSON = "You have a message on %%app%% from %(person)s..." MESSAGES_FROM_PERSON = "You have messages on %%app%% from %(person)s..." MESSAGES_IN_ROOM = "There are some messages on %%app%% for you in the %(room)s room..." MESSAGES_IN_ROOMS = "Here are some messages on %%app%% you may have missed..." -INVITE_FROM_PERSON_TO_ROOM = "%(person)s has invited you to join the %(room)s room on %%app%%..." +INVITE_FROM_PERSON_TO_ROOM = "%(person)s has invited you to join the " \ + "%(room)s room on %%app%%..." INVITE_FROM_PERSON = "%(person)s has invited you to chat on %%app%%..." CONTEXT_BEFORE = 1 From 1f0f5ffa1e22dcbe2b0bb605ccaf12bf571dc624 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 May 2016 09:51:03 +0100 Subject: [PATCH 078/134] Add bulk fetch storage API --- synapse/handlers/_base.py | 10 +++------- synapse/handlers/sync.py | 2 +- synapse/storage/account_data.py | 25 ++++++++++++++++++++++--- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 0912274e1..81f7929b5 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -95,13 +95,9 @@ class BaseHandler(object): row["event_id"] for rows in forgotten for row in rows ) - # Maps user_id -> account data content - ignore_dict_content = yield defer.gatherResults([ - preserve_fn(self.store.get_global_account_data_by_type_for_user)( - user_id, "m.ignored_user_list" - ).addCallback(lambda d, u: (u, d), user_id) - for user_id, is_peeking in user_tuples - ]).addCallback(dict) + ignore_dict_content = yield self.store.get_global_account_data_by_type_for_users( + "m.ignored_user_list", user_ids=[user_id for user_id, _ in user_tuples] + ) # FIXME: This will explode if people upload something incorrect. ignore_dict = { diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 27c3c1b52..0bb191328 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -522,7 +522,7 @@ class SyncHandler(BaseHandler): ) ignored_account_data = yield self.store.get_global_account_data_by_type_for_user( - user_id, "m.ignored_user_list" + "m.ignored_user_list", user_id=user_id, ) if ignored_account_data: diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index cc0b92bc8..ec7e8d40d 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -16,7 +16,7 @@ from ._base import SQLBaseStore from twisted.internet import defer -from synapse.util.caches.descriptors import cached, cachedInlineCallbacks +from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks import ujson as json import logging @@ -64,7 +64,7 @@ class AccountDataStore(SQLBaseStore): ) @cachedInlineCallbacks(num_args=2) - def get_global_account_data_by_type_for_user(self, user_id, data_type): + def get_global_account_data_by_type_for_user(self, data_type, user_id): """ Returns: Deferred: A dict @@ -85,6 +85,25 @@ class AccountDataStore(SQLBaseStore): else: defer.returnValue(None) + @cachedList(cached_method_name="get_global_account_data_by_type_for_user", + num_args=2, list_name="user_ids", inlineCallbacks=True) + def get_global_account_data_by_type_for_users(self, data_type, user_ids): + rows = yield self._simple_select_many_batch( + table="account_data", + column="user_id", + iterable=user_ids, + keyvalues={ + "account_data_type": data_type, + }, + retcols=("user_id", "content",), + desc="get_global_account_data_by_type_for_users", + ) + + defer.returnValue({ + row["user_id"]: json.loads(row["content"]) if row["content"] else None + for row in rows + }) + def get_account_data_for_room(self, user_id, room_id): """Get all the client account_data for a user for a room. @@ -261,7 +280,7 @@ class AccountDataStore(SQLBaseStore): txn.call_after(self.get_account_data_for_user.invalidate, (user_id,)) txn.call_after( self.get_global_account_data_by_type_for_user.invalidate, - (user_id, account_data_type,) + (account_data_type, user_id,) ) self._update_max_stream_id(txn, next_id) From 5d8a93a10ebdd58fd15d29f4e6e4f389b65855cb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 May 2016 10:29:21 +0100 Subject: [PATCH 079/134] Add some log information at returned replication streams --- synapse/replication/resource.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/synapse/replication/resource.py b/synapse/replication/resource.py index ff78c60f1..d7c49462c 100644 --- a/synapse/replication/resource.py +++ b/synapse/replication/resource.py @@ -159,6 +159,17 @@ class ReplicationResource(Resource): result = yield self.notifier.wait_for_replication(replicate, timeout) + for stream_name, stream_content in result.items(): + logger.info( + "Replicating %d rows of %s from %s -> %s", + len(stream_content["rows"]), + stream_name, + stream_content["position"], + request_streams.get(stream_name), + ) + if stream_content["position"] == request_streams.get(stream_name): + logger.warn("Returning same position for stream: %s", stream_name) + request.write(json.dumps(result, ensure_ascii=False)) finish_request(request) From a5974f89d645790632a5d0ebb92d253984fe17db Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 May 2016 11:25:56 +0100 Subject: [PATCH 080/134] fix app branding --- res/templates-vector/notif_mail.txt | 2 +- res/templates/notif_mail.html | 2 +- res/templates/notif_mail.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/res/templates-vector/notif_mail.txt b/res/templates-vector/notif_mail.txt index 24843042a..dec2e5960 100644 --- a/res/templates-vector/notif_mail.txt +++ b/res/templates-vector/notif_mail.txt @@ -1,6 +1,6 @@ Hi {{ user_display_name }}, -{{ summary_text }} +{{ summary_text|replace("%app%", "Vector") }} {% for room in rooms %} {% include 'room.txt' with context %} diff --git a/res/templates/notif_mail.html b/res/templates/notif_mail.html index cc3573e65..0290fdea0 100644 --- a/res/templates/notif_mail.html +++ b/res/templates/notif_mail.html @@ -8,7 +8,7 @@
Hi {{ user_display_name }},
-
{{ summary_text }}
+
{{ summary_text|replace("%app%", "Matrix") }}
{% for room in rooms %} {% include 'room.html' with context %} diff --git a/res/templates/notif_mail.txt b/res/templates/notif_mail.txt index 24843042a..5d5a8442f 100644 --- a/res/templates/notif_mail.txt +++ b/res/templates/notif_mail.txt @@ -1,6 +1,6 @@ Hi {{ user_display_name }}, -{{ summary_text }} +{{ summary_text|replace("%app%", "Matrix") }} {% for room in rooms %} {% include 'room.txt' with context %} From 9c272da05fcf51534aaa877647bc3b82bf841cf3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 5 May 2016 13:42:44 +0100 Subject: [PATCH 081/134] Add an openidish mechanism for proving to third parties that you own a given user_id --- synapse/federation/federation_server.py | 5 ++ synapse/federation/transport/server.py | 47 ++++++++++- synapse/rest/__init__.py | 2 + synapse/rest/client/v2_alpha/openid.py | 96 ++++++++++++++++++++++ synapse/storage/__init__.py | 4 +- synapse/storage/openid.py | 32 ++++++++ synapse/storage/schema/delta/32/openid.sql | 9 ++ 7 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 synapse/rest/client/v2_alpha/openid.py create mode 100644 synapse/storage/openid.py create mode 100644 synapse/storage/schema/delta/32/openid.sql diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 429ab6dde..f1d231b9d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -387,6 +387,11 @@ class FederationServer(FederationBase): "events": [ev.get_pdu_json(time_now) for ev in missing_events], }) + @log_function + def on_openid_userinfo(self, token): + ts_now_ms = self._clock.time_msec() + return self.store.get_user_id_for_open_id_token(token, ts_now_ms) + @log_function def _get_persisted_pdu(self, origin, event_id, do_auth=True): """ Get a PDU from the database with given origin and id. diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 3e552b6c4..5b6c7d11d 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.urls import FEDERATION_PREFIX as PREFIX from synapse.api.errors import Codes, SynapseError from synapse.http.server import JsonResource -from synapse.http.servlet import parse_json_object_from_request +from synapse.http.servlet import parse_json_object_from_request, parse_string from synapse.util.ratelimitutils import FederationRateLimiter import functools @@ -448,6 +448,50 @@ class On3pidBindServlet(BaseFederationServlet): return code +class OpenIdUserInfo(BaseFederationServlet): + """ + Exchange a bearer token for information about a user. + + The response format should be compatible with: + http://openid.net/specs/openid-connect-core-1_0.html#UserInfoResponse + + GET /openid/userinfo?access_token=ABDEFGH HTTP/1.1 + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "sub": "@userpart:example.org", + } + """ + + PATH = "/openid/userinfo" + + @defer.inlineCallbacks + def on_GET(self, request): + token = parse_string(request, "access_token") + if token is None: + defer.returnValue((401, { + "errcode": "M_MISSING_TOKEN", "error": "Access Token required" + })) + return + + user_id = yield self.handler.on_openid_userinfo(token) + + if user_id is None: + defer.returnValue((401, { + "errcode": "M_UNKNOWN_TOKEN", + "error": "Access Token unknown or expired" + })) + + defer.returnValue((200, {"sub": user_id})) + + # Avoid doing remote HS authorization checks which are done by default by + # BaseFederationServlet. + def _wrap(self, code): + return code + + SERVLET_CLASSES = ( FederationSendServlet, FederationPullServlet, @@ -468,6 +512,7 @@ SERVLET_CLASSES = ( FederationClientKeysClaimServlet, FederationThirdPartyInviteExchangeServlet, On3pidBindServlet, + OpenIdUserInfo, ) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index e805cb911..8b223e032 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -45,6 +45,7 @@ from synapse.rest.client.v2_alpha import ( tags, account_data, report_event, + openid, ) from synapse.http.server import JsonResource @@ -88,3 +89,4 @@ class ClientRestResource(JsonResource): tags.register_servlets(hs, client_resource) account_data.register_servlets(hs, client_resource) report_event.register_servlets(hs, client_resource) + openid.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/openid.py b/synapse/rest/client/v2_alpha/openid.py new file mode 100644 index 000000000..ddea75032 --- /dev/null +++ b/synapse/rest/client/v2_alpha/openid.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# Copyright 2015, 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ._base import client_v2_patterns + +from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.api.errors import AuthError +from synapse.util.stringutils import random_string + +from twisted.internet import defer + +import logging + +logger = logging.getLogger(__name__) + + +class IdTokenServlet(RestServlet): + """ + Get a bearer token that may be passed to a third party to confirm ownership + of a matrix user id. + + The format of the response could be made compatible with the format given + in http://openid.net/specs/openid-connect-core-1_0.html#TokenResponse + + But instead of returning a signed "id_token" the response contains the + name of the issuing matrix homeserver. This means that for now the third + party will need to check the validity of the "id_token" against the + federation /openid/userinfo endpoint of the homeserver. + + Request: + + POST /user/{user_id}/openid/token?access_token=... HTTP/1.1 + + {} + + Response: + + HTTP/1.1 200 OK + { + "access_token": "ABDEFGH", + "token_type": "Bearer", + "matrix_server_name": "example.com", + "expires_in": 3600, + } + """ + PATTERNS = client_v2_patterns( + "/user/(?P[^/]*)/openid/token" + ) + + EXPIRES_MS = 3600 * 1000 + + def __init__(self, hs): + super(IdTokenServlet, self).__init__() + self.auth = hs.get_auth() + self.store = hs.get_datastore() + self.clock = hs.get_clock() + self.server_name = hs.config.server_name + + @defer.inlineCallbacks + def on_POST(self, request, user_id): + requester = yield self.auth.get_user_by_req(request) + if user_id != requester.user.to_string(): + raise AuthError(403, "Cannot request tokens for other users.") + + # Parse the request body to make sure it's JSON, but ignore the contents + # for now. + parse_json_object_from_request(request) + + token = random_string(24) + ts_valid_until_ms = self.clock.time_msec() + self.EXPIRES_MS + + yield self.store.insert_open_id_token(token, ts_valid_until_ms, user_id) + + defer.returnValue((200, { + "access_token": token, + "token_type": "Bearer", + "matrix_server_name": self.server_name, + "expires_in": self.EXPIRES_MS / 1000, + })) + + +def register_servlets(hs, http_server): + IdTokenServlet(hs).register(http_server) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7122b0cbb..d970fde9e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,6 +44,7 @@ from .receipts import ReceiptsStore from .search import SearchStore from .tags import TagsStore from .account_data import AccountDataStore +from .openid import OpenIdStore from .util.id_generators import IdGenerator, StreamIdGenerator, ChainedIdGenerator @@ -81,7 +82,8 @@ class DataStore(RoomMemberStore, RoomStore, SearchStore, TagsStore, AccountDataStore, - EventPushActionsStore + EventPushActionsStore, + OpenIdStore, ): def __init__(self, db_conn, hs): diff --git a/synapse/storage/openid.py b/synapse/storage/openid.py new file mode 100644 index 000000000..5dabb607b --- /dev/null +++ b/synapse/storage/openid.py @@ -0,0 +1,32 @@ +from ._base import SQLBaseStore + + +class OpenIdStore(SQLBaseStore): + def insert_open_id_token(self, token, ts_valid_until_ms, user_id): + return self._simple_insert( + table="open_id_tokens", + values={ + "token": token, + "ts_valid_until_ms": ts_valid_until_ms, + "user_id": user_id, + }, + desc="insert_open_id_token" + ) + + def get_user_id_for_open_id_token(self, token, ts_now_ms): + def get_user_id_for_token_txn(txn): + sql = ( + "SELECT user_id FROM open_id_tokens" + " WHERE token = ? AND ? <= ts_valid_until_ms" + ) + + txn.execute(sql, (token, ts_now_ms)) + + rows = txn.fetchall() + if not rows: + return None + else: + return rows[0][0] + return self.runInteraction( + "get_user_id_for_token", get_user_id_for_token_txn + ) diff --git a/synapse/storage/schema/delta/32/openid.sql b/synapse/storage/schema/delta/32/openid.sql new file mode 100644 index 000000000..36f37b11c --- /dev/null +++ b/synapse/storage/schema/delta/32/openid.sql @@ -0,0 +1,9 @@ + +CREATE TABLE open_id_tokens ( + token TEXT NOT NULL PRIMARY KEY, + ts_valid_until_ms bigint NOT NULL, + user_id TEXT NOT NULL, + UNIQUE (token) +); + +CREATE index open_id_tokens_ts_valid_until_ms ON open_id_tokens(ts_valid_until_ms); From 8940281d1b2e67720ae257d224dbef7280cfa55c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 5 May 2016 15:10:03 +0100 Subject: [PATCH 082/134] Don't warn --- synapse/replication/resource.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/replication/resource.py b/synapse/replication/resource.py index d7c49462c..69ad1de86 100644 --- a/synapse/replication/resource.py +++ b/synapse/replication/resource.py @@ -167,8 +167,6 @@ class ReplicationResource(Resource): stream_content["position"], request_streams.get(stream_name), ) - if stream_content["position"] == request_streams.get(stream_name): - logger.warn("Returning same position for stream: %s", stream_name) request.write(json.dumps(result, ensure_ascii=False)) finish_request(request) From 573ef3f1c953542693a1784311154d3345caf5c1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 5 May 2016 15:15:00 +0100 Subject: [PATCH 083/134] Rename openid/token to openid/request_token --- synapse/rest/client/v2_alpha/openid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/openid.py b/synapse/rest/client/v2_alpha/openid.py index ddea75032..aa1cae8e1 100644 --- a/synapse/rest/client/v2_alpha/openid.py +++ b/synapse/rest/client/v2_alpha/openid.py @@ -42,7 +42,7 @@ class IdTokenServlet(RestServlet): Request: - POST /user/{user_id}/openid/token?access_token=... HTTP/1.1 + POST /user/{user_id}/openid/request_token?access_token=... HTTP/1.1 {} @@ -57,7 +57,7 @@ class IdTokenServlet(RestServlet): } """ PATTERNS = client_v2_patterns( - "/user/(?P[^/]*)/openid/token" + "/user/(?P[^/]*)/openid/request_token" ) EXPIRES_MS = 3600 * 1000 From 81c2176cbadf7646fc075ca45dea4360ce7b8258 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 May 2016 15:54:29 +0100 Subject: [PATCH 084/134] fix layout; handle app naming in synapse, not jinja --- res/templates-vector/mail.css | 5 ++- res/templates-vector/notif_mail.html | 4 +-- res/templates-vector/notif_mail.txt | 2 +- res/templates/notif_mail.html | 2 +- res/templates/notif_mail.txt | 2 +- synapse/config/emailconfig.py | 5 +++ synapse/push/mailer.py | 47 ++++++++++++++++++---------- 7 files changed, 45 insertions(+), 22 deletions(-) diff --git a/res/templates-vector/mail.css b/res/templates-vector/mail.css index 642948fdc..103c5ff8c 100644 --- a/res/templates-vector/mail.css +++ b/res/templates-vector/mail.css @@ -85,11 +85,14 @@ body { .sender_name { margin-left: 75px; display: inline; - font-weight: bold; + font-size: 13px; + color: #a2a2a2; } .message_time { float: right; + font-size: 11px; + color: #a2a2a2; } .message_body { diff --git a/res/templates-vector/notif_mail.html b/res/templates-vector/notif_mail.html index 86e7b6e86..c49cc6654 100644 --- a/res/templates-vector/notif_mail.html +++ b/res/templates-vector/notif_mail.html @@ -8,9 +8,9 @@
- +
Hi {{ user_display_name }},
-
{{ summary_text|replace("%app%", "Vector") }}
+
{{ summary_text }}
{% for room in rooms %} diff --git a/res/templates-vector/notif_mail.txt b/res/templates-vector/notif_mail.txt index dec2e5960..24843042a 100644 --- a/res/templates-vector/notif_mail.txt +++ b/res/templates-vector/notif_mail.txt @@ -1,6 +1,6 @@ Hi {{ user_display_name }}, -{{ summary_text|replace("%app%", "Vector") }} +{{ summary_text }} {% for room in rooms %} {% include 'room.txt' with context %} diff --git a/res/templates/notif_mail.html b/res/templates/notif_mail.html index 0290fdea0..cc3573e65 100644 --- a/res/templates/notif_mail.html +++ b/res/templates/notif_mail.html @@ -8,7 +8,7 @@
Hi {{ user_display_name }},
-
{{ summary_text|replace("%app%", "Matrix") }}
+
{{ summary_text }}
{% for room in rooms %} {% include 'room.html' with context %} diff --git a/res/templates/notif_mail.txt b/res/templates/notif_mail.txt index 5d5a8442f..24843042a 100644 --- a/res/templates/notif_mail.txt +++ b/res/templates/notif_mail.txt @@ -1,6 +1,6 @@ Hi {{ user_display_name }}, -{{ summary_text|replace("%app%", "Matrix") }} +{{ summary_text }} {% for room in rooms %} {% include 'room.txt' with context %} diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 7a3868043..d6f4f83a1 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -65,6 +65,10 @@ class EmailConfig(Config): self.email_template_dir = email_config["template_dir"] self.email_notif_template_html = email_config["notif_template_html"] self.email_notif_template_text = email_config["notif_template_text"] + if "app_name" in email_config: + self.email_app_name = email_config["app_name"] + else: + self.email_app_name = "Matrix" # make sure it's valid parsed = email.utils.parseaddr(self.email_notif_from) @@ -83,6 +87,7 @@ class EmailConfig(Config): # smtp_host: "localhost" # smtp_port: 25 # notif_from: Your Friendly Matrix Home Server + # app_name: Matrix # template_dir: res/templates # notif_template_html: notif_mail.html # notif_template_text: notif_mail.txt diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index aab638712..497905d81 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -39,15 +39,15 @@ import logging logger = logging.getLogger(__name__) -MESSAGE_FROM_PERSON_IN_ROOM = "You have a message on %%app%% from %(person)s " \ +MESSAGE_FROM_PERSON_IN_ROOM = "You have a message on %(app)s from %(person)s " \ "in the %s room..." -MESSAGE_FROM_PERSON = "You have a message on %%app%% from %(person)s..." -MESSAGES_FROM_PERSON = "You have messages on %%app%% from %(person)s..." -MESSAGES_IN_ROOM = "There are some messages on %%app%% for you in the %(room)s room..." -MESSAGES_IN_ROOMS = "Here are some messages on %%app%% you may have missed..." +MESSAGE_FROM_PERSON = "You have a message on %(app)s from %(person)s..." +MESSAGES_FROM_PERSON = "You have messages on %(app)s from %(person)s..." +MESSAGES_IN_ROOM = "There are some messages on %(app)s for you in the %(room)s room..." +MESSAGES_IN_ROOMS = "Here are some messages on %(app)s you may have missed..." INVITE_FROM_PERSON_TO_ROOM = "%(person)s has invited you to join the " \ - "%(room)s room on %%app%%..." -INVITE_FROM_PERSON = "%(person)s has invited you to chat on %%app%%..." + "%(room)s room on %(app)s..." +INVITE_FROM_PERSON = "%(person)s has invited you to chat on %(app)s..." CONTEXT_BEFORE = 1 CONTEXT_AFTER = 1 @@ -79,6 +79,7 @@ class Mailer(object): self.store = self.hs.get_datastore() self.state_handler = self.hs.get_state_handler() loader = jinja2.FileSystemLoader(self.hs.config.email_template_dir) + self.app_name = self.hs.config.email_app_name env = jinja2.Environment(loader=loader) env.filters["format_ts"] = format_ts_filter env.filters["mxc_to_http"] = self.mxc_to_http_filter @@ -306,10 +307,15 @@ class Mailer(object): inviter_name = name_from_member_event(inviter_member_event) if room_name is None: - return INVITE_FROM_PERSON % {"person": inviter_name} + return INVITE_FROM_PERSON % { + "person": inviter_name, + "app": self.app_name + } else: return INVITE_FROM_PERSON_TO_ROOM % { - "person": inviter_name, "room": room_name + "person": inviter_name, + "room": room_name, + "app": self.app_name, } sender_name = None @@ -322,18 +328,22 @@ class Mailer(object): if sender_name is not None and room_name is not None: return MESSAGE_FROM_PERSON_IN_ROOM % { - "person": sender_name, "room": room_name + "person": sender_name, + "room": room_name, + "app": self.app_name, } elif sender_name is not None: return MESSAGE_FROM_PERSON % { - "person": sender_name + "person": sender_name, + "app": self.app_name, } else: # There's more than one notification for this room, so just # say there are several if room_name is not None: return MESSAGES_IN_ROOM % { - "room": room_name + "room": room_name, + "app": self.app_name, } else: # If the room doesn't have a name, say who the messages @@ -347,19 +357,24 @@ class Mailer(object): "person": descriptor_from_member_events([ state_by_room[room_id][("m.room.member", s)] for s in sender_ids - ]) + ]), + "app": self.app_name, } else: # Stuff's happened in multiple different rooms - return MESSAGES_IN_ROOMS + return MESSAGES_IN_ROOMS % { + "app": self.app_name, + } def make_room_link(self, room_id): # XXX: matrix.to - return "https://vector.im/#/room/%s" % (room_id,) + # need /beta for Universal Links to work on iOS + return "https://vector.im/beta/#/room/%s" % (room_id,) def make_notif_link(self, notif): # XXX: matrix.to - return "https://vector.im/#/room/%s/%s" % ( + # need /beta for Universal Links to work on iOS + return "https://vector.im/beta/#/room/%s/%s" % ( notif['room_id'], notif['event_id'] ) From 53ca739f1f3a61f8605dac3f3ab9540dadda0178 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 5 May 2016 15:55:44 +0100 Subject: [PATCH 085/134] better mail subject lines --- synapse/push/mailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 497905d81..f60ac94a8 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -156,7 +156,7 @@ class Mailer(object): text_part = MIMEText(plain_text, "plain", "utf8") multipart_msg = MIMEMultipart('alternative') - multipart_msg['Subject'] = "New Matrix Notifications" + multipart_msg['Subject'] = "[%s] %s" % (self.app_name, summary_text) multipart_msg['From'] = self.hs.config.email_notif_from multipart_msg['To'] = email_address multipart_msg['Date'] = email.utils.formatdate() From 56b5e83e36f22f3eab1ebf7a46b9f23f0c1a3e8d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 May 2016 11:20:18 +0100 Subject: [PATCH 086/134] Reduce database inserts when sending transactions --- synapse/handlers/presence.py | 2 +- synapse/storage/transactions.py | 157 +++++++++++++++++++++++--------- 2 files changed, 114 insertions(+), 45 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index d0c8f1328..639567953 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -168,7 +168,7 @@ class PresenceHandler(BaseHandler): # The initial delay is to allow disconnected clients a chance to # reconnect before we treat them as offline. self.clock.call_later( - 0 * 1000, + 30 * 1000, self.clock.looping_call, self._handle_timeouts, 5000, diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index d338dfcf0..17fc60198 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -16,16 +16,54 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached +from twisted.internet import defer, reactor + from canonicaljson import encode_canonical_json + +from collections import namedtuple + +import itertools import logging logger = logging.getLogger(__name__) +_TransactionRow = namedtuple( + "_TransactionRow", ( + "id", "transaction_id", "destination", "ts", "response_code", + "response_json", + ) +) + +_UpdateTransactionRow = namedtuple( + "_TransactionRow", ( + "response_code", "response_json", + ) +) + + class TransactionStore(SQLBaseStore): """A collection of queries for handling PDUs. """ + def __init__(self, hs): + super(TransactionStore, self).__init__(hs) + + # New transactions that are currently in flights + self.inflight_transactions = {} + + # Newly delievered transactions that *weren't* persisted while in flight + self.new_delivered_transactions = {} + + # Newly delivered transactions that *were* persisted while in flight + self.update_delivered_transactions = {} + + reactor.addSystemEventTrigger("before", "shutdown", self._persist_in_mem_txns) + hs.get_clock().looping_call( + self._persist_in_mem_txns, + 1000, + ) + def get_received_txn_response(self, transaction_id, origin): """For an incoming transaction from a given origin, check if we have already responded to it. If so, return the response code and response @@ -108,17 +146,28 @@ class TransactionStore(SQLBaseStore): list: A list of previous transaction ids. """ - return self.runInteraction( - "prep_send_transaction", - self._prep_send_transaction, - transaction_id, destination, origin_server_ts + auto_id = self._transaction_id_gen.get_next() + + txn_row = _TransactionRow( + id=auto_id, + transaction_id=transaction_id, + destination=destination, + ts=origin_server_ts, + response_code=0, + response_json=None, ) - def _prep_send_transaction(self, txn, transaction_id, destination, - origin_server_ts): + self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row - next_id = self._transaction_id_gen.get_next() + # TODO: Fetch prev_txns + return self.runInteraction( + "prep_send_transaction", + self._get_prevs_txn, + destination, + ) + + def _get_prevs_txn(self, txn, destination): # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, # we can simply take the last one. @@ -133,23 +182,6 @@ class TransactionStore(SQLBaseStore): prev_txns = [r["transaction_id"] for r in results] - # Actually add the new transaction to the sent_transactions table. - - self._simple_insert_txn( - txn, - table="sent_transactions", - values={ - "id": next_id, - "transaction_id": transaction_id, - "destination": destination, - "ts": origin_server_ts, - "response_code": 0, - "response_json": None, - } - ) - - # TODO Update the tx id -> pdu id mapping - return prev_txns def delivered_txn(self, transaction_id, destination, code, response_dict): @@ -161,27 +193,21 @@ class TransactionStore(SQLBaseStore): code (int) response_json (str) """ - return self.runInteraction( - "delivered_txn", - self._delivered_txn, - transaction_id, destination, code, - buffer(encode_canonical_json(response_dict)), - ) - def _delivered_txn(self, txn, transaction_id, destination, - code, response_json): - self._simple_update_one_txn( - txn, - table="sent_transactions", - keyvalues={ - "transaction_id": transaction_id, - "destination": destination, - }, - updatevalues={ - "response_code": code, - "response_json": None, # For now, don't persist response_json - } - ) + txn_row = self.inflight_transactions.get( + destination, {} + ).pop(transaction_id, None) + + if txn_row: + d = self.new_delivered_transactions.setdefault(destination, {}) + d[transaction_id] = txn_row._replace( + response_code=code, + response_json=None, # For now, don't persist response + ) + else: + d = self.update_delivered_transactions.setdefault(destination, {}) + # For now, don't persist response + d[transaction_id] = _UpdateTransactionRow(code, None) def get_transactions_after(self, transaction_id, destination): """Get all transactions after a given local transaction_id. @@ -305,3 +331,46 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (self._clock.time_msec(),)) return self.cursor_to_dict(txn) + + @defer.inlineCallbacks + def _persist_in_mem_txns(self): + try: + inflight = self.inflight_transactions + new_delivered = self.new_delivered_transactions + update_delivered = self.update_delivered_transactions + + self.inflight_transactions = {} + self.new_delivered_transactions = {} + self.update_delivered_transactions = {} + + full_rows = [ + row._asdict() + for txn_map in itertools.chain(inflight.values(), new_delivered.values()) + for row in txn_map.values() + ] + + def f(txn): + self._simple_insert_many_txn( + txn=txn, + table="sent_transactions", + values=full_rows + ) + + for dest, txn_map in update_delivered.items(): + for txn_id, update_row in txn_map.items(): + self._simple_update_one_txn( + txn, + table="sent_transactions", + keyvalues={ + "transaction_id": txn_id, + "destination": dest, + }, + updatevalues={ + "response_code": update_row.response_code, + "response_json": None, # For now, don't persist response + } + ) + + yield self.runInteraction("_persist_in_mem_txns", f) + except: + logger.exception("Failed to persist transactions!") From 1d275dba69272f6df5a79871dc4b8d31e71514d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 May 2016 11:25:58 +0100 Subject: [PATCH 087/134] Don't needlessly enter transaction --- synapse/storage/transactions.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 17fc60198..ba1969b24 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -159,10 +159,8 @@ class TransactionStore(SQLBaseStore): self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row - # TODO: Fetch prev_txns - return self.runInteraction( - "prep_send_transaction", + "_get_prevs_txn", self._get_prevs_txn, destination, ) @@ -350,11 +348,12 @@ class TransactionStore(SQLBaseStore): ] def f(txn): - self._simple_insert_many_txn( - txn=txn, - table="sent_transactions", - values=full_rows - ) + if full_rows: + self._simple_insert_many_txn( + txn=txn, + table="sent_transactions", + values=full_rows + ) for dest, txn_map in update_delivered.items(): for txn_id, update_row in txn_map.items(): @@ -371,6 +370,7 @@ class TransactionStore(SQLBaseStore): } ) - yield self.runInteraction("_persist_in_mem_txns", f) + if full_rows or update_delivered: + yield self.runInteraction("_persist_in_mem_txns", f) except: logger.exception("Failed to persist transactions!") From d13459636fab9637a43c950d17f883ca77b832f7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 May 2016 11:30:55 +0100 Subject: [PATCH 088/134] Pull prev txn from in memory --- synapse/storage/transactions.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index ba1969b24..6c7481a72 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -58,6 +58,8 @@ class TransactionStore(SQLBaseStore): # Newly delivered transactions that *were* persisted while in flight self.update_delivered_transactions = {} + self.last_transaction = {} + reactor.addSystemEventTrigger("before", "shutdown", self._persist_in_mem_txns) hs.get_clock().looping_call( self._persist_in_mem_txns, @@ -159,11 +161,15 @@ class TransactionStore(SQLBaseStore): self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row - return self.runInteraction( - "_get_prevs_txn", - self._get_prevs_txn, - destination, - ) + prev_txn = self.last_transaction.get(destination) + if prev_txn: + return defer.succeed(prev_txn) + else: + return self.runInteraction( + "_get_prevs_txn", + self._get_prevs_txn, + destination, + ) def _get_prevs_txn(self, txn, destination): # First we find out what the prev_txns should be. @@ -196,6 +202,8 @@ class TransactionStore(SQLBaseStore): destination, {} ).pop(transaction_id, None) + self.last_transaction[destination] = transaction_id + if txn_row: d = self.new_delivered_transactions.setdefault(destination, {}) d[transaction_id] = txn_row._replace( From b6e0be701eb8ae36236242549ef356397f5f1d06 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 May 2016 14:31:38 +0100 Subject: [PATCH 089/134] Queue events for persistence --- synapse/storage/events.py | 157 +++++++++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 12 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0307b2af3..07e873fce 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -19,12 +19,14 @@ from twisted.internet import defer, reactor from synapse.events import FrozenEvent, USE_FROZEN_DICTS from synapse.events.utils import prune_event +from synapse.util.async import ObservableDeferred from synapse.util.logcontext import preserve_fn, PreserveLoggingContext from synapse.util.logutils import log_function from synapse.api.constants import EventTypes from canonicaljson import encode_canonical_json -from collections import namedtuple +from collections import deque, namedtuple + import logging import math @@ -50,6 +52,80 @@ EVENT_QUEUE_ITERATIONS = 3 # No. times we block waiting for requests for events EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for requests for events +class _EventPeristenceQueue(object): + """Queues up events so that they can be persisted in bulk with only one + concurrent transaction per room. + """ + + _EventPersistQueueItem = namedtuple("_EventPersistQueueItem", ( + "events_and_contexts", "current_state", "backfilled", "deferred", + )) + + def __init__(self): + self._event_persist_queues = {} + self._currently_persisting_rooms = set() + + def add_to_queue(self, room_id, events_and_contexts, backfilled, current_state): + """Add events to the queue, with the given persist_event options. + """ + queue = self._event_persist_queues.setdefault(room_id, deque()) + if queue: + end_item = queue[-1] + if end_item.current_state or current_state: + # We perist events with current_state set to True one at a time + pass + if end_item.backfilled == backfilled: + end_item.events_and_contexts.extend(events_and_contexts) + return end_item.deferred.observe() + + deferred = ObservableDeferred(defer.Deferred()) + + queue.append(self._EventPersistQueueItem( + events_and_contexts=events_and_contexts, + backfilled=backfilled, + current_state=current_state, + deferred=deferred, + )) + + return deferred.observe() + + def handle_queue(self, room_id, callback): + """Attempts to handle the queue for a room if not already being handled. + + The given callback will be invoked with a 'queue' arg, which is a + generator over _EventPersistQueueItem's. The queue will finish if there + are no longer any items in the room queue. + + This function should therefore be called whenever anything is added + to the queue. + + If another callback is currently handling the queue then it will not be + invoked. + """ + + if room_id in self._currently_persisting_rooms: + return + + self._currently_persisting_rooms.add(room_id) + + try: + callback(self._get_drainining_queue(room_id)) + finally: + self._currently_persisting_rooms.discard(room_id) + + def _get_drainining_queue(self, room_id): + queue = self._event_persist_queues.pop(room_id, None) + if not queue: + return + + try: + while True: + yield queue.popleft() + except IndexError: + # Queue has been drained. + pass + + class EventsStore(SQLBaseStore): EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts" @@ -59,19 +135,80 @@ class EventsStore(SQLBaseStore): self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts ) - @defer.inlineCallbacks + self._event_persist_queue = _EventPeristenceQueue() + def persist_events(self, events_and_contexts, backfilled=False): """ Write events to the database Args: events_and_contexts: list of tuples of (event, context) backfilled: ? - - Returns: Tuple of stream_orderings where the first is the minimum and - last is the maximum stream ordering assigned to the events when - persisting. - """ + partitioned = {} + for event, ctx in events_and_contexts: + partitioned.setdefault(event.room_id, []).append((event, ctx)) + + deferreds = [] + for room_id, evs_ctxs in partitioned.items(): + d = self._event_persist_queue.add_to_queue( + room_id, evs_ctxs, + backfilled=backfilled, + current_state=None, + ) + deferreds.append(d) + + for room_id in partitioned.keys(): + self._maybe_start_persisting(room_id) + + return defer.gatherResults(deferreds, consumeErrors=True) + + @defer.inlineCallbacks + @log_function + def persist_event(self, event, context, current_state=None, backfilled=False): + deferred = self._event_persist_queue.add_to_queue( + event.room_id, [(event, context)], + backfilled=backfilled, + current_state=current_state, + ) + + self._maybe_start_persisting(event.room_id) + + yield deferred + + max_persisted_id = yield self._stream_id_gen.get_current_token() + defer.returnValue((event.internal_metadata.stream_ordering, max_persisted_id)) + + def _maybe_start_persisting(self, room_id): + @defer.inlineCallbacks + def persisting_queue(queue): + for item in queue: + try: + ret = None + if item.current_state: + for event, context in item.events_and_contexts: + # There should only ever be one item in + # events_and_contexts when current_state is + # not None + yield self._persist_event( + event, context, + current_state=item.current_state, + backfilled=item.backfilled, + ) + else: + yield self._persist_events( + item.events_and_contexts, + backfilled=item.backfilled, + ) + logger.info("Resolving with ret: %r", ret) + item.deferred.callback(ret) + except Exception as e: + logger.exception("Failed to persist events") + item.deferred.errback(e) + + self._event_persist_queue.handle_queue(room_id, persisting_queue) + + @defer.inlineCallbacks + def _persist_events(self, events_and_contexts, backfilled=False): if not events_and_contexts: return @@ -118,8 +255,7 @@ class EventsStore(SQLBaseStore): @defer.inlineCallbacks @log_function - def persist_event(self, event, context, current_state=None, backfilled=False): - + def _persist_event(self, event, context, current_state=None, backfilled=False): try: with self._stream_id_gen.get_next() as stream_ordering: with self._state_groups_id_gen.get_next() as state_group_id: @@ -136,9 +272,6 @@ class EventsStore(SQLBaseStore): except _RollbackButIsFineException: pass - max_persisted_id = yield self._stream_id_gen.get_current_token() - defer.returnValue((stream_ordering, max_persisted_id)) - @defer.inlineCallbacks def get_event(self, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False, From fd85b167ecb650451bf8bbfc68b771f607bb1d9d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 May 2016 15:38:42 +0100 Subject: [PATCH 090/134] Pull loop one level up --- synapse/storage/events.py | 77 +++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 07e873fce..9f8f0a082 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -19,7 +19,7 @@ from twisted.internet import defer, reactor from synapse.events import FrozenEvent, USE_FROZEN_DICTS from synapse.events.utils import prune_event -from synapse.util.async import ObservableDeferred +from synapse.util.async import ObservableDeferred, run_on_reactor from synapse.util.logcontext import preserve_fn, PreserveLoggingContext from synapse.util.logutils import log_function from synapse.api.constants import EventTypes @@ -89,12 +89,14 @@ class _EventPeristenceQueue(object): return deferred.observe() - def handle_queue(self, room_id, callback): + def handle_queue(self, room_id, per_item_callback): """Attempts to handle the queue for a room if not already being handled. - The given callback will be invoked with a 'queue' arg, which is a - generator over _EventPersistQueueItem's. The queue will finish if there - are no longer any items in the room queue. + The given callback will be invoked with for each item in the queue,1 + of type _EventPersistQueueItem. The per_item_callback will continuously + be called with new items, unless the queue becomnes empty. The return + value of the function will be given to the deferreds waiting on the item, + exceptions will be passed to the deferres as well. This function should therefore be called whenever anything is added to the queue. @@ -108,15 +110,26 @@ class _EventPeristenceQueue(object): self._currently_persisting_rooms.add(room_id) - try: - callback(self._get_drainining_queue(room_id)) - finally: - self._currently_persisting_rooms.discard(room_id) + @defer.inlineCallbacks + def handle_queue_loop(): + try: + queue = self._get_drainining_queue(room_id) + for item in queue: + try: + ret = yield per_item_callback(item) + item.deferred.callback(ret) + except Exception as e: + item.deferred.errback(e) + finally: + queue = self._event_persist_queues.pop(room_id, None) + if queue: + self._event_persist_queues[room_id] = queue + self._currently_persisting_rooms.discard(room_id) + + preserve_fn(handle_queue_loop)() def _get_drainining_queue(self, room_id): - queue = self._event_persist_queues.pop(room_id, None) - if not queue: - return + queue = self._event_persist_queues.setdefault(room_id, deque()) try: while True: @@ -180,30 +193,22 @@ class EventsStore(SQLBaseStore): def _maybe_start_persisting(self, room_id): @defer.inlineCallbacks - def persisting_queue(queue): - for item in queue: - try: - ret = None - if item.current_state: - for event, context in item.events_and_contexts: - # There should only ever be one item in - # events_and_contexts when current_state is - # not None - yield self._persist_event( - event, context, - current_state=item.current_state, - backfilled=item.backfilled, - ) - else: - yield self._persist_events( - item.events_and_contexts, - backfilled=item.backfilled, - ) - logger.info("Resolving with ret: %r", ret) - item.deferred.callback(ret) - except Exception as e: - logger.exception("Failed to persist events") - item.deferred.errback(e) + def persisting_queue(item): + if item.current_state: + for event, context in item.events_and_contexts: + # There should only ever be one item in + # events_and_contexts when current_state is + # not None + yield self._persist_event( + event, context, + current_state=item.current_state, + backfilled=item.backfilled, + ) + else: + yield self._persist_events( + item.events_and_contexts, + backfilled=item.backfilled, + ) self._event_persist_queue.handle_queue(room_id, persisting_queue) From fcb2c3f0db70593709c3bfbfc66772066c4aa061 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 May 2016 15:47:40 +0100 Subject: [PATCH 091/134] Remove unused import --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 9f8f0a082..a27b7919c 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -19,7 +19,7 @@ from twisted.internet import defer, reactor from synapse.events import FrozenEvent, USE_FROZEN_DICTS from synapse.events.utils import prune_event -from synapse.util.async import ObservableDeferred, run_on_reactor +from synapse.util.async import ObservableDeferred from synapse.util.logcontext import preserve_fn, PreserveLoggingContext from synapse.util.logutils import log_function from synapse.api.constants import EventTypes From 4ea762c1a28edbf18d0a183ed35fb8a5a11847c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 May 2016 10:08:21 +0100 Subject: [PATCH 092/134] Add cache to get_user_by_id --- synapse/storage/registration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 7af0cae6a..bda84a744 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -101,6 +101,7 @@ class RegistrationStore(SQLBaseStore): make_guest, appservice_id ) + self.get_user_by_id.invalidate((user_id,)) self.is_guest.invalidate((user_id,)) def _register( @@ -156,6 +157,7 @@ class RegistrationStore(SQLBaseStore): (next_id, user_id, token,) ) + @cached() def get_user_by_id(self, user_id): return self._simple_select_one( table="users", @@ -193,6 +195,7 @@ class RegistrationStore(SQLBaseStore): }, { 'password_hash': password_hash }) + self.get_user_by_id.invalidate((user_id,)) @defer.inlineCallbacks def user_delete_access_tokens(self, user_id, except_token_ids=[]): From f6ebaf4a3255fa298cb0c8a7f23294265038de7b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 May 2016 10:10:06 +0100 Subject: [PATCH 093/134] Run transaction queue on reactor This ensures that any CPU work that happens doesn't block message sending. --- synapse/federation/transaction_queue.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 1928da03b..5787f854d 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -20,6 +20,7 @@ from .persistence import TransactionActions from .units import Transaction from synapse.api.errors import HttpResponseException +from synapse.util.async import run_on_reactor from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext from synapse.util.retryutils import ( @@ -199,6 +200,8 @@ class TransactionQueue(object): @defer.inlineCallbacks @log_function def _attempt_new_transaction(self, destination): + yield run_on_reactor() + # list of (pending_pdu, deferred, order) if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending From 08dfa8eee258a43177b843e6d708c8c98357c6d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 May 2016 10:36:03 +0100 Subject: [PATCH 094/134] Add and use get_domian_from_id --- synapse/api/auth.py | 16 ++++++++-------- synapse/handlers/_base.py | 8 +++----- synapse/handlers/federation.py | 12 ++++-------- synapse/handlers/presence.py | 4 ++-- synapse/storage/roommember.py | 7 ++----- synapse/types.py | 4 ++++ 6 files changed, 23 insertions(+), 28 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9e912fdfb..d3e9837c8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -22,7 +22,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError, EventSizeError -from synapse.types import Requester, RoomID, UserID, EventID +from synapse.types import Requester, UserID, get_domian_from_id from synapse.util.logutils import log_function from synapse.util.logcontext import preserve_context_over_fn from synapse.util.metrics import Measure @@ -91,8 +91,8 @@ class Auth(object): "Room %r does not exist" % (event.room_id,) ) - creating_domain = RoomID.from_string(event.room_id).domain - originating_domain = UserID.from_string(event.sender).domain + creating_domain = get_domian_from_id(event.room_id) + originating_domain = get_domian_from_id(event.sender) if creating_domain != originating_domain: if not self.can_federate(event, auth_events): raise AuthError( @@ -219,7 +219,7 @@ class Auth(object): for event in curr_state.values(): if event.type == EventTypes.Member: try: - if UserID.from_string(event.state_key).domain != host: + if get_domian_from_id(event.state_key) != host: continue except: logger.warn("state_key not user_id: %s", event.state_key) @@ -266,8 +266,8 @@ class Auth(object): target_user_id = event.state_key - creating_domain = RoomID.from_string(event.room_id).domain - target_domain = UserID.from_string(target_user_id).domain + creating_domain = get_domian_from_id(event.room_id) + target_domain = get_domian_from_id(target_user_id) if creating_domain != target_domain: if not self.can_federate(event, auth_events): raise AuthError( @@ -889,8 +889,8 @@ class Auth(object): if user_level >= redact_level: return False - redacter_domain = EventID.from_string(event.event_id).domain - redactee_domain = EventID.from_string(event.redacts).domain + redacter_domain = get_domian_from_id(event.event_id) + redactee_domain = get_domian_from_id(event.redacts) if redacter_domain == redactee_domain: return True diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 13a675b20..287024c1c 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError, SynapseError, AuthError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes -from synapse.types import UserID, RoomAlias, Requester +from synapse.types import UserID, RoomAlias, Requester, get_domian_from_id from synapse.push.action_generator import ActionGenerator from synapse.util.logcontext import PreserveLoggingContext, preserve_fn @@ -296,7 +296,7 @@ class BaseHandler(object): return True for (state_key, membership) in room_members: if ( - UserID.from_string(state_key).domain == self.hs.hostname + self.hs.is_mine_id(state_key) and membership == Membership.JOIN ): return True @@ -421,9 +421,7 @@ class BaseHandler(object): try: if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: - destinations.add( - UserID.from_string(s.state_key).domain - ) + destinations.add(get_domian_from_id(s.state_key)) except SynapseError: logger.warn( "Failed to get destination from event %s", s.event_id diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d95e0b23b..f38c6a871 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -33,7 +33,7 @@ from synapse.util.frozenutils import unfreeze from synapse.crypto.event_signing import ( compute_event_signature, add_hashes_and_signatures, ) -from synapse.types import UserID +from synapse.types import UserID, get_domian_from_id from synapse.events.utils import prune_event @@ -453,7 +453,7 @@ class FederationHandler(BaseHandler): joined_domains = {} for u, d in joined_users: try: - dom = UserID.from_string(u).domain + dom = get_domian_from_id(u) old_d = joined_domains.get(dom) if old_d: joined_domains[dom] = min(d, old_d) @@ -743,9 +743,7 @@ class FederationHandler(BaseHandler): try: if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: - destinations.add( - UserID.from_string(s.state_key).domain - ) + destinations.add(get_domian_from_id(s.state_key)) except: logger.warn( "Failed to get destination from event %s", s.event_id @@ -970,9 +968,7 @@ class FederationHandler(BaseHandler): try: if k[0] == EventTypes.Member: if s.content["membership"] == Membership.LEAVE: - destinations.add( - UserID.from_string(s.state_key).domain - ) + destinations.add(get_domian_from_id(s.state_key)) except: logger.warn( "Failed to get destination from event %s", s.event_id diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 639567953..a8529cce4 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -33,7 +33,7 @@ from synapse.util.logcontext import preserve_fn from synapse.util.logutils import log_function from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer -from synapse.types import UserID +from synapse.types import UserID, get_domian_from_id import synapse.metrics from ._base import BaseHandler @@ -440,7 +440,7 @@ class PresenceHandler(BaseHandler): if not local_states: continue - host = UserID.from_string(user_id).domain + host = get_domian_from_id(user_id) hosts_to_states.setdefault(host, []).extend(local_states) # TODO: de-dup hosts_to_states, as a single host might have multiple diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 08a54cbdd..9d6bfd524 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -21,7 +21,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached, cachedInlineCallbacks from synapse.api.constants import Membership -from synapse.types import UserID +from synapse.types import get_domian_from_id import logging @@ -273,10 +273,7 @@ class RoomMemberStore(SQLBaseStore): room_id, membership=Membership.JOIN ) - joined_domains = set( - UserID.from_string(r["user_id"]).domain - for r in rows - ) + joined_domains = set(get_domian_from_id(r["user_id"]) for r in rows) return joined_domains diff --git a/synapse/types.py b/synapse/types.py index 5b166835b..42fd9c720 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -21,6 +21,10 @@ from collections import namedtuple Requester = namedtuple("Requester", ["user", "access_token_id", "is_guest"]) +def get_domian_from_id(string): + return string.split(":", 1)[1] + + class DomainSpecificString( namedtuple("DomainSpecificString", ("localpart", "domain")) ): From 6b45ffd2d1190e1c34fd181dacdf17b4a18f76e6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 May 2016 20:16:56 +0200 Subject: [PATCH 095/134] Switch from CSS to Table layout for HTML mails so they work in Outlook aka Word Remove templates-vector and theme templates with variables instead Switch to matrix.to URLs by default for links --- res/templates/mail-Vector.css | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 res/templates/mail-Vector.css diff --git a/res/templates/mail-Vector.css b/res/templates/mail-Vector.css new file mode 100644 index 000000000..6a3e36eda --- /dev/null +++ b/res/templates/mail-Vector.css @@ -0,0 +1,7 @@ +.header { + border-bottom: 4px solid #e4f7ed ! important; +} + +.notif_link a, .footer a { + color: #76CFA6 ! important; +} From 55996088876664280e7a9232ca21b1875db6c413 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 10 May 2016 00:14:48 +0200 Subject: [PATCH 096/134] Switch from CSS to Table layout for HTML mails so they work in Outlook aka Word Remove templates-vector and theme templates with variables instead Switch to matrix.to URLs by default for links --- res/templates-vector/mail.css | 115 ---------------------- res/templates-vector/notif.html | 34 ------- res/templates-vector/notif.txt | 12 --- res/templates-vector/notif_mail.html | 25 ----- res/templates-vector/notif_mail.txt | 10 -- res/templates-vector/room.html | 27 ------ res/templates-vector/room.txt | 9 -- res/templates/mail.css | 137 +++++++++++++++++++++++++-- res/templates/notif.html | 50 ++++++---- res/templates/notif.txt | 2 +- res/templates/notif_mail.html | 42 +++++--- res/templates/room.html | 54 ++++++----- synapse/push/mailer.py | 25 +++-- 13 files changed, 240 insertions(+), 302 deletions(-) delete mode 100644 res/templates-vector/mail.css delete mode 100644 res/templates-vector/notif.html delete mode 100644 res/templates-vector/notif.txt delete mode 100644 res/templates-vector/notif_mail.html delete mode 100644 res/templates-vector/notif_mail.txt delete mode 100644 res/templates-vector/room.html delete mode 100644 res/templates-vector/room.txt diff --git a/res/templates-vector/mail.css b/res/templates-vector/mail.css deleted file mode 100644 index 103c5ff8c..000000000 --- a/res/templates-vector/mail.css +++ /dev/null @@ -1,115 +0,0 @@ -body { - margin: 0px; -} - -#page { - font-family: 'Open Sans', Helvetica, Arial, Sans-Serif; - font-size: 12pt; - - margin: auto; - max-width: 640px; - padding: 20px; -} - -.header { - height: 87px; - border-bottom: 4px solid #e4f7ed; -} - -.logo { - float: right; - margin-left: 20px; -} - -.salutation { - padding-top: 10px; - font-weight: bold; -} - -.summarytext { -} - -.room_header { - padding-top: 38px; - padding-bottom: 10px; - border-bottom: 1px solid #e5e5e5; -} - -.room_header h2 { - margin-top: 0px; - margin-left: 75px; - font-size: 20px; -} - -.room_avatar { - float: left; - margin-top: -8px; -} - -.room_avatar img { - width: 48px; - height: 48px; - object-fit: cover; - border-radius: 24px; - margin-left: 7px; -} - -.room_content { - clear: left; -} - -.notif { - border-bottom: 1px solid #e5e5e5; - margin-top: 16px; - padding-bottom: 16px; -} - -.historical { - opacity: 0.3; -} - -.message { - position: relative; - margin-bottom: 10px; -} - -.sender_avatar { - width: 32px; - height: 32px; - border-radius: 16px; - position: absolute; - margin-left: 14px; - margin-top: -2px; -} - -.sender_name { - margin-left: 75px; - display: inline; - font-size: 13px; - color: #a2a2a2; -} - -.message_time { - float: right; - font-size: 11px; - color: #a2a2a2; -} - -.message_body { - margin-left: 75px; -} - -.notif_link { - margin-left: 75px; - font-weight: bold; -} - -.notif_link a, .footer a { - color: #76CFA6; - text-decoration: none; -} - -.footer { - margin-top: 20px; - text-align: center; -} diff --git a/res/templates-vector/notif.html b/res/templates-vector/notif.html deleted file mode 100644 index 97ea42501..000000000 --- a/res/templates-vector/notif.html +++ /dev/null @@ -1,34 +0,0 @@ -
-
- {% for message in notif.messages %} -
- {% if message.sender_avatar_url %} - - {% else %} - {% if message.sender_hash % 3 == 0 %} - - {% elif message.sender_hash % 3 == 1 %} - - {% else %} - - {% endif %} - - {% endif %} -
{{ message.sender_name }}
-
{{ message.ts|format_ts("%H:%M") }}
-
- {% if message.msgtype == "m.text" %} - {{ message.body_text_html }} - {% elif message.msgtype == "m.image" %} - - {% elif message.msgtype == "m.file" %} - {{ message.body_text_plain }} - {% endif %} -
-
- {% endfor %} -
-
- View {{ room.title }} -
-
diff --git a/res/templates-vector/notif.txt b/res/templates-vector/notif.txt deleted file mode 100644 index b515f394c..000000000 --- a/res/templates-vector/notif.txt +++ /dev/null @@ -1,12 +0,0 @@ -{% for message in notif.messages %} -{{ message.sender_name }} ({{ message.ts|format_ts("%H:%M") }}) -{% if message.msgtype == "m.text" %} -{{ message.body_text_plain }} -{% elif message.msgtype == "m.image" %} -{{ message.body_text_plain }} -{% elif message.msgtype == "m.file" %} -{{ message.body_text_plain }} -{% endif %} -{% endfor %} - -View at {{ notif.link }} diff --git a/res/templates-vector/notif_mail.html b/res/templates-vector/notif_mail.html deleted file mode 100644 index c49cc6654..000000000 --- a/res/templates-vector/notif_mail.html +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - -
-
- -
Hi {{ user_display_name }},
-
{{ summary_text }}
-
-
- {% for room in rooms %} - {% include 'room.html' with context %} - {% endfor %} -
-
- Unsubscribe -
-
- - diff --git a/res/templates-vector/notif_mail.txt b/res/templates-vector/notif_mail.txt deleted file mode 100644 index 24843042a..000000000 --- a/res/templates-vector/notif_mail.txt +++ /dev/null @@ -1,10 +0,0 @@ -Hi {{ user_display_name }}, - -{{ summary_text }} - -{% for room in rooms %} -{% include 'room.txt' with context %} -{% endfor %} - -You can disable these notifications at {{ unsubscribe_link }} - diff --git a/res/templates-vector/room.html b/res/templates-vector/room.html deleted file mode 100644 index ab1149257..000000000 --- a/res/templates-vector/room.html +++ /dev/null @@ -1,27 +0,0 @@ -
-
-
- {% if room.avatar_url %} - - {% else %} - {% if room.hash % 3 == 0 %} - - {% elif room.hash % 3 == 1 %} - - {% else %} - - {% endif %} - {% endif %} -
-

{{ room.title }}

-
-
- {% if room.invite %} - Join the conversation. - {% else %} - {% for notif in room.notifs %} - {% include 'notif.html' with context %} - {% endfor %} - {% endif %} -
-
diff --git a/res/templates-vector/room.txt b/res/templates-vector/room.txt deleted file mode 100644 index 84648c710..000000000 --- a/res/templates-vector/room.txt +++ /dev/null @@ -1,9 +0,0 @@ -{{ room.title }} - -{% if room.invite %} - You've been invited, join at {{ room.link }} -{% else %} - {% for notif in room.notifs %} - {% include 'notif.txt' with context %} - {% endfor %} -{% endif %} diff --git a/res/templates/mail.css b/res/templates/mail.css index 61953ba51..b02f509e5 100644 --- a/res/templates/mail.css +++ b/res/templates/mail.css @@ -1,21 +1,146 @@ +body { + margin: 0px; +} + +#page { + font-family: 'Open Sans', Helvetica, Arial, Sans-Serif; + font-color: #454545; + font-size: 12pt; + width: 100%; + padding: 20px; +} + +#inner { + width: 640px; +} + +.header { + width: 100%; + height: 87px; + color: #454545; + border-bottom: 4px solid #e5e5e5; +} + +.logo { + text-align: right; + margin-left: 20px; +} + +.salutation { + padding-top: 10px; + font-weight: bold; +} + +.summarytext { +} + +.room { + width: 100%; + color: #454545; + border-bottom: 1px solid #e5e5e5; +} + +.room_header td { + padding-top: 38px; + padding-bottom: 10px; + border-bottom: 1px solid #e5e5e5; +} + +.room_name { + vertical-align: middle; + font-size: 18px; + font-weight: bold; +} + +.room_header h2 { + margin-top: 0px; + margin-left: 75px; + font-size: 20px; +} + .room_avatar { + width: 56px; + line-height: 0px; + text-align: center; + vertical-align: middle; +} + +.room_avatar img { width: 48px; height: 48px; - float: left; + object-fit: cover; + border-radius: 24px; } -.room_content { - clear: left; +.notif { + border-bottom: 1px solid #e5e5e5; + margin-top: 16px; + padding-bottom: 16px; } -.historical { - color: #888; +.historical_message .sender_avatar { + opacity: 0.3; +} + +/* spell out opacity and historical_message class names for Outlook aka Word */ +.historical_message .sender_name { + color: #e3e3e3; +} + +.historical_message .message_time { + color: #e3e3e3; +} + +.historical_message .message_body { + color: #c7c7c7; +} + +.historical_message td, +.message td { + padding-top: 10px; +} + +.sender_avatar { + width: 56px; + text-align: center; + vertical-align: top; +} + +.sender_avatar img { + margin-top: -2px; + width: 32px; + height: 32px; + border-radius: 16px; } .sender_name { display: inline; + font-size: 13px; + color: #a2a2a2; } .message_time { - display: inline; + text-align: right; + width: 100px; + font-size: 11px; + color: #a2a2a2; } + +.message_body { +} + +.notif_link td { + padding-top: 10px; + padding-bottom: 10px; + font-weight: bold; +} + +.notif_link a, .footer a { + color: #454545; + text-decoration: none; +} + +.footer { + margin-top: 20px; + text-align: center; +} \ No newline at end of file diff --git a/res/templates/notif.html b/res/templates/notif.html index 3112df970..92e80352f 100644 --- a/res/templates/notif.html +++ b/res/templates/notif.html @@ -1,9 +1,9 @@ -
-
- {% for message in notif.messages %} -
+{% for message in notif.messages %} + + + {% if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %} {% if message.sender_avatar_url %} - + {% else %} {% if message.sender_hash % 3 == 0 %} @@ -14,21 +14,29 @@ {% endif %} {% endif %} + {% endif %} + + + {% if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %}
{{ message.sender_name }}
-
{{ message.ts|format_ts("%H:%M") }}
-
- {% if message.msgtype == "m.text" %} - {{ message.body_text_html }} - {% elif message.msgtype == "m.image" %} - - {% elif message.msgtype == "m.file" %} - {{ message.body_text_plain }} - {% endif %} -
+ {% endif %} +
+ {% if message.msgtype == "m.text" %} + {{ message.body_text_html }} + {% elif message.msgtype == "m.image" %} + + {% elif message.msgtype == "m.file" %} + {{ message.body_text_plain }} + {% endif %}
- {% endfor %} -
-
- View -
-
+ + {{ message.ts|format_ts("%H:%M") }} + +{% endfor %} + + + + View {{ room.title }} + + + diff --git a/res/templates/notif.txt b/res/templates/notif.txt index b515f394c..a3ddac80c 100644 --- a/res/templates/notif.txt +++ b/res/templates/notif.txt @@ -9,4 +9,4 @@ {% endif %} {% endfor %} -View at {{ notif.link }} +View {{ room.title }} at {{ notif.link }} diff --git a/res/templates/notif_mail.html b/res/templates/notif_mail.html index cc3573e65..4034f3c60 100644 --- a/res/templates/notif_mail.html +++ b/res/templates/notif_mail.html @@ -3,20 +3,38 @@ -
-
Hi {{ user_display_name }},
-
{{ summary_text }}
-
- {% for room in rooms %} - {% include 'room.html' with context %} - {% endfor %} -
-
- Unsubscribe -
-
+ + + + + + +
+ + + + + +
+
Hi {{ user_display_name }},
+
{{ summary_text }}
+
+ {% for room in rooms %} + {% include 'room.html' with context %} + {% endfor %} +
+ Unsubscribe +
+
diff --git a/res/templates/room.html b/res/templates/room.html index 3c0a4607b..723c222d2 100644 --- a/res/templates/room.html +++ b/res/templates/room.html @@ -1,25 +1,33 @@ -
-
- {% if room.avatar_url %} - - {% else %} - {% if room.hash % 3 == 0 %} - - {% elif room.hash % 3 == 1 %} - + + + + + + {% if room.invite %} + + + + + + {% else %} + {% for notif in room.notifs %} + {% include 'notif.html' with context %} + {% endfor %} + {% endif %} +
+ {% if room.avatar_url %} + {% else %} - + {% if room.hash % 3 == 0 %} + + {% elif room.hash % 3 == 1 %} + + {% else %} + + {% endif %} {% endif %} - {% endif %} - -

{{ room.title }}

-
- {% if room.invite %} - Join the conversation. - {% else %} - {% for notif in room.notifs %} - {% include 'notif.html' with context %} - {% endfor %} - {% endif %} -
- + 		+ {{ room.title }} +
+ Join the conversation. +
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index f60ac94a8..3c38321fd 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -146,6 +146,7 @@ class Mailer(object): "user_display_name": user_display_name, "unsubscribe_link": self.make_unsubscribe_link(), "summary_text": summary_text, + "app_name": self.app_name, "rooms": rooms, } @@ -164,6 +165,9 @@ class Mailer(object): multipart_msg.attach(text_part) multipart_msg.attach(html_part) + logger.info("Sending email push notification to %s" % email_address) + #logger.debug(html_text) + yield sendmail( self.hs.config.email_smtp_host, raw_from, raw_to, multipart_msg.as_string(), @@ -367,19 +371,26 @@ class Mailer(object): } def make_room_link(self, room_id): - # XXX: matrix.to # need /beta for Universal Links to work on iOS - return "https://vector.im/beta/#/room/%s" % (room_id,) + if self.app_name == "Vector": + return "https://vector.im/beta/#/room/%s" % (room_id,) + else: + return "https://matrix.to/#/room/%s" % (room_id,) def make_notif_link(self, notif): - # XXX: matrix.to # need /beta for Universal Links to work on iOS - return "https://vector.im/beta/#/room/%s/%s" % ( - notif['room_id'], notif['event_id'] - ) + if self.app_name == "Vector": + return "https://vector.im/beta/#/room/%s/%s" % ( + notif['room_id'], notif['event_id'] + ) + else: + return "https://matrix.to/#/room/%s/%s" % ( + notif['room_id'], notif['event_id'] + ) def make_unsubscribe_link(self): - return "https://vector.im/#/settings" # XXX: matrix.to + # XXX: matrix.to + return "https://vector.im/#/settings" def mxc_to_http_filter(self, value, width, height, resize_method="crop"): if value[0:6] != "mxc://": From e04b1d6b0a6b6f2934e59e73a605e99da6ca9f5e Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 14:23:16 +0200 Subject: [PATCH 097/134] Make pep8 happy --- synapse/push/mailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 3c38321fd..7031fa6d5 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -166,7 +166,7 @@ class Mailer(object): multipart_msg.attach(html_part) logger.info("Sending email push notification to %s" % email_address) - #logger.debug(html_text) + # logger.debug(html_text) yield sendmail( self.hs.config.email_smtp_host, From 94040b0798a7e4db88e75485906fd8a2b31b117c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 14:34:53 +0200 Subject: [PATCH 098/134] Add config option to not send email notifs for new users --- synapse/config/emailconfig.py | 4 ++++ synapse/rest/client/v2_alpha/register.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index d6f4f83a1..b239619c9 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -65,6 +65,9 @@ class EmailConfig(Config): self.email_template_dir = email_config["template_dir"] self.email_notif_template_html = email_config["notif_template_html"] self.email_notif_template_text = email_config["notif_template_text"] + self.email_notifs_for_new_users = email_config.get( + "notif_for_new_users", True + ) if "app_name" in email_config: self.email_app_name = email_config["app_name"] else: @@ -91,4 +94,5 @@ class EmailConfig(Config): # template_dir: res/templates # notif_template_html: notif_mail.html # notif_template_text: notif_mail.txt + # notif_for_new_users: True """ diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 883b1c129..ad0438355 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -219,7 +219,10 @@ class RegisterRestServlet(RestServlet): # if email notifications are enabled (so people don't start # getting mail spam where they weren't before if email # notifs are set up on a home server) - if self.hs.config.email_enable_notifs: + if ( + self.hs.config.email_enable_notifs and + self.hs.config.email_notifs_for_new_users + ): # Pull the ID of the access token back out of the db # It would really make more sense for this to be passed # up when the access token is saved, but that's quite an From c00b484eff179257f34eeb48be98bb9435598f5e Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 14:39:16 +0200 Subject: [PATCH 099/134] More consistent config naming --- synapse/config/emailconfig.py | 2 +- synapse/rest/client/v2_alpha/register.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index b239619c9..90bdd08f0 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -65,7 +65,7 @@ class EmailConfig(Config): self.email_template_dir = email_config["template_dir"] self.email_notif_template_html = email_config["notif_template_html"] self.email_notif_template_text = email_config["notif_template_text"] - self.email_notifs_for_new_users = email_config.get( + self.email_notif_for_new_users = email_config.get( "notif_for_new_users", True ) if "app_name" in email_config: diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index ad0438355..1ecc02d94 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -221,7 +221,7 @@ class RegisterRestServlet(RestServlet): # notifs are set up on a home server) if ( self.hs.config.email_enable_notifs and - self.hs.config.email_notifs_for_new_users + self.hs.config.email_notif_for_new_users ): # Pull the ID of the access token back out of the db # It would really make more sense for this to be passed From 3b1930e8ecd571b6839ebb90c636dc63539d34a1 Mon Sep 17 00:00:00 2001 From: Matrix Date: Tue, 10 May 2016 16:42:37 +0100 Subject: [PATCH 100/134] unbreak schema --- synapse/storage/schema/delta/{31 => 32}/events.sql | 0 synapse/storage/schema/delta/{31 => 32}/pusher_throttle.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename synapse/storage/schema/delta/{31 => 32}/events.sql (100%) rename synapse/storage/schema/delta/{31 => 32}/pusher_throttle.sql (100%) diff --git a/synapse/storage/schema/delta/31/events.sql b/synapse/storage/schema/delta/32/events.sql similarity index 100% rename from synapse/storage/schema/delta/31/events.sql rename to synapse/storage/schema/delta/32/events.sql diff --git a/synapse/storage/schema/delta/31/pusher_throttle.sql b/synapse/storage/schema/delta/32/pusher_throttle.sql similarity index 100% rename from synapse/storage/schema/delta/31/pusher_throttle.sql rename to synapse/storage/schema/delta/32/pusher_throttle.sql From d46b18a00f0811489299b4835482af2a71dcaf55 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 18:27:06 +0200 Subject: [PATCH 101/134] Pass through _get_event_txn --- synapse/crypto/context_factory.py | 2 +- synapse/replication/slave/storage/events.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index aad4752fe..4fb32d210 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -43,7 +43,7 @@ class ServerContextFactory(ssl.ContextFactory): context.use_privatekey(config.tls_private_key) context.load_tmp_dh(config.tls_dh_params_path) - context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH") + context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH:HIGH") def getContext(self): return self._context diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py index 86f00b6ff..56e8b9d90 100644 --- a/synapse/replication/slave/storage/events.py +++ b/synapse/replication/slave/storage/events.py @@ -104,6 +104,7 @@ class SlavedEventStore(BaseSlavedStore): _invalidate_get_event_cache = DataStore._invalidate_get_event_cache.__func__ _parse_events_txn = DataStore._parse_events_txn.__func__ _get_events_txn = DataStore._get_events_txn.__func__ + _get_event_txn = DataStore._get_event_txn.__func__ _enqueue_events = DataStore._enqueue_events.__func__ _do_fetch = DataStore._do_fetch.__func__ _fetch_events_txn = DataStore._fetch_events_txn.__func__ From 5f46be19a78f1811eee9a7f8fd9fde70563fdfe9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 18:43:40 +0200 Subject: [PATCH 102/134] Pass through get_events to pusher too --- synapse/replication/slave/storage/events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py index 56e8b9d90..7ba7a6f6e 100644 --- a/synapse/replication/slave/storage/events.py +++ b/synapse/replication/slave/storage/events.py @@ -83,6 +83,7 @@ class SlavedEventStore(BaseSlavedStore): DataStore.get_push_action_users_in_range.__func__ ) get_event = DataStore.get_event.__func__ + get_events = DataStore.get_events.__func__ get_current_state = DataStore.get_current_state.__func__ get_current_state_for_key = DataStore.get_current_state_for_key.__func__ get_rooms_for_user_where_membership_is = ( From f28643cea9ad41cd199b5b4a53e32a505dbfd961 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 18:44:32 +0200 Subject: [PATCH 103/134] Uncommit accidentally commited edit to cipher list --- synapse/crypto/context_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 4fb32d210..aad4752fe 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -43,7 +43,7 @@ class ServerContextFactory(ssl.ContextFactory): context.use_privatekey(config.tls_private_key) context.load_tmp_dh(config.tls_dh_params_path) - context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH:HIGH") + context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH") def getContext(self): return self._context From 0c4ccdcb83308396670b896d7b9f6c40c1e94660 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 18:51:14 +0200 Subject: [PATCH 104/134] Also pass through get_profile_displayname --- synapse/app/pusher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 89c8d5c7c..9fec6869f 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -123,6 +123,10 @@ class PusherSlaveStore( DataStore.get_time_of_last_push_action_before.__func__ ) + get_profile_displayname = ( + DataStore.get_profile_displayname.__func__ + ) + class PusherServer(HomeServer): From 3367e65476f224b12a94277552787650ab57d984 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 18:53:15 +0200 Subject: [PATCH 105/134] Pass through get_state_groups --- synapse/app/pusher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 9fec6869f..6f5e7a2e3 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -127,6 +127,10 @@ class PusherSlaveStore( DataStore.get_profile_displayname.__func__ ) + get_state_groups = ( + DataStore.get_state_groups.__func__ + ) + class PusherServer(HomeServer): From 35b6e6d2a8639dba4e23372dff0be76c4d9a8936 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 18:56:40 +0200 Subject: [PATCH 106/134] Pass though _get_state_group_for_events --- synapse/app/pusher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 6f5e7a2e3..d59922422 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -131,6 +131,10 @@ class PusherSlaveStore( DataStore.get_state_groups.__func__ ) + _get_state_group_for_events = ( + DataStore._get_state_group_for_events.__func__ + ) + class PusherServer(HomeServer): From 89b5ef7c4be824af4314497ad1338c2ac0e66ed8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 19:05:22 +0200 Subject: [PATCH 107/134] Cached functions must be accessed through the dict --- synapse/app/pusher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index d59922422..64371aaba 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -23,6 +23,7 @@ from synapse.config.logger import LoggingConfig from synapse.config.emailconfig import EmailConfig from synapse.http.site import SynapseSite from synapse.metrics.resource import MetricsResource, METRICS_PREFIX +from synapse.storage.events import EventsStore from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.pushers import SlavedPusherStore from synapse.replication.slave.storage.receipts import SlavedReceiptsStore @@ -132,7 +133,7 @@ class PusherSlaveStore( ) _get_state_group_for_events = ( - DataStore._get_state_group_for_events.__func__ + EventsStore.__dict__["_get_state_group_for_events"] ) From 90afc07f39cdb01dfd33a13ae5bdfb9d89505331 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 19:10:46 +0200 Subject: [PATCH 108/134] StateStore, not EventsStore --- synapse/app/pusher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 64371aaba..b0a42fe67 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -23,7 +23,7 @@ from synapse.config.logger import LoggingConfig from synapse.config.emailconfig import EmailConfig from synapse.http.site import SynapseSite from synapse.metrics.resource import MetricsResource, METRICS_PREFIX -from synapse.storage.events import EventsStore +from synapse.storage.state import StateStore from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.pushers import SlavedPusherStore from synapse.replication.slave.storage.receipts import SlavedReceiptsStore @@ -133,7 +133,7 @@ class PusherSlaveStore( ) _get_state_group_for_events = ( - EventsStore.__dict__["_get_state_group_for_events"] + StateStore.__dict__["_get_state_group_for_events"] ) From ae1af262f6ad32ffeb4f17b19a43599b9bf9e129 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2016 19:18:03 +0200 Subject: [PATCH 109/134] Pass through _get_state_group_for_events --- synapse/app/pusher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index b0a42fe67..8e9c0e196 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -136,6 +136,10 @@ class PusherSlaveStore( StateStore.__dict__["_get_state_group_for_events"] ) + _get_state_group_for_event = ( + StateStore.__dict__["_get_state_group_for_event"] + ) + class PusherServer(HomeServer): From 30057b1e154a0fdf1f778aa952c2f7c88656004e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 May 2016 09:09:20 +0100 Subject: [PATCH 110/134] Move _create_new_client_event and handle_new_client_event out of base handler --- synapse/handlers/_base.py | 198 +------------------------------ synapse/handlers/federation.py | 17 ++- synapse/handlers/message.py | 199 +++++++++++++++++++++++++++++++- synapse/handlers/room_member.py | 4 +- 4 files changed, 214 insertions(+), 204 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 2c811906d..ac716a811 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -15,13 +15,11 @@ from twisted.internet import defer -from synapse.api.errors import LimitExceededError, SynapseError, AuthError -from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.api.errors import LimitExceededError from synapse.api.constants import Membership, EventTypes -from synapse.types import UserID, RoomAlias, Requester, get_domian_from_id -from synapse.push.action_generator import ActionGenerator +from synapse.types import UserID, Requester -from synapse.util.logcontext import PreserveLoggingContext, preserve_fn +from synapse.util.logcontext import preserve_fn import logging @@ -65,7 +63,6 @@ class BaseHandler(object): self.clock = hs.get_clock() self.hs = hs - self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.event_builder_factory = hs.get_event_builder_factory() @@ -248,56 +245,6 @@ class BaseHandler(object): retry_after_ms=int(1000 * (time_allowed - time_now)), ) - @defer.inlineCallbacks - def _create_new_client_event(self, builder, prev_event_ids=None): - if prev_event_ids: - prev_events = yield self.store.add_event_hashes(prev_event_ids) - prev_max_depth = yield self.store.get_max_depth_of_events(prev_event_ids) - depth = prev_max_depth + 1 - else: - latest_ret = yield self.store.get_latest_event_ids_and_hashes_in_room( - builder.room_id, - ) - - if latest_ret: - depth = max([d for _, _, d in latest_ret]) + 1 - else: - depth = 1 - - prev_events = [ - (event_id, prev_hashes) - for event_id, prev_hashes, _ in latest_ret - ] - - builder.prev_events = prev_events - builder.depth = depth - - state_handler = self.state_handler - - context = yield state_handler.compute_event_context(builder) - - if builder.is_state(): - builder.prev_state = yield self.store.add_event_hashes( - context.prev_state_events - ) - - yield self.auth.add_auth_events(builder, context) - - add_hashes_and_signatures( - builder, self.server_name, self.signing_key - ) - - event = builder.build() - - logger.debug( - "Created event %s with current state: %s", - event.event_id, context.current_state, - ) - - defer.returnValue( - (event, context,) - ) - def is_host_in_room(self, current_state): room_members = [ (state_key, event.membership) @@ -318,145 +265,6 @@ class BaseHandler(object): return True return False - @defer.inlineCallbacks - def handle_new_client_event( - self, - requester, - event, - context, - ratelimit=True, - extra_users=[] - ): - # We now need to go and hit out to wherever we need to hit out to. - - if ratelimit: - self.ratelimit(requester) - - try: - self.auth.check(event, auth_events=context.current_state) - except AuthError as err: - logger.warn("Denying new event %r because %s", event, err) - raise err - - yield self.maybe_kick_guest_users(event, context.current_state.values()) - - if event.type == EventTypes.CanonicalAlias: - # Check the alias is acually valid (at this time at least) - room_alias_str = event.content.get("alias", None) - if room_alias_str: - room_alias = RoomAlias.from_string(room_alias_str) - directory_handler = self.hs.get_handlers().directory_handler - mapping = yield directory_handler.get_association(room_alias) - - if mapping["room_id"] != event.room_id: - raise SynapseError( - 400, - "Room alias %s does not point to the room" % ( - room_alias_str, - ) - ) - - federation_handler = self.hs.get_handlers().federation_handler - - if event.type == EventTypes.Member: - if event.content["membership"] == Membership.INVITE: - def is_inviter_member_event(e): - return ( - e.type == EventTypes.Member and - e.sender == event.sender - ) - - event.unsigned["invite_room_state"] = [ - { - "type": e.type, - "state_key": e.state_key, - "content": e.content, - "sender": e.sender, - } - for k, e in context.current_state.items() - if e.type in self.hs.config.room_invite_state_types - or is_inviter_member_event(e) - ] - - invitee = UserID.from_string(event.state_key) - if not self.hs.is_mine(invitee): - # TODO: Can we add signature from remote server in a nicer - # way? If we have been invited by a remote server, we need - # to get them to sign the event. - - returned_invite = yield federation_handler.send_invite( - invitee.domain, - event, - ) - - event.unsigned.pop("room_state", None) - - # TODO: Make sure the signatures actually are correct. - event.signatures.update( - returned_invite.signatures - ) - - if event.type == EventTypes.Redaction: - if self.auth.check_redaction(event, auth_events=context.current_state): - original_event = yield self.store.get_event( - event.redacts, - check_redacted=False, - get_prev_content=False, - allow_rejected=False, - allow_none=False - ) - if event.user_id != original_event.user_id: - raise AuthError( - 403, - "You don't have permission to redact events" - ) - - if event.type == EventTypes.Create and context.current_state: - raise AuthError( - 403, - "Changing the room create event is forbidden", - ) - - action_generator = ActionGenerator(self.hs) - yield action_generator.handle_push_actions_for_event( - event, context, self - ) - - (event_stream_id, max_stream_id) = yield self.store.persist_event( - event, context=context - ) - - # this intentionally does not yield: we don't care about the result - # and don't need to wait for it. - preserve_fn(self.hs.get_pusherpool().on_new_notifications)( - event_stream_id, max_stream_id - ) - - destinations = set() - for k, s in context.current_state.items(): - try: - if k[0] == EventTypes.Member: - if s.content["membership"] == Membership.JOIN: - destinations.add(get_domian_from_id(s.state_key)) - except SynapseError: - logger.warn( - "Failed to get destination from event %s", s.event_id - ) - - with PreserveLoggingContext(): - # Don't block waiting on waking up all the listeners. - self.notifier.on_new_room_event( - event, event_stream_id, max_stream_id, - extra_users=extra_users - ) - - # If invite, remove room_state from unsigned before sending. - event.unsigned.pop("invite_room_state", None) - - federation_handler.handle_new_event( - event, destinations=destinations, - ) - @defer.inlineCallbacks def maybe_kick_guest_users(self, event, current_state): # Technically this function invalidates current_state by changing it. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f38c6a871..4a65b246e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -682,7 +682,8 @@ class FederationHandler(BaseHandler): }) try: - event, context = yield self._create_new_client_event( + message_handler = self.hs.get_handlers().message_handler + event, context = yield message_handler._create_new_client_event( builder=builder, ) except AuthError as e: @@ -913,7 +914,8 @@ class FederationHandler(BaseHandler): "state_key": user_id, }) - event, context = yield self._create_new_client_event( + message_handler = self.hs.get_handlers().message_handler + event, context = yield message_handler._create_new_client_event( builder=builder, ) @@ -1688,7 +1690,10 @@ class FederationHandler(BaseHandler): if (yield self.auth.check_host_in_room(room_id, self.hs.hostname)): builder = self.event_builder_factory.new(event_dict) EventValidator().validate_new(builder) - event, context = yield self._create_new_client_event(builder=builder) + message_handler = self.hs.get_handlers().message_handler + event, context = yield message_handler._create_new_client_event( + builder=builder + ) event, context = yield self.add_display_name_to_third_party_invite( event_dict, event, context @@ -1716,7 +1721,8 @@ class FederationHandler(BaseHandler): def on_exchange_third_party_invite_request(self, origin, room_id, event_dict): builder = self.event_builder_factory.new(event_dict) - event, context = yield self._create_new_client_event( + message_handler = self.hs.get_handlers().message_handler + event, context = yield message_handler._create_new_client_event( builder=builder, ) @@ -1755,7 +1761,8 @@ class FederationHandler(BaseHandler): event_dict["content"]["third_party_invite"]["display_name"] = display_name builder = self.event_builder_factory.new(event_dict) EventValidator().validate_new(builder) - event, context = yield self._create_new_client_event(builder=builder) + message_handler = self.hs.get_handlers().message_handler + event, context = yield message_handler._create_new_client_event(builder=builder) defer.returnValue((event, context)) @defer.inlineCallbacks diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7d9e3cf36..45d3d47fc 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -17,13 +17,18 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.api.errors import AuthError, Codes, SynapseError -from synapse.streams.config import PaginationConfig +from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator +from synapse.push.action_generator import ActionGenerator +from synapse.streams.config import PaginationConfig +from synapse.types import ( + UserID, RoomAlias, RoomStreamToken, StreamToken, get_domian_from_id +) from synapse.util import unwrapFirstError from synapse.util.async import concurrently_execute from synapse.util.caches.snapshot_cache import SnapshotCache -from synapse.types import UserID, RoomStreamToken, StreamToken +from synapse.util.logcontext import PreserveLoggingContext, preserve_fn from ._base import BaseHandler @@ -43,6 +48,7 @@ class MessageHandler(BaseHandler): self.clock = hs.get_clock() self.validator = EventValidator() self.snapshot_cache = SnapshotCache() + self.signing_key = hs.config.signing_key[0] @defer.inlineCallbacks def get_messages(self, requester, room_id=None, pagin_config=None, @@ -724,3 +730,192 @@ class MessageHandler(BaseHandler): ret["membership"] = membership defer.returnValue(ret) + + @defer.inlineCallbacks + def _create_new_client_event(self, builder, prev_event_ids=None): + if prev_event_ids: + prev_events = yield self.store.add_event_hashes(prev_event_ids) + prev_max_depth = yield self.store.get_max_depth_of_events(prev_event_ids) + depth = prev_max_depth + 1 + else: + latest_ret = yield self.store.get_latest_event_ids_and_hashes_in_room( + builder.room_id, + ) + + if latest_ret: + depth = max([d for _, _, d in latest_ret]) + 1 + else: + depth = 1 + + prev_events = [ + (event_id, prev_hashes) + for event_id, prev_hashes, _ in latest_ret + ] + + builder.prev_events = prev_events + builder.depth = depth + + state_handler = self.state_handler + + context = yield state_handler.compute_event_context(builder) + + if builder.is_state(): + builder.prev_state = yield self.store.add_event_hashes( + context.prev_state_events + ) + + yield self.auth.add_auth_events(builder, context) + + add_hashes_and_signatures( + builder, self.server_name, self.signing_key + ) + + event = builder.build() + + logger.debug( + "Created event %s with current state: %s", + event.event_id, context.current_state, + ) + + defer.returnValue( + (event, context,) + ) + + @defer.inlineCallbacks + def handle_new_client_event( + self, + requester, + event, + context, + ratelimit=True, + extra_users=[] + ): + # We now need to go and hit out to wherever we need to hit out to. + + if ratelimit: + self.ratelimit(requester) + + try: + self.auth.check(event, auth_events=context.current_state) + except AuthError as err: + logger.warn("Denying new event %r because %s", event, err) + raise err + + yield self.maybe_kick_guest_users(event, context.current_state.values()) + + if event.type == EventTypes.CanonicalAlias: + # Check the alias is acually valid (at this time at least) + room_alias_str = event.content.get("alias", None) + if room_alias_str: + room_alias = RoomAlias.from_string(room_alias_str) + directory_handler = self.hs.get_handlers().directory_handler + mapping = yield directory_handler.get_association(room_alias) + + if mapping["room_id"] != event.room_id: + raise SynapseError( + 400, + "Room alias %s does not point to the room" % ( + room_alias_str, + ) + ) + + federation_handler = self.hs.get_handlers().federation_handler + + if event.type == EventTypes.Member: + if event.content["membership"] == Membership.INVITE: + def is_inviter_member_event(e): + return ( + e.type == EventTypes.Member and + e.sender == event.sender + ) + + event.unsigned["invite_room_state"] = [ + { + "type": e.type, + "state_key": e.state_key, + "content": e.content, + "sender": e.sender, + } + for k, e in context.current_state.items() + if e.type in self.hs.config.room_invite_state_types + or is_inviter_member_event(e) + ] + + invitee = UserID.from_string(event.state_key) + if not self.hs.is_mine(invitee): + # TODO: Can we add signature from remote server in a nicer + # way? If we have been invited by a remote server, we need + # to get them to sign the event. + + returned_invite = yield federation_handler.send_invite( + invitee.domain, + event, + ) + + event.unsigned.pop("room_state", None) + + # TODO: Make sure the signatures actually are correct. + event.signatures.update( + returned_invite.signatures + ) + + if event.type == EventTypes.Redaction: + if self.auth.check_redaction(event, auth_events=context.current_state): + original_event = yield self.store.get_event( + event.redacts, + check_redacted=False, + get_prev_content=False, + allow_rejected=False, + allow_none=False + ) + if event.user_id != original_event.user_id: + raise AuthError( + 403, + "You don't have permission to redact events" + ) + + if event.type == EventTypes.Create and context.current_state: + raise AuthError( + 403, + "Changing the room create event is forbidden", + ) + + action_generator = ActionGenerator(self.hs) + yield action_generator.handle_push_actions_for_event( + event, context, self + ) + + (event_stream_id, max_stream_id) = yield self.store.persist_event( + event, context=context + ) + + # this intentionally does not yield: we don't care about the result + # and don't need to wait for it. + preserve_fn(self.hs.get_pusherpool().on_new_notifications)( + event_stream_id, max_stream_id + ) + + destinations = set() + for k, s in context.current_state.items(): + try: + if k[0] == EventTypes.Member: + if s.content["membership"] == Membership.JOIN: + destinations.add(get_domian_from_id(s.state_key)) + except SynapseError: + logger.warn( + "Failed to get destination from event %s", s.event_id + ) + + with PreserveLoggingContext(): + # Don't block waiting on waking up all the listeners. + self.notifier.on_new_room_event( + event, event_stream_id, max_stream_id, + extra_users=extra_users + ) + + # If invite, remove room_state from unsigned before sending. + event.unsigned.pop("invite_room_state", None) + + federation_handler.handle_new_event( + event, destinations=destinations, + ) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index ed2cda837..69de145c6 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -113,7 +113,7 @@ class RoomMemberHandler(BaseHandler): prev_event_ids=prev_event_ids, ) - yield self.handle_new_client_event( + yield self.msg_handler.handle_new_client_event( requester, event, context, @@ -357,7 +357,7 @@ class RoomMemberHandler(BaseHandler): # so don't really fit into the general auth process. raise AuthError(403, "Guest access not allowed") - yield self.handle_new_client_event( + yield message_handler.handle_new_client_event( requester, event, context, From 458a4351145e87f1dd48b83da811ce4d8742d8c1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 May 2016 10:35:33 +0100 Subject: [PATCH 111/134] Fix typo --- synapse/handlers/room_member.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 69de145c6..b44e52a51 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -113,7 +113,7 @@ class RoomMemberHandler(BaseHandler): prev_event_ids=prev_event_ids, ) - yield self.msg_handler.handle_new_client_event( + yield msg_handler.handle_new_client_event( requester, event, context, From 1400bb1663657565be9e3051058a3af8ac02de96 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 11 May 2016 12:06:02 +0100 Subject: [PATCH 112/134] Correctly handle NULL password hashes from the database --- synapse/handlers/auth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 61fe56032..6e7d080ec 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -615,4 +615,7 @@ class AuthHandler(BaseHandler): Returns: Whether self.hash(password) == stored_hash (bool). """ - return bcrypt.hashpw(password, stored_hash) == stored_hash + if stored_hash: + return bcrypt.hashpw(password, stored_hash) == stored_hash + else: + return False From 1620578b13fbcdf902f6bef5c15faa98fe871f1c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 May 2016 12:20:57 +0100 Subject: [PATCH 113/134] Shuffle when we get the signing_key attribute. Wait until we sign a message to get the signing key from the homeserver config. This means that the message handler can be created without having a signing key in the config which means that separate processes like the pusher that don't send messages and don't need to sign them can still access the handlers. --- synapse/handlers/message.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 45d3d47fc..f9e2c98f3 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -48,7 +48,6 @@ class MessageHandler(BaseHandler): self.clock = hs.get_clock() self.validator = EventValidator() self.snapshot_cache = SnapshotCache() - self.signing_key = hs.config.signing_key[0] @defer.inlineCallbacks def get_messages(self, requester, room_id=None, pagin_config=None, @@ -766,8 +765,9 @@ class MessageHandler(BaseHandler): yield self.auth.add_auth_events(builder, context) + signing_key = self.hs.config.signing_key[0] add_hashes_and_signatures( - builder, self.server_name, self.signing_key + builder, self.server_name, signing_key ) event = builder.build() From 7e23476814b2cd3c8cd8ef87d2b312fbca400da6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 May 2016 13:42:37 +0100 Subject: [PATCH 114/134] move filter_events_for_client out of base handler --- synapse/handlers/_base.py | 184 -------------------- synapse/handlers/message.py | 18 +- synapse/handlers/room.py | 7 +- synapse/handlers/search.py | 17 +- synapse/handlers/sync.py | 7 +- synapse/notifier.py | 5 +- synapse/push/action_generator.py | 4 +- synapse/push/bulk_push_rule_evaluator.py | 7 +- synapse/push/mailer.py | 6 +- synapse/visibility.py | 210 +++++++++++++++++++++++ 10 files changed, 251 insertions(+), 214 deletions(-) create mode 100644 synapse/visibility.py diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index ac716a811..c904c6c50 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -19,7 +19,6 @@ from synapse.api.errors import LimitExceededError from synapse.api.constants import Membership, EventTypes from synapse.types import UserID, Requester -from synapse.util.logcontext import preserve_fn import logging @@ -27,23 +26,6 @@ import logging logger = logging.getLogger(__name__) -VISIBILITY_PRIORITY = ( - "world_readable", - "shared", - "invited", - "joined", -) - - -MEMBERSHIP_PRIORITY = ( - Membership.JOIN, - Membership.INVITE, - Membership.KNOCK, - Membership.LEAVE, - Membership.BAN, -) - - class BaseHandler(object): """ Common base class for the event handlers. @@ -67,172 +49,6 @@ class BaseHandler(object): self.event_builder_factory = hs.get_event_builder_factory() - @defer.inlineCallbacks - def filter_events_for_clients(self, user_tuples, events, event_id_to_state): - """ Returns dict of user_id -> list of events that user is allowed to - see. - - Args: - user_tuples (str, bool): (user id, is_peeking) for each user to be - checked. is_peeking should be true if: - * the user is not currently a member of the room, and: - * the user has not been a member of the room since the - given events - events ([synapse.events.EventBase]): list of events to filter - """ - forgotten = yield defer.gatherResults([ - preserve_fn(self.store.who_forgot_in_room)( - room_id, - ) - for room_id in frozenset(e.room_id for e in events) - ], consumeErrors=True) - - # Set of membership event_ids that have been forgotten - event_id_forgotten = frozenset( - row["event_id"] for rows in forgotten for row in rows - ) - - ignore_dict_content = yield self.store.get_global_account_data_by_type_for_users( - "m.ignored_user_list", user_ids=[user_id for user_id, _ in user_tuples] - ) - - # FIXME: This will explode if people upload something incorrect. - ignore_dict = { - user_id: frozenset( - content.get("ignored_users", {}).keys() if content else [] - ) - for user_id, content in ignore_dict_content.items() - } - - def allowed(event, user_id, is_peeking, ignore_list): - """ - Args: - event (synapse.events.EventBase): event to check - user_id (str) - is_peeking (bool) - ignore_list (list): list of users to ignore - """ - if not event.is_state() and event.sender in ignore_list: - return False - - state = event_id_to_state[event.event_id] - - # get the room_visibility at the time of the event. - visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None) - if visibility_event: - visibility = visibility_event.content.get("history_visibility", "shared") - else: - visibility = "shared" - - if visibility not in VISIBILITY_PRIORITY: - visibility = "shared" - - # if it was world_readable, it's easy: everyone can read it - if visibility == "world_readable": - return True - - # Always allow history visibility events on boundaries. This is done - # by setting the effective visibility to the least restrictive - # of the old vs new. - if event.type == EventTypes.RoomHistoryVisibility: - prev_content = event.unsigned.get("prev_content", {}) - prev_visibility = prev_content.get("history_visibility", None) - - if prev_visibility not in VISIBILITY_PRIORITY: - prev_visibility = "shared" - - new_priority = VISIBILITY_PRIORITY.index(visibility) - old_priority = VISIBILITY_PRIORITY.index(prev_visibility) - if old_priority < new_priority: - visibility = prev_visibility - - # likewise, if the event is the user's own membership event, use - # the 'most joined' membership - membership = None - if event.type == EventTypes.Member and event.state_key == user_id: - membership = event.content.get("membership", None) - if membership not in MEMBERSHIP_PRIORITY: - membership = "leave" - - prev_content = event.unsigned.get("prev_content", {}) - prev_membership = prev_content.get("membership", None) - if prev_membership not in MEMBERSHIP_PRIORITY: - prev_membership = "leave" - - new_priority = MEMBERSHIP_PRIORITY.index(membership) - old_priority = MEMBERSHIP_PRIORITY.index(prev_membership) - if old_priority < new_priority: - membership = prev_membership - - # otherwise, get the user's membership at the time of the event. - if membership is None: - membership_event = state.get((EventTypes.Member, user_id), None) - if membership_event: - if membership_event.event_id not in event_id_forgotten: - membership = membership_event.membership - - # if the user was a member of the room at the time of the event, - # they can see it. - if membership == Membership.JOIN: - return True - - if visibility == "joined": - # we weren't a member at the time of the event, so we can't - # see this event. - return False - - elif visibility == "invited": - # user can also see the event if they were *invited* at the time - # of the event. - return membership == Membership.INVITE - - else: - # visibility is shared: user can also see the event if they have - # become a member since the event - # - # XXX: if the user has subsequently joined and then left again, - # ideally we would share history up to the point they left. But - # we don't know when they left. - return not is_peeking - - defer.returnValue({ - user_id: [ - event - for event in events - if allowed(event, user_id, is_peeking, ignore_dict.get(user_id, [])) - ] - for user_id, is_peeking in user_tuples - }) - - @defer.inlineCallbacks - def filter_events_for_client(self, user_id, events, is_peeking=False): - """ - Check which events a user is allowed to see - - Args: - user_id(str): user id to be checked - events([synapse.events.EventBase]): list of events to be checked - is_peeking(bool): should be True if: - * the user is not currently a member of the room, and: - * the user has not been a member of the room since the given - events - - Returns: - [synapse.events.EventBase] - """ - types = ( - (EventTypes.RoomHistoryVisibility, ""), - (EventTypes.Member, user_id), - ) - event_id_to_state = yield self.store.get_state_for_events( - frozenset(e.event_id for e in events), - types=types - ) - res = yield self.filter_events_for_clients( - [(user_id, is_peeking)], events, event_id_to_state - ) - defer.returnValue(res.get(user_id, [])) - def ratelimit(self, requester): time_now = self.clock.time() allowed, time_allowed = self.ratelimiter.send_message( diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index f9e2c98f3..13154edb7 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -29,6 +29,7 @@ from synapse.util import unwrapFirstError from synapse.util.async import concurrently_execute from synapse.util.caches.snapshot_cache import SnapshotCache from synapse.util.logcontext import PreserveLoggingContext, preserve_fn +from synapse.visibility import filter_events_for_client from ._base import BaseHandler @@ -128,7 +129,8 @@ class MessageHandler(BaseHandler): "end": next_token.to_string(), }) - events = yield self.filter_events_for_client( + events = yield filter_events_for_client( + self.store, user_id, events, is_peeking=(member_event_id is None), @@ -488,8 +490,8 @@ class MessageHandler(BaseHandler): ] ).addErrback(unwrapFirstError) - messages = yield self.filter_events_for_client( - user_id, messages + messages = yield filter_events_for_client( + self.store, user_id, messages ) start_token = now_token.copy_and_replace("room_key", token[0]) @@ -624,8 +626,8 @@ class MessageHandler(BaseHandler): end_token=stream_token ) - messages = yield self.filter_events_for_client( - user_id, messages, is_peeking=is_peeking + messages = yield filter_events_for_client( + self.store, user_id, messages, is_peeking=is_peeking ) start_token = StreamToken.START.copy_and_replace("room_key", token[0]) @@ -705,8 +707,8 @@ class MessageHandler(BaseHandler): consumeErrors=True, ).addErrback(unwrapFirstError) - messages = yield self.filter_events_for_client( - user_id, messages, is_peeking=is_peeking, + messages = yield filter_events_for_client( + self.store, user_id, messages, is_peeking=is_peeking, ) start_token = now_token.copy_and_replace("room_key", token[0]) @@ -882,7 +884,7 @@ class MessageHandler(BaseHandler): action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( - event, context, self + event, context ) (event_stream_id, max_stream_id) = yield self.store.persist_event( diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index fdebc9c43..3d63b3c51 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -26,6 +26,7 @@ from synapse.api.errors import AuthError, StoreError, SynapseError from synapse.util import stringutils from synapse.util.async import concurrently_execute from synapse.util.caches.response_cache import ResponseCache +from synapse.visibility import filter_events_for_client from collections import OrderedDict @@ -449,10 +450,12 @@ class RoomContextHandler(BaseHandler): now_token = yield self.hs.get_event_sources().get_current_token() def filter_evts(events): - return self.filter_events_for_client( + return filter_events_for_client( + self.store, user.to_string(), events, - is_peeking=is_guest) + is_peeking=is_guest + ) event = yield self.store.get_event(event_id, get_prev_content=True, allow_none=True) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index a937e8740..df75d70fa 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -21,6 +21,7 @@ from synapse.api.constants import Membership, EventTypes from synapse.api.filtering import Filter from synapse.api.errors import SynapseError from synapse.events.utils import serialize_event +from synapse.visibility import filter_events_for_client from unpaddedbase64 import decode_base64, encode_base64 @@ -172,8 +173,8 @@ class SearchHandler(BaseHandler): filtered_events = search_filter.filter([r["event"] for r in results]) - events = yield self.filter_events_for_client( - user.to_string(), filtered_events + events = yield filter_events_for_client( + self.store, user.to_string(), filtered_events ) events.sort(key=lambda e: -rank_map[e.event_id]) @@ -223,8 +224,8 @@ class SearchHandler(BaseHandler): r["event"] for r in results ]) - events = yield self.filter_events_for_client( - user.to_string(), filtered_events + events = yield filter_events_for_client( + self.store, user.to_string(), filtered_events ) room_events.extend(events) @@ -281,12 +282,12 @@ class SearchHandler(BaseHandler): event.room_id, event.event_id, before_limit, after_limit ) - res["events_before"] = yield self.filter_events_for_client( - user.to_string(), res["events_before"] + res["events_before"] = yield filter_events_for_client( + self.store, user.to_string(), res["events_before"] ) - res["events_after"] = yield self.filter_events_for_client( - user.to_string(), res["events_after"] + res["events_after"] = yield filter_events_for_client( + self.store, user.to_string(), res["events_after"] ) res["start"] = now_token.copy_and_replace( diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b7dcbc6b1..921215469 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -22,6 +22,7 @@ from synapse.util.logcontext import LoggingContext from synapse.util.metrics import Measure from synapse.util.caches.response_cache import ResponseCache from synapse.push.clientformat import format_push_rules_for_user +from synapse.visibility import filter_events_for_client from twisted.internet import defer @@ -697,7 +698,8 @@ class SyncHandler(BaseHandler): if recents is not None: recents = sync_config.filter_collection.filter_room_timeline(recents) - recents = yield self.filter_events_for_client( + recents = yield filter_events_for_client( + self.store, sync_config.user.to_string(), recents, ) @@ -718,7 +720,8 @@ class SyncHandler(BaseHandler): loaded_recents = sync_config.filter_collection.filter_room_timeline( events ) - loaded_recents = yield self.filter_events_for_client( + loaded_recents = yield filter_events_for_client( + self.store, sync_config.user.to_string(), loaded_recents, ) diff --git a/synapse/notifier.py b/synapse/notifier.py index cb58dfffd..33b79c0ec 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -21,6 +21,7 @@ from synapse.util.logutils import log_function from synapse.util.async import ObservableDeferred from synapse.util.logcontext import PreserveLoggingContext from synapse.types import StreamToken +from synapse.visibility import filter_events_for_client import synapse.metrics from collections import namedtuple @@ -398,8 +399,8 @@ class Notifier(object): ) if name == "room": - room_member_handler = self.hs.get_handlers().room_member_handler - new_events = yield room_member_handler.filter_events_for_client( + new_events = yield filter_events_for_client( + self.store, user.to_string(), new_events, is_peeking=is_peeking, diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index a0160994b..9b208668b 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -37,14 +37,14 @@ class ActionGenerator: # tag (ie. we just need all the users). @defer.inlineCallbacks - def handle_push_actions_for_event(self, event, context, handler): + def handle_push_actions_for_event(self, event, context): with Measure(self.clock, "handle_push_actions_for_event"): bulk_evaluator = yield evaluator_for_event( event, self.hs, self.store ) actions_by_user = yield bulk_evaluator.action_for_event_by_user( - event, handler, context.current_state + event, context.current_state ) context.push_actions = [ diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index f97df36d8..25e13b342 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -22,6 +22,7 @@ from .baserules import list_with_base_rules from .push_rule_evaluator import PushRuleEvaluatorForEvent from synapse.api.constants import EventTypes +from synapse.visibility import filter_events_for_clients logger = logging.getLogger(__name__) @@ -126,7 +127,7 @@ class BulkPushRuleEvaluator: self.store = store @defer.inlineCallbacks - def action_for_event_by_user(self, event, handler, current_state): + def action_for_event_by_user(self, event, current_state): actions_by_user = {} # None of these users can be peeking since this list of users comes @@ -136,8 +137,8 @@ class BulkPushRuleEvaluator: (u, False) for u in self.rules_by_user.keys() ] - filtered_by_user = yield handler.filter_events_for_clients( - user_tuples, [event], {event.event_id: current_state} + filtered_by_user = yield filter_events_for_clients( + self.store, user_tuples, [event], {event.event_id: current_state} ) room_members = yield self.store.get_users_in_room(self.room_id) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 7031fa6d5..5d60c1efc 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -28,6 +28,7 @@ from synapse.util.presentable_names import ( from synapse.types import UserID from synapse.api.errors import StoreError from synapse.api.constants import EventTypes +from synapse.visibility import filter_events_for_client import jinja2 import bleach @@ -227,9 +228,8 @@ class Mailer(object): "messages": [], } - handler = self.hs.get_handlers().message_handler - the_events = yield handler.filter_events_for_client( - user_id, results["events_before"] + the_events = yield filter_events_for_client( + self.store, user_id, results["events_before"] ) the_events.append(notif_event) diff --git a/synapse/visibility.py b/synapse/visibility.py new file mode 100644 index 000000000..948ad5177 --- /dev/null +++ b/synapse/visibility.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 - 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.constants import Membership, EventTypes + +from synapse.util.logcontext import preserve_fn + +import logging + + +logger = logging.getLogger(__name__) + + +VISIBILITY_PRIORITY = ( + "world_readable", + "shared", + "invited", + "joined", +) + + +MEMBERSHIP_PRIORITY = ( + Membership.JOIN, + Membership.INVITE, + Membership.KNOCK, + Membership.LEAVE, + Membership.BAN, +) + + +@defer.inlineCallbacks +def filter_events_for_clients(store, user_tuples, events, event_id_to_state): + """ Returns dict of user_id -> list of events that user is allowed to + see. + + Args: + user_tuples (str, bool): (user id, is_peeking) for each user to be + checked. is_peeking should be true if: + * the user is not currently a member of the room, and: + * the user has not been a member of the room since the + given events + events ([synapse.events.EventBase]): list of events to filter + """ + forgotten = yield defer.gatherResults([ + preserve_fn(store.who_forgot_in_room)( + room_id, + ) + for room_id in frozenset(e.room_id for e in events) + ], consumeErrors=True) + + # Set of membership event_ids that have been forgotten + event_id_forgotten = frozenset( + row["event_id"] for rows in forgotten for row in rows + ) + + ignore_dict_content = yield store.get_global_account_data_by_type_for_users( + "m.ignored_user_list", user_ids=[user_id for user_id, _ in user_tuples] + ) + + # FIXME: This will explode if people upload something incorrect. + ignore_dict = { + user_id: frozenset( + content.get("ignored_users", {}).keys() if content else [] + ) + for user_id, content in ignore_dict_content.items() + } + + def allowed(event, user_id, is_peeking, ignore_list): + """ + Args: + event (synapse.events.EventBase): event to check + user_id (str) + is_peeking (bool) + ignore_list (list): list of users to ignore + """ + if not event.is_state() and event.sender in ignore_list: + return False + + state = event_id_to_state[event.event_id] + + # get the room_visibility at the time of the event. + visibility_event = state.get((EventTypes.RoomHistoryVisibility, ""), None) + if visibility_event: + visibility = visibility_event.content.get("history_visibility", "shared") + else: + visibility = "shared" + + if visibility not in VISIBILITY_PRIORITY: + visibility = "shared" + + # if it was world_readable, it's easy: everyone can read it + if visibility == "world_readable": + return True + + # Always allow history visibility events on boundaries. This is done + # by setting the effective visibility to the least restrictive + # of the old vs new. + if event.type == EventTypes.RoomHistoryVisibility: + prev_content = event.unsigned.get("prev_content", {}) + prev_visibility = prev_content.get("history_visibility", None) + + if prev_visibility not in VISIBILITY_PRIORITY: + prev_visibility = "shared" + + new_priority = VISIBILITY_PRIORITY.index(visibility) + old_priority = VISIBILITY_PRIORITY.index(prev_visibility) + if old_priority < new_priority: + visibility = prev_visibility + + # likewise, if the event is the user's own membership event, use + # the 'most joined' membership + membership = None + if event.type == EventTypes.Member and event.state_key == user_id: + membership = event.content.get("membership", None) + if membership not in MEMBERSHIP_PRIORITY: + membership = "leave" + + prev_content = event.unsigned.get("prev_content", {}) + prev_membership = prev_content.get("membership", None) + if prev_membership not in MEMBERSHIP_PRIORITY: + prev_membership = "leave" + + new_priority = MEMBERSHIP_PRIORITY.index(membership) + old_priority = MEMBERSHIP_PRIORITY.index(prev_membership) + if old_priority < new_priority: + membership = prev_membership + + # otherwise, get the user's membership at the time of the event. + if membership is None: + membership_event = state.get((EventTypes.Member, user_id), None) + if membership_event: + if membership_event.event_id not in event_id_forgotten: + membership = membership_event.membership + + # if the user was a member of the room at the time of the event, + # they can see it. + if membership == Membership.JOIN: + return True + + if visibility == "joined": + # we weren't a member at the time of the event, so we can't + # see this event. + return False + + elif visibility == "invited": + # user can also see the event if they were *invited* at the time + # of the event. + return membership == Membership.INVITE + + else: + # visibility is shared: user can also see the event if they have + # become a member since the event + # + # XXX: if the user has subsequently joined and then left again, + # ideally we would share history up to the point they left. But + # we don't know when they left. + return not is_peeking + + defer.returnValue({ + user_id: [ + event + for event in events + if allowed(event, user_id, is_peeking, ignore_dict.get(user_id, [])) + ] + for user_id, is_peeking in user_tuples + }) + + +@defer.inlineCallbacks +def filter_events_for_client(store, user_id, events, is_peeking=False): + """ + Check which events a user is allowed to see + + Args: + user_id(str): user id to be checked + events([synapse.events.EventBase]): list of events to be checked + is_peeking(bool): should be True if: + * the user is not currently a member of the room, and: + * the user has not been a member of the room since the given + events + + Returns: + [synapse.events.EventBase] + """ + types = ( + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, user_id), + ) + event_id_to_state = yield store.get_state_for_events( + frozenset(e.event_id for e in events), + types=types + ) + res = yield filter_events_for_clients( + store, [(user_id, is_peeking)], events, event_id_to_state + ) + defer.returnValue(res.get(user_id, [])) From a458a40337484efd25694cd2bf4d600440719aed Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 12 May 2016 18:19:58 +0100 Subject: [PATCH 115/134] missed a spot --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4a65b246e..c21d9d4d8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1113,7 +1113,7 @@ class FederationHandler(BaseHandler): if not event.internal_metadata.is_outlier(): action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( - event, context, self + event, context ) event_stream_id, max_stream_id = yield self.store.persist_event( From 13d37c3c568ac7bfea2e03eff373189c57c0f3fd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 May 2016 11:25:02 +0100 Subject: [PATCH 116/134] Fixup add_pusher --- synapse/storage/_base.py | 8 +++++++- synapse/storage/pusher.py | 13 ++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1e27c2c0c..258b25114 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -453,7 +453,9 @@ class SQLBaseStore(object): keyvalues (dict): The unique key tables and their new values values (dict): The nonunique columns and their new values insertion_values (dict): key/values to use when inserting - Returns: A deferred + Returns: + Deferred(bool): True if a new entry was created, False if an + exisitng one was updated. """ return self.runInteraction( desc, @@ -498,6 +500,10 @@ class SQLBaseStore(object): ) txn.execute(sql, allvalues.values()) + return True + else: + return False + def _simple_select_one(self, table, keyvalues, retcols, allow_none=False, desc="_simple_select_one"): """Executes a SELECT query on the named table, which is expected to diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index d9afd7ec8..9e8e2e296 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -156,8 +156,7 @@ class PusherStore(SQLBaseStore): profile_tag=""): with self._pushers_id_gen.get_next() as stream_id: def f(txn): - txn.call_after(self.get_users_with_pushers_in_room.invalidate_all) - return self._simple_upsert_txn( + newly_inserted = self._simple_upsert_txn( txn, "pushers", { @@ -178,11 +177,18 @@ class PusherStore(SQLBaseStore): "id": stream_id, }, ) - defer.returnValue((yield self.runInteraction("add_pusher", f))) + if newly_inserted: + # get_users_with_pushers_in_room only cares if the user has + # at least *one* pusher. + txn.call_after(self.get_users_with_pushers_in_room.invalidate_all) + + yield self.runInteraction("add_pusher", f) @defer.inlineCallbacks def delete_pusher_by_app_id_pushkey_user_id(self, app_id, pushkey, user_id): def delete_pusher_txn(txn, stream_id): + txn.call_after(self.get_users_with_pushers_in_room.invalidate_all) + self._simple_delete_one_txn( txn, "pushers", @@ -194,6 +200,7 @@ class PusherStore(SQLBaseStore): {"app_id": app_id, "pushkey": pushkey, "user_id": user_id}, {"stream_id": stream_id}, ) + with self._pushers_id_gen.get_next() as stream_id: yield self.runInteraction( "delete_pusher", delete_pusher_txn, stream_id From b5e646a18ce2a293e5d35dcb560ba50183a87429 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 13 May 2016 11:36:50 +0100 Subject: [PATCH 117/134] Make email notifs work on the pusher synapse Plus general bugfix to email notif code --- synapse/app/pusher.py | 47 ++++++++++++++++++++++++++++++++++++++++++ synapse/push/mailer.py | 1 + 2 files changed, 48 insertions(+) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 8e9c0e196..662cd0dc6 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -24,6 +24,8 @@ from synapse.config.emailconfig import EmailConfig from synapse.http.site import SynapseSite from synapse.metrics.resource import MetricsResource, METRICS_PREFIX from synapse.storage.state import StateStore +from synapse.storage.roommember import RoomMemberStore +from synapse.storage.account_data import AccountDataStore from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.pushers import SlavedPusherStore from synapse.replication.slave.storage.receipts import SlavedReceiptsStore @@ -60,6 +62,7 @@ class SlaveConfig(DatabaseConfig): self.soft_file_limit = config.get("soft_file_limit") self.daemonize = config.get("daemonize") self.pid_file = self.abspath(config.get("pid_file")) + self.public_baseurl = config["public_baseurl"] def default_config(self, server_name, **kwargs): pid_file = self.abspath("pusher.pid") @@ -132,6 +135,30 @@ class PusherSlaveStore( DataStore.get_state_groups.__func__ ) + _get_state_for_groups = ( + DataStore._get_state_for_groups.__func__ + ) + + _get_all_state_from_cache = ( + DataStore._get_all_state_from_cache.__func__ + ) + + get_events_around = ( + DataStore.get_events_around.__func__ + ) + + _get_events_around_txn = ( + DataStore._get_events_around_txn.__func__ + ) + + get_state_for_events = ( + DataStore.get_state_for_events.__func__ + ) + + _get_some_state_from_cache = ( + DataStore._get_some_state_from_cache.__func__ + ) + _get_state_group_for_events = ( StateStore.__dict__["_get_state_group_for_events"] ) @@ -140,6 +167,26 @@ class PusherSlaveStore( StateStore.__dict__["_get_state_group_for_event"] ) + _get_state_groups_from_groups = ( + StateStore.__dict__["_get_state_groups_from_groups"] + ) + + _get_state_group_from_group = ( + StateStore.__dict__["_get_state_group_from_group"] + ) + + get_global_account_data_by_type_for_users = ( + AccountDataStore.__dict__["get_global_account_data_by_type_for_users"] + ) + + get_global_account_data_by_type_for_user = ( + AccountDataStore.__dict__["get_global_account_data_by_type_for_user"] + ) + + who_forgot_in_room = ( + RoomMemberStore.__dict__["who_forgot_in_room"] + ) + class PusherServer(HomeServer): diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 5d60c1efc..2be294f52 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -397,6 +397,7 @@ class Mailer(object): return "" serverAndMediaId = value[6:] + fragment = None if '#' in serverAndMediaId: (serverAndMediaId, fragment) = serverAndMediaId.split('#', 1) fragment = "#" + fragment From 6da7f39d952f7c8bb805d8b863c737d98b240217 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 May 2016 11:41:23 +0100 Subject: [PATCH 118/134] Use tree cache for get_linearized_receipts_for_room --- synapse/storage/receipts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 935fc503d..669fc8ada 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -100,7 +100,7 @@ class ReceiptsStore(SQLBaseStore): defer.returnValue([ev for res in results.values() for ev in res]) - @cachedInlineCallbacks(num_args=3, max_entries=5000) + @cachedInlineCallbacks(num_args=3, max_entries=5000, lru=True, tree=True) def get_linearized_receipts_for_room(self, room_id, to_key, from_key=None): """Get receipts for a single room for sending to clients. @@ -232,7 +232,7 @@ class ReceiptsStore(SQLBaseStore): self.get_receipts_for_user.invalidate, (user_id, receipt_type) ) # FIXME: This shouldn't invalidate the whole cache - txn.call_after(self.get_linearized_receipts_for_room.invalidate_all) + txn.call_after(self.get_linearized_receipts_for_room.invalidate_many, (room_id,)) txn.call_after( self._receipts_stream_cache.entity_has_changed, @@ -367,7 +367,7 @@ class ReceiptsStore(SQLBaseStore): self.get_receipts_for_user.invalidate, (user_id, receipt_type) ) # FIXME: This shouldn't invalidate the whole cache - txn.call_after(self.get_linearized_receipts_for_room.invalidate_all) + txn.call_after(self.get_linearized_receipts_for_room.invalidate_many, (room_id,)) self._simple_delete_txn( txn, From 3547e66bc684ce3f0fbc83297fbe319a683c2a15 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 11:53:00 +0100 Subject: [PATCH 119/134] Make sure we advance our stream position --- synapse/replication/slave/storage/events.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py index 7ba7a6f6e..635febb17 100644 --- a/synapse/replication/slave/storage/events.py +++ b/synapse/replication/slave/storage/events.py @@ -146,12 +146,14 @@ class SlavedEventStore(BaseSlavedStore): stream = result.get("forward_ex_outliers") if stream: + self._stream_id_gen.advance(stream["position"]) for row in stream["rows"]: event_id = row[1] self._invalidate_get_event_cache(event_id) stream = result.get("backward_ex_outliers") if stream: + self._backfill_id_gen.advance(-stream["position"]) for row in stream["rows"]: event_id = row[1] self._invalidate_get_event_cache(event_id) From 0e792e7903d34f36a48ac09725a1f01b4fea6810 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 11:54:44 +0100 Subject: [PATCH 120/134] Log the stream IDs in an order that makes sense --- synapse/replication/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/replication/resource.py b/synapse/replication/resource.py index 69ad1de86..0e983ae7f 100644 --- a/synapse/replication/resource.py +++ b/synapse/replication/resource.py @@ -164,8 +164,8 @@ class ReplicationResource(Resource): "Replicating %d rows of %s from %s -> %s", len(stream_content["rows"]), stream_name, - stream_content["position"], request_streams.get(stream_name), + stream_content["position"], ) request.write(json.dumps(result, ensure_ascii=False)) From 5e500584738d4e9ce897e92056e20c7909052601 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 May 2016 13:28:07 +0100 Subject: [PATCH 121/134] Remove unused indices This includes removing both unused indices and indices that are subsets of other indices. --- .../schema/delta/32/remove_indices.sql | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 synapse/storage/schema/delta/32/remove_indices.sql diff --git a/synapse/storage/schema/delta/32/remove_indices.sql b/synapse/storage/schema/delta/32/remove_indices.sql new file mode 100644 index 000000000..314fa5128 --- /dev/null +++ b/synapse/storage/schema/delta/32/remove_indices.sql @@ -0,0 +1,42 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +-- The following indices are redundant, other indices are equivalent or +-- supersets +DROP INDEX IF EXISTS events_room_id; +DROP INDEX IF EXISTS events_order; +DROP INDEX IF EXISTS events_topological_ordering; +DROP INDEX IF EXISTS events_stream_ordering; +DROP INDEX IF EXISTS state_groups_id; +DROP INDEX IF EXISTS event_to_state_groups_id; +DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag; +DROP INDEX IF EXISTS event_push_actions_room_id_user_id; + +DROP INDEX IF EXISTS event_destinations_id; +DROP INDEX IF EXISTS st_extrem_id; +DROP INDEX IF EXISTS event_content_hashes_id; +DROP INDEX IF EXISTS event_signatures_id; +DROP INDEX IF EXISTS event_edge_hashes_id; +DROP INDEX IF EXISTS redactions_event_id; +DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id; +DROP INDEX IF EXISTS room_hosts_room_id; +DROP INDEX IF EXISTS event_search_ev_ridx; + + +-- The following indices were unused +DROP INDEX IF EXISTS evauth_edges_auth_id; +DROP INDEX IF EXISTS topics_room_id; +DROP INDEX IF EXISTS presence_stream_state; From 9295fa30a83c5ecdb6315b1cc39910af3284d6be Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 14:16:57 +0100 Subject: [PATCH 122/134] Annotate the removed indicies with why they were removed. --- .../schema/delta/32/remove_indices.sql | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/synapse/storage/schema/delta/32/remove_indices.sql b/synapse/storage/schema/delta/32/remove_indices.sql index 314fa5128..91eab6d63 100644 --- a/synapse/storage/schema/delta/32/remove_indices.sql +++ b/synapse/storage/schema/delta/32/remove_indices.sql @@ -16,27 +16,24 @@ -- The following indices are redundant, other indices are equivalent or -- supersets -DROP INDEX IF EXISTS events_room_id; -DROP INDEX IF EXISTS events_order; -DROP INDEX IF EXISTS events_topological_ordering; -DROP INDEX IF EXISTS events_stream_ordering; -DROP INDEX IF EXISTS state_groups_id; -DROP INDEX IF EXISTS event_to_state_groups_id; -DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag; -DROP INDEX IF EXISTS event_push_actions_room_id_user_id; - -DROP INDEX IF EXISTS event_destinations_id; -DROP INDEX IF EXISTS st_extrem_id; -DROP INDEX IF EXISTS event_content_hashes_id; -DROP INDEX IF EXISTS event_signatures_id; -DROP INDEX IF EXISTS event_edge_hashes_id; -DROP INDEX IF EXISTS redactions_event_id; -DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id; -DROP INDEX IF EXISTS room_hosts_room_id; -DROP INDEX IF EXISTS event_search_ev_ridx; +DROP INDEX IF EXISTS events_room_id; -- Prefix of events_room_stream +DROP INDEX IF EXISTS events_order; -- Prefix of events_order_topo_stream_room +DROP INDEX IF EXISTS events_topological_ordering; -- Prefix of events_order_topo_stream_room +DROP INDEX IF EXISTS events_stream_ordering; -- Duplicate of PRIMARY KEY +DROP INDEX IF EXISTS state_groups_id; -- Duplicate of PRIMARY KEY +DROP INDEX IF EXISTS event_to_state_groups_id; -- Duplicate of PRIMARY KEY +DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag; -- Duplicate of UNIQUE CONSTRAINT +DROP INDEX IF EXISTS event_destinations_id; -- Prefix of UNIQUE CONSTRAINT +DROP INDEX IF EXISTS st_extrem_id; -- Prefix of UNIQUE CONSTRAINT +DROP INDEX IF EXISTS event_content_hashes_id; -- Prefix of UNIQUE CONSTRAINT +DROP INDEX IF EXISTS event_signatures_id; -- Prefix of UNIQUE CONSTRAINT +DROP INDEX IF EXISTS event_edge_hashes_id; -- Prefix of UNIQUE CONSTRAINT +DROP INDEX IF EXISTS redactions_event_id; -- Duplicate of UNIQUE CONSTRAINT +DROP INDEX IF EXISTS room_hosts_room_id; -- Prefix of UNIQUE CONSTRAINT -- The following indices were unused +DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id; DROP INDEX IF EXISTS evauth_edges_auth_id; DROP INDEX IF EXISTS topics_room_id; DROP INDEX IF EXISTS presence_stream_state; From 40aa6e8349b348802d6f87084c31c3895f728708 Mon Sep 17 00:00:00 2001 From: Negi Fazeli Date: Wed, 20 Apr 2016 16:21:40 +0200 Subject: [PATCH 123/134] Create user with expiry - Add unittests for client, api and handler Signed-off-by: Negar Fazeli --- synapse/api/auth.py | 3 +- synapse/config/key.py | 5 ++ synapse/config/registration.py | 6 ++ synapse/handlers/auth.py | 4 +- synapse/handlers/register.py | 53 ++++++++++++++++ synapse/rest/client/v1/register.py | 71 +++++++++++++++++++++ tests/api/test_auth.py | 12 ++-- tests/handlers/test_register.py | 67 ++++++++++++++++++++ tests/rest/client/v1/test_register.py | 88 +++++++++++++++++++++++++++ tests/utils.py | 1 + 10 files changed, 301 insertions(+), 9 deletions(-) create mode 100644 tests/handlers/test_register.py create mode 100644 tests/rest/client/v1/test_register.py diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d3e9837c8..44e38b777 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -612,7 +612,8 @@ class Auth(object): def get_user_from_macaroon(self, macaroon_str): try: macaroon = pymacaroons.Macaroon.deserialize(macaroon_str) - self.validate_macaroon(macaroon, "access", False) + + self.validate_macaroon(macaroon, "access", self.hs.config.expire_access_token) user_prefix = "user_id = " user = None diff --git a/synapse/config/key.py b/synapse/config/key.py index a072aec71..6ee643793 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -57,6 +57,8 @@ class KeyConfig(Config): seed = self.signing_key[0].seed self.macaroon_secret_key = hashlib.sha256(seed) + self.expire_access_token = config.get("expire_access_token", False) + def default_config(self, config_dir_path, server_name, is_generating_file=False, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) @@ -69,6 +71,9 @@ class KeyConfig(Config): return """\ macaroon_secret_key: "%(macaroon_secret_key)s" + # Used to enable access token expiration. + expire_access_token: False + ## Signing Keys ## # Path to the signing key to sign messages with diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 87e500c97..cc3f87985 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -32,6 +32,7 @@ class RegistrationConfig(Config): ) self.registration_shared_secret = config.get("registration_shared_secret") + self.user_creation_max_duration = int(config["user_creation_max_duration"]) self.bcrypt_rounds = config.get("bcrypt_rounds", 12) self.trusted_third_party_id_servers = config["trusted_third_party_id_servers"] @@ -54,6 +55,11 @@ class RegistrationConfig(Config): # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" + # Sets the expiry for the short term user creation in + # milliseconds. For instance the bellow duration is two weeks + # in milliseconds. + user_creation_max_duration: 1209600000 + # Set the number of bcrypt rounds used to generate password hash. # Larger numbers increase the work factor needed to generate the hash. # The default number of rounds is 12. diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 61fe56032..3d36d3460 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -521,11 +521,11 @@ class AuthHandler(BaseHandler): )) return m.serialize() - def generate_short_term_login_token(self, user_id): + def generate_short_term_login_token(self, user_id, duration_in_ms=(2 * 60 * 1000)): macaroon = self._generate_base_macaroon(user_id) macaroon.add_first_party_caveat("type = login") now = self.hs.get_clock().time_msec() - expiry = now + (2 * 60 * 1000) + expiry = now + duration_in_ms macaroon.add_first_party_caveat("time < %d" % (expiry,)) return macaroon.serialize() diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b0862067e..5883b9111 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -358,6 +358,59 @@ class RegistrationHandler(BaseHandler): ) defer.returnValue(data) + @defer.inlineCallbacks + def get_or_create_user(self, localpart, displayname, duration_seconds): + """Creates a new user or returns an access token for an existing one + + Args: + localpart : The local part of the user ID to register. If None, + one will be randomly generated. + Returns: + A tuple of (user_id, access_token). + Raises: + RegistrationError if there was a problem registering. + """ + yield run_on_reactor() + + if localpart is None: + raise SynapseError(400, "Request must include user id") + + need_register = True + + try: + yield self.check_username(localpart) + except SynapseError as e: + if e.errcode == Codes.USER_IN_USE: + need_register = False + else: + raise + + user = UserID(localpart, self.hs.hostname) + user_id = user.to_string() + auth_handler = self.hs.get_handlers().auth_handler + token = auth_handler.generate_short_term_login_token(user_id, duration_seconds) + + if need_register: + yield self.store.register( + user_id=user_id, + token=token, + password_hash=None + ) + + yield registered_user(self.distributor, user) + else: + yield self.store.flush_user(user_id=user_id) + yield self.store.add_access_token_to_user(user_id=user_id, token=token) + + if displayname is not None: + logger.info("setting user display name: %s -> %s", user_id, displayname) + profile_handler = self.hs.get_handlers().profile_handler + yield profile_handler.set_displayname( + user, user, displayname + ) + + defer.returnValue((user_id, token)) + def auth_handler(self): return self.hs.get_handlers().auth_handler diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py index c6a2ef2cc..e3f4fbb0b 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1/register.py @@ -355,5 +355,76 @@ class RegisterRestServlet(ClientV1RestServlet): ) +class CreateUserRestServlet(ClientV1RestServlet): + """Handles user creation via a server-to-server interface + """ + + PATTERNS = client_path_patterns("/createUser$", releases=()) + + def __init__(self, hs): + super(CreateUserRestServlet, self).__init__(hs) + self.store = hs.get_datastore() + self.direct_user_creation_max_duration = hs.config.user_creation_max_duration + + @defer.inlineCallbacks + def on_POST(self, request): + user_json = parse_json_object_from_request(request) + + if "access_token" not in request.args: + raise SynapseError(400, "Expected application service token.") + + app_service = yield self.store.get_app_service_by_token( + request.args["access_token"][0] + ) + if not app_service: + raise SynapseError(403, "Invalid application service token.") + + logger.debug("creating user: %s", user_json) + + response = yield self._do_create(user_json) + + defer.returnValue((200, response)) + + def on_OPTIONS(self, request): + return 403, {} + + @defer.inlineCallbacks + def _do_create(self, user_json): + yield run_on_reactor() + + if "localpart" not in user_json: + raise SynapseError(400, "Expected 'localpart' key.") + + if "displayname" not in user_json: + raise SynapseError(400, "Expected 'displayname' key.") + + if "duration_seconds" not in user_json: + raise SynapseError(400, "Expected 'duration_seconds' key.") + + localpart = user_json["localpart"].encode("utf-8") + displayname = user_json["displayname"].encode("utf-8") + duration_seconds = 0 + try: + duration_seconds = int(user_json["duration_seconds"]) + except ValueError: + raise SynapseError(400, "Failed to parse 'duration_seconds'") + if duration_seconds > self.direct_user_creation_max_duration: + duration_seconds = self.direct_user_creation_max_duration + + handler = self.handlers.registration_handler + user_id, token = yield handler.get_or_create_user( + localpart=localpart, + displayname=displayname, + duration_seconds=duration_seconds + ) + + defer.returnValue({ + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + }) + + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) + CreateUserRestServlet(hs).register(http_server) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 7e7b0b4b1..ad269af0e 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -284,12 +284,12 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("time < 1") # ms self.hs.clock.now = 5000 # seconds - - yield self.auth.get_user_from_macaroon(macaroon.serialize()) + self.hs.config.expire_access_token = True + # yield self.auth.get_user_from_macaroon(macaroon.serialize()) # TODO(daniel): Turn on the check that we validate expiration, when we # validate expiration (and remove the above line, which will start # throwing). - # with self.assertRaises(AuthError) as cm: - # yield self.auth.get_user_from_macaroon(macaroon.serialize()) - # self.assertEqual(401, cm.exception.code) - # self.assertIn("Invalid macaroon", cm.exception.msg) + with self.assertRaises(AuthError) as cm: + yield self.auth.get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py new file mode 100644 index 000000000..8b7be96bd --- /dev/null +++ b/tests/handlers/test_register.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright 2015, 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer +from .. import unittest + +from synapse.handlers.register import RegistrationHandler + +from tests.utils import setup_test_homeserver + +from mock import Mock + + +class RegistrationHandlers(object): + def __init__(self, hs): + self.registration_handler = RegistrationHandler(hs) + + +class RegistrationTestCase(unittest.TestCase): + """ Tests the RegistrationHandler. """ + + @defer.inlineCallbacks + def setUp(self): + self.mock_distributor = Mock() + self.mock_distributor.declare("registered_user") + self.mock_captcha_client = Mock() + hs = yield setup_test_homeserver( + handlers=None, + http_client=None, + expire_access_token=True) + hs.handlers = RegistrationHandlers(hs) + self.handler = hs.get_handlers().registration_handler + hs.get_handlers().profile_handler = Mock() + self.mock_handler = Mock(spec=[ + "generate_short_term_login_token", + ]) + + hs.get_handlers().auth_handler = self.mock_handler + + @defer.inlineCallbacks + def test_user_is_created_and_logged_in_if_doesnt_exist(self): + """ + Returns: + The user doess not exist in this case so it will register and log it in + """ + duration_ms = 200 + local_part = "someone" + display_name = "someone" + user_id = "@someone:test" + mock_token = self.mock_handler.generate_short_term_login_token + mock_token.return_value = 'secret' + result_user_id, result_token = yield self.handler.get_or_create_user( + local_part, display_name, duration_ms) + self.assertEquals(result_user_id, user_id) + self.assertEquals(result_token, 'secret') diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py new file mode 100644 index 000000000..4a898a034 --- /dev/null +++ b/tests/rest/client/v1/test_register.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# Copyright 2015, 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.rest.client.v1.register import CreateUserRestServlet +from twisted.internet import defer +from mock import Mock +from tests import unittest +import json + + +class CreateUserServletTestCase(unittest.TestCase): + + def setUp(self): + # do the dance to hook up request data to self.request_data + self.request_data = "" + self.request = Mock( + content=Mock(read=Mock(side_effect=lambda: self.request_data)), + path='/_matrix/client/api/v1/createUser' + ) + self.request.args = {} + + self.appservice = None + self.auth = Mock(get_appservice_by_req=Mock( + side_effect=lambda x: defer.succeed(self.appservice)) + ) + + self.auth_result = (False, None, None, None) + self.auth_handler = Mock( + check_auth=Mock(side_effect=lambda x, y, z: self.auth_result), + get_session_data=Mock(return_value=None) + ) + self.registration_handler = Mock() + self.identity_handler = Mock() + self.login_handler = Mock() + + # do the dance to hook it up to the hs global + self.handlers = Mock( + auth_handler=self.auth_handler, + registration_handler=self.registration_handler, + identity_handler=self.identity_handler, + login_handler=self.login_handler + ) + self.hs = Mock() + self.hs.hostname = "supergbig~testing~thing.com" + self.hs.get_auth = Mock(return_value=self.auth) + self.hs.get_handlers = Mock(return_value=self.handlers) + self.hs.config.enable_registration = True + # init the thing we're testing + self.servlet = CreateUserRestServlet(self.hs) + + @defer.inlineCallbacks + def test_POST_createuser_with_valid_user(self): + user_id = "@someone:interesting" + token = "my token" + self.request.args = { + "access_token": "i_am_an_app_service" + } + self.request_data = json.dumps({ + "localpart": "someone", + "displayname": "someone interesting", + "duration_seconds": 200 + }) + + self.registration_handler.get_or_create_user = Mock( + return_value=(user_id, token) + ) + + (code, result) = yield self.servlet.on_POST(self.request) + self.assertEquals(code, 200) + + det_data = { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname + } + self.assertDictContainsSubset(det_data, result) diff --git a/tests/utils.py b/tests/utils.py index c179df31e..9d7978a64 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -49,6 +49,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.event_cache_size = 1 config.enable_registration = True config.macaroon_secret_key = "not even a little secret" + config.expire_access_token = False config.server_name = "server.under.test" config.trusted_third_party_id_servers = [] config.room_invite_state_types = [] From c9aff0736c75046b8923057309af91d1bd7a6985 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 May 2016 14:40:38 +0100 Subject: [PATCH 124/134] Remove topics table --- synapse/storage/schema/delta/32/remove_indices.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/schema/delta/32/remove_indices.sql b/synapse/storage/schema/delta/32/remove_indices.sql index 91eab6d63..f859be46a 100644 --- a/synapse/storage/schema/delta/32/remove_indices.sql +++ b/synapse/storage/schema/delta/32/remove_indices.sql @@ -35,5 +35,4 @@ DROP INDEX IF EXISTS room_hosts_room_id; -- Prefix of UNIQUE CONSTRAINT -- The following indices were unused DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id; DROP INDEX IF EXISTS evauth_edges_auth_id; -DROP INDEX IF EXISTS topics_room_id; DROP INDEX IF EXISTS presence_stream_state; From 0c11c1be884d69bf0d45b6d1f55a71861e16ee8f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 May 2016 14:42:25 +0100 Subject: [PATCH 125/134] Spelling --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 258b25114..e0d709869 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -455,7 +455,7 @@ class SQLBaseStore(object): insertion_values (dict): key/values to use when inserting Returns: Deferred(bool): True if a new entry was created, False if an - exisitng one was updated. + existing one was updated. """ return self.runInteraction( desc, From 3abab26458cc9fe8a77d5ccee664e87ce407ed58 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 15:34:06 +0100 Subject: [PATCH 126/134] Add a slaved datastore for account data --- .../replication/slave/storage/account_data.py | 61 +++++++++++++++++++ .../slave/storage/test_account_data.py | 56 +++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 synapse/replication/slave/storage/account_data.py create mode 100644 tests/replication/slave/storage/test_account_data.py diff --git a/synapse/replication/slave/storage/account_data.py b/synapse/replication/slave/storage/account_data.py new file mode 100644 index 000000000..f59b0eabb --- /dev/null +++ b/synapse/replication/slave/storage/account_data.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import BaseSlavedStore +from ._slaved_id_tracker import SlavedIdTracker +from synapse.storage.account_data import AccountDataStore + + +class SlavedAccountDataStore(BaseSlavedStore): + + def __init__(self, db_conn, hs): + super(SlavedAccountDataStore, self).__init__(db_conn, hs) + self._account_data_id_gen = SlavedIdTracker( + db_conn, "account_data_max_stream_id", "stream_id", + ) + + get_global_account_data_by_type_for_users = ( + AccountDataStore.__dict__["get_global_account_data_by_type_for_users"] + ) + + get_global_account_data_by_type_for_user = ( + AccountDataStore.__dict__["get_global_account_data_by_type_for_user"] + ) + + def stream_positions(self): + result = super(SlavedAccountDataStore, self).stream_positions() + position = self._account_data_id_gen.get_current_token() + result["user_account_data"] = position + result["room_account_data"] = position + result["tag_account_data"] = position + return result + + def process_replication(self, result): + stream = result.get("user_account_data") + if stream: + self._account_data_id_gen.advance(int(stream["position"])) + for row in stream["rows"]: + user_id, data_type = row[1:3] + self.get_global_account_data_by_type_for_user.invalidate( + (data_type, user_id,) + ) + + stream = result.get("room_account_data") + if stream: + self._account_data_id_gen.advance(int(stream["position"])) + + stream = result.get("tag_account_data") + if stream: + self._account_data_id_gen.advance(int(stream["position"])) diff --git a/tests/replication/slave/storage/test_account_data.py b/tests/replication/slave/storage/test_account_data.py new file mode 100644 index 000000000..da54d478c --- /dev/null +++ b/tests/replication/slave/storage/test_account_data.py @@ -0,0 +1,56 @@ +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ._base import BaseSlavedStoreTestCase + +from synapse.replication.slave.storage.account_data import SlavedAccountDataStore + +from twisted.internet import defer + +USER_ID = "@feeling:blue" +TYPE = "my.type" + + +class SlavedAccountDataStoreTestCase(BaseSlavedStoreTestCase): + + STORE_TYPE = SlavedAccountDataStore + + @defer.inlineCallbacks + def test_user_account_data(self): + yield self.master_store.add_account_data_for_user( + USER_ID, TYPE, {"a": 1} + ) + yield self.replicate() + yield self.check( + "get_global_account_data_by_type_for_user", + [TYPE, USER_ID], {"a": 1} + ) + yield self.check( + "get_global_account_data_by_type_for_users", + [TYPE, [USER_ID]], {USER_ID: {"a": 1}} + ) + + yield self.master_store.add_account_data_for_user( + USER_ID, TYPE, {"a": 2} + ) + yield self.replicate() + yield self.check( + "get_global_account_data_by_type_for_user", + [TYPE, USER_ID], {"a": 2} + ) + yield self.check( + "get_global_account_data_by_type_for_users", + [TYPE, [USER_ID]], {USER_ID: {"a": 2}} + ) From b7381d5338e37b8c4c374f5458e578e05eb762d0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 15:46:41 +0100 Subject: [PATCH 127/134] Allow receipts for events we haven't seen in the db --- synapse/storage/receipts.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 94be820f8..fdcf28f3e 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -249,9 +249,11 @@ class ReceiptsStore(SQLBaseStore): table="events", retcols=["topological_ordering", "stream_ordering"], keyvalues={"event_id": event_id}, + allow_none=True ) - topological_ordering = int(res["topological_ordering"]) - stream_ordering = int(res["stream_ordering"]) + + topological_ordering = int(res["topological_ordering"]) if res else None + stream_ordering = int(res["stream_ordering"]) if res else None # We don't want to clobber receipts for more recent events, so we # have to compare orderings of existing receipts @@ -264,7 +266,7 @@ class ReceiptsStore(SQLBaseStore): txn.execute(sql, (room_id, receipt_type, user_id)) results = txn.fetchall() - if results: + if results and topological_ordering: for to, so, _ in results: if int(to) > topological_ordering: return False @@ -294,7 +296,7 @@ class ReceiptsStore(SQLBaseStore): } ) - if receipt_type == "m.read": + if receipt_type == "m.read" and topological_ordering: self._remove_push_actions_before_txn( txn, room_id=room_id, From 206eb9fd947ba86060340ba2154d1112570b76cd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 16:58:14 +0100 Subject: [PATCH 128/134] Shift some of the state_group methods into the SlavedEventStore --- synapse/app/pusher.py | 45 --------------------- synapse/replication/slave/storage/events.py | 19 +++++++++ 2 files changed, 19 insertions(+), 45 deletions(-) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 662cd0dc6..9d41b62db 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -23,7 +23,6 @@ from synapse.config.logger import LoggingConfig from synapse.config.emailconfig import EmailConfig from synapse.http.site import SynapseSite from synapse.metrics.resource import MetricsResource, METRICS_PREFIX -from synapse.storage.state import StateStore from synapse.storage.roommember import RoomMemberStore from synapse.storage.account_data import AccountDataStore from synapse.replication.slave.storage.events import SlavedEventStore @@ -131,50 +130,6 @@ class PusherSlaveStore( DataStore.get_profile_displayname.__func__ ) - get_state_groups = ( - DataStore.get_state_groups.__func__ - ) - - _get_state_for_groups = ( - DataStore._get_state_for_groups.__func__ - ) - - _get_all_state_from_cache = ( - DataStore._get_all_state_from_cache.__func__ - ) - - get_events_around = ( - DataStore.get_events_around.__func__ - ) - - _get_events_around_txn = ( - DataStore._get_events_around_txn.__func__ - ) - - get_state_for_events = ( - DataStore.get_state_for_events.__func__ - ) - - _get_some_state_from_cache = ( - DataStore._get_some_state_from_cache.__func__ - ) - - _get_state_group_for_events = ( - StateStore.__dict__["_get_state_group_for_events"] - ) - - _get_state_group_for_event = ( - StateStore.__dict__["_get_state_group_for_event"] - ) - - _get_state_groups_from_groups = ( - StateStore.__dict__["_get_state_groups_from_groups"] - ) - - _get_state_group_from_group = ( - StateStore.__dict__["_get_state_group_from_group"] - ) - get_global_account_data_by_type_for_users = ( AccountDataStore.__dict__["get_global_account_data_by_type_for_users"] ) diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py index 7ba7a6f6e..0e29bd51d 100644 --- a/synapse/replication/slave/storage/events.py +++ b/synapse/replication/slave/storage/events.py @@ -75,6 +75,18 @@ class SlavedEventStore(BaseSlavedStore): get_unread_event_push_actions_by_room_for_user = ( EventPushActionsStore.__dict__["get_unread_event_push_actions_by_room_for_user"] ) + _get_state_group_for_events = ( + StateStore.__dict__["_get_state_group_for_events"] + ) + _get_state_group_for_event = ( + StateStore.__dict__["_get_state_group_for_event"] + ) + _get_state_groups_from_groups = ( + StateStore.__dict__["_get_state_groups_from_groups"] + ) + _get_state_group_from_group = ( + StateStore.__dict__["_get_state_group_from_group"] + ) get_unread_push_actions_for_user_in_range = ( DataStore.get_unread_push_actions_for_user_in_range.__func__ @@ -96,6 +108,9 @@ class SlavedEventStore(BaseSlavedStore): get_room_events_stream_for_room = ( DataStore.get_room_events_stream_for_room.__func__ ) + get_events_around = DataStore.get_events_around.__func__ + get_state_for_events = DataStore.get_state_for_events.__func__ + get_state_groups = DataStore.get_state_groups.__func__ _set_before_and_after = DataStore._set_before_and_after @@ -116,6 +131,10 @@ class SlavedEventStore(BaseSlavedStore): DataStore._get_rooms_for_user_where_membership_is_txn.__func__ ) _get_members_rows_txn = DataStore._get_members_rows_txn.__func__ + _get_state_for_groups = DataStore._get_state_for_groups.__func__ + _get_all_state_from_cache = DataStore._get_all_state_from_cache.__func__ + _get_events_around_txn = DataStore._get_events_around_txn.__func__ + _get_some_state_from_cache = DataStore._get_some_state_from_cache.__func__ def stream_positions(self): result = super(SlavedEventStore, self).stream_positions() From f03ddc98ec38771a329f47c13c76dd4e93fbef16 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 17:01:28 +0100 Subject: [PATCH 129/134] Use the SlavedAccountDataStore --- synapse/app/pusher.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 9d41b62db..8ff8329f8 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -24,10 +24,10 @@ from synapse.config.emailconfig import EmailConfig from synapse.http.site import SynapseSite from synapse.metrics.resource import MetricsResource, METRICS_PREFIX from synapse.storage.roommember import RoomMemberStore -from synapse.storage.account_data import AccountDataStore from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.pushers import SlavedPusherStore from synapse.replication.slave.storage.receipts import SlavedReceiptsStore +from synapse.replication.slave.storage.account_data import SlavedAccountDataStore from synapse.storage.engines import create_engine from synapse.storage import DataStore from synapse.util.async import sleep @@ -100,7 +100,8 @@ class PusherSlaveConfig(SlaveConfig, LoggingConfig, EmailConfig): class PusherSlaveStore( - SlavedEventStore, SlavedPusherStore, SlavedReceiptsStore + SlavedEventStore, SlavedPusherStore, SlavedReceiptsStore, + SlavedAccountDataStore ): update_pusher_last_stream_ordering_and_success = ( DataStore.update_pusher_last_stream_ordering_and_success.__func__ @@ -130,14 +131,6 @@ class PusherSlaveStore( DataStore.get_profile_displayname.__func__ ) - get_global_account_data_by_type_for_users = ( - AccountDataStore.__dict__["get_global_account_data_by_type_for_users"] - ) - - get_global_account_data_by_type_for_user = ( - AccountDataStore.__dict__["get_global_account_data_by_type_for_user"] - ) - who_forgot_in_room = ( RoomMemberStore.__dict__["who_forgot_in_room"] ) From b3f29dc1e59ccb3e53a124d9c0d85d26377350f4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 17:16:27 +0100 Subject: [PATCH 130/134] Manually expire broken caches like the who_forgot_in_room --- synapse/app/pusher.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 8ff8329f8..135dd58c1 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -131,6 +131,11 @@ class PusherSlaveStore( DataStore.get_profile_displayname.__func__ ) + # XXX: This is a bit broken because we don't persist forgotten rooms + # in a way that they can be streamed. This means that we don't have a + # way to invalidate the forgotten rooms cache correctly. + # For now we expire the cache every 10 minutes. + BROKEN_CACHE_EXPIRY_MS = 60 * 60 * 1000 who_forgot_in_room = ( RoomMemberStore.__dict__["who_forgot_in_room"] ) @@ -214,6 +219,7 @@ class PusherServer(HomeServer): store = self.get_datastore() replication_url = self.config.replication_url pusher_pool = self.get_pusherpool() + clock = self.get_clock() def stop_pusher(user_id, app_id, pushkey): key = "%s:%s" % (app_id, pushkey) @@ -265,11 +271,21 @@ class PusherServer(HomeServer): min_stream_id, max_stream_id, affected_room_ids ) + def expire_broken_caches(): + store.who_forgot_in_room.invalidate_all() + + next_expire_broken_caches_ms = 0 while True: try: args = store.stream_positions() args["timeout"] = 30000 result = yield http_client.get_json(replication_url, args=args) + now_ms = clock.time_msec() + if now_ms > next_expire_broken_caches_ms: + expire_broken_caches() + next_expire_broken_caches_ms = ( + now_ms + store.BROKEN_CACHE_EXPIRY_MS + ) yield store.process_replication(result) poke_pushers(result) except: From 0466454b003860dba23363f882916eb4f7d27648 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 13 May 2016 17:33:44 +0100 Subject: [PATCH 131/134] Assert that stream replicated stream positions are ints --- synapse/replication/slave/storage/events.py | 8 ++++---- synapse/replication/slave/storage/pushers.py | 4 ++-- synapse/replication/slave/storage/receipts.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py index 99cddf251..c0d741452 100644 --- a/synapse/replication/slave/storage/events.py +++ b/synapse/replication/slave/storage/events.py @@ -149,7 +149,7 @@ class SlavedEventStore(BaseSlavedStore): stream = result.get("events") if stream: - self._stream_id_gen.advance(stream["position"]) + self._stream_id_gen.advance(int(stream["position"])) for row in stream["rows"]: self._process_replication_row( row, backfilled=False, state_resets=state_resets @@ -157,7 +157,7 @@ class SlavedEventStore(BaseSlavedStore): stream = result.get("backfill") if stream: - self._backfill_id_gen.advance(-stream["position"]) + self._backfill_id_gen.advance(-int(stream["position"])) for row in stream["rows"]: self._process_replication_row( row, backfilled=True, state_resets=state_resets @@ -165,14 +165,14 @@ class SlavedEventStore(BaseSlavedStore): stream = result.get("forward_ex_outliers") if stream: - self._stream_id_gen.advance(stream["position"]) + self._stream_id_gen.advance(int(stream["position"])) for row in stream["rows"]: event_id = row[1] self._invalidate_get_event_cache(event_id) stream = result.get("backward_ex_outliers") if stream: - self._backfill_id_gen.advance(-stream["position"]) + self._backfill_id_gen.advance(-int(stream["position"])) for row in stream["rows"]: event_id = row[1] self._invalidate_get_event_cache(event_id) diff --git a/synapse/replication/slave/storage/pushers.py b/synapse/replication/slave/storage/pushers.py index 8faddb259..d88206b3b 100644 --- a/synapse/replication/slave/storage/pushers.py +++ b/synapse/replication/slave/storage/pushers.py @@ -43,10 +43,10 @@ class SlavedPusherStore(BaseSlavedStore): def process_replication(self, result): stream = result.get("pushers") if stream: - self._pushers_id_gen.advance(stream["position"]) + self._pushers_id_gen.advance(int(stream["position"])) stream = result.get("deleted_pushers") if stream: - self._pushers_id_gen.advance(stream["position"]) + self._pushers_id_gen.advance(int(stream["position"])) return super(SlavedPusherStore, self).process_replication(result) diff --git a/synapse/replication/slave/storage/receipts.py b/synapse/replication/slave/storage/receipts.py index b55d5dfd0..ec007516d 100644 --- a/synapse/replication/slave/storage/receipts.py +++ b/synapse/replication/slave/storage/receipts.py @@ -50,7 +50,7 @@ class SlavedReceiptsStore(BaseSlavedStore): def process_replication(self, result): stream = result.get("receipts") if stream: - self._receipts_id_gen.advance(stream["position"]) + self._receipts_id_gen.advance(int(stream["position"])) for row in stream["rows"]: room_id, receipt_type, user_id = row[1:4] self.invalidate_caches_for_receipt(room_id, receipt_type, user_id) From 782471b7e110165697fd3de19739802852cc2a91 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 13 May 2016 17:49:53 +0100 Subject: [PATCH 132/134] fix matrix.to URLs --- synapse/push/mailer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 2be294f52..2fd38a036 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -375,7 +375,7 @@ class Mailer(object): if self.app_name == "Vector": return "https://vector.im/beta/#/room/%s" % (room_id,) else: - return "https://matrix.to/#/room/%s" % (room_id,) + return "https://matrix.to/#/%s" % (room_id,) def make_notif_link(self, notif): # need /beta for Universal Links to work on iOS @@ -384,7 +384,7 @@ class Mailer(object): notif['room_id'], notif['event_id'] ) else: - return "https://matrix.to/#/room/%s/%s" % ( + return "https://matrix.to/#/%s/%s" % ( notif['room_id'], notif['event_id'] ) From 0e8364c7f538bed2aca15200839e5e9bdabe1667 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 13 May 2016 17:53:31 +0100 Subject: [PATCH 133/134] rogue img --- res/templates/notif.html | 1 - 1 file changed, 1 deletion(-) diff --git a/res/templates/notif.html b/res/templates/notif.html index 92e80352f..834840861 100644 --- a/res/templates/notif.html +++ b/res/templates/notif.html @@ -12,7 +12,6 @@ {% else %} {% endif %} - {% endif %} {% endif %} From c0112fabc2b580ad3838cf4c1f9143dc345c1cda Mon Sep 17 00:00:00 2001 From: Matrix Date: Fri, 13 May 2016 18:06:56 +0100 Subject: [PATCH 134/134] fix logo --- res/templates/notif_mail.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res/templates/notif_mail.html b/res/templates/notif_mail.html index 4034f3c60..42560fdfd 100644 --- a/res/templates/notif_mail.html +++ b/res/templates/notif_mail.html @@ -21,7 +21,7 @@ {% if app_name == "Vector" %} [Vector] {% else %} - [matrix] + [matrix] {% endif %}