From 231afe464aa9d5313de29ff569a1e2fea842fcb5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 22 Sep 2014 13:42:52 +0100 Subject: [PATCH 01/92] Add a deletions table --- synapse/storage/schema/delta/v4.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 synapse/storage/schema/delta/v4.sql diff --git a/synapse/storage/schema/delta/v4.sql b/synapse/storage/schema/delta/v4.sql new file mode 100644 index 000000000..1652ef292 --- /dev/null +++ b/synapse/storage/schema/delta/v4.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS deletions ( + event_id TEXT NOT NULL, + deletes TEXT NOT NULL, + CONSTRAINT ev_uniq UNIQUE (event_id) +); From 512f2cc9c49d585b566b3f5200b85748359b4b1e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 23 Sep 2014 12:22:14 +0100 Subject: [PATCH 02/92] Fix SYWEB-8 : Buggy tab-complete. The first red blink was caused by an uninitialised search index. There is no caching of entries, since this then wouldn't update if someone joined/left during the tab. Instead, set to search index to MAX_VALUE then fix it to a valid index AFTER the search is complete. Also ditched trailing space on ": ". --- webclient/room/room-directive.js | 59 ++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/webclient/room/room-directive.js b/webclient/room/room-directive.js index e033b003e..8db4cb5d9 100644 --- a/webclient/room/room-directive.js +++ b/webclient/room/room-directive.js @@ -21,39 +21,53 @@ angular.module('RoomController') return function (scope, element, attrs) { element.bind("keydown keypress", function (event) { // console.log("event: " + event.which); - if (event.which === 9) { + var TAB = 9; + var SHIFT = 16; + var keypressCode = event.which; + if (keypressCode === TAB) { if (!scope.tabCompleting) { // cache our starting text - // console.log("caching " + element[0].value); scope.tabCompleteOriginal = element[0].value; scope.tabCompleting = true; + scope.tabCompleteIndex = 0; } + // loop in the right direction if (event.shiftKey) { scope.tabCompleteIndex--; if (scope.tabCompleteIndex < 0) { - scope.tabCompleteIndex = 0; + // wrap to the last search match, and fix up to a real + // index value after we've matched + scope.tabCompleteIndex = Number.MAX_VALUE; } } else { scope.tabCompleteIndex++; } + var searchIndex = 0; var targetIndex = scope.tabCompleteIndex; var text = scope.tabCompleteOriginal; - // console.log("targetIndex: " + targetIndex + ", text=" + text); + // console.log("targetIndex: " + targetIndex + ", + // text=" + text); // FIXME: use the correct regexp to recognise userIDs + // XXX: I don't really know what the point of this is. You + // WANT to match freeform text given you want to match display + // names AND user IDs. Surely you just want to get the last + // word out of the input text and that's that? + // Am I missing something here? -- Kegan var search = /@?([a-zA-Z0-9_\-:\.]+)$/.exec(text); - if (targetIndex === 0) { - element[0].value = text; - - // Force angular to wake up and update the input ng-model by firing up input event + + if (targetIndex === 0) { // 0 is always the original text + element[0].value = text; + // Force angular to wake up and update the input ng-model + // by firing up input event angular.element(element[0]).triggerHandler('input'); } else if (search && search[1]) { - // console.log("search found: " + search); + // console.log("search found: " + search+" from "+text); var expansion; // FIXME: could do better than linear search here @@ -68,6 +82,7 @@ angular.module('RoomController') if (searchIndex < targetIndex) { // then search raw mxids angular.forEach(scope.members, function(item, name) { if (searchIndex < targetIndex) { + // === 1 because mxids are @username if (name.toLowerCase().indexOf(search[1].toLowerCase()) === 1) { expansion = name; searchIndex++; @@ -76,18 +91,22 @@ angular.module('RoomController') }); } - if (searchIndex === targetIndex) { - // xchat-style tab complete + if (searchIndex === targetIndex || + targetIndex === Number.MAX_VALUE) { + // xchat-style tab complete, add a colon if tab + // completing at the start of the text if (search[0].length === text.length) - expansion += " : "; + expansion += ": "; else expansion += " "; element[0].value = text.replace(/@?([a-zA-Z0-9_\-:\.]+)$/, expansion); // cancel blink element[0].className = ""; - - // Force angular to wake up and update the input ng-model by firing up input event - angular.element(element[0]).triggerHandler('input'); + if (targetIndex === Number.MAX_VALUE) { + // wrap the index around to the last index found + scope.tabCompleteIndex = searchIndex; + targetIndex = searchIndex; + } } else { // console.log("wrapped!"); @@ -97,17 +116,19 @@ angular.module('RoomController') }, 150); element[0].value = text; scope.tabCompleteIndex = 0; - - // Force angular to wake up and update the input ng-model by firing up input event - angular.element(element[0]).triggerHandler('input'); } + + // Force angular to wak up and update the input ng-model by + // firing up input event + angular.element(element[0]).triggerHandler('input'); } else { scope.tabCompleteIndex = 0; } + // prevent the default TAB operation (typically focus shifting) event.preventDefault(); } - else if (event.which !== 16 && scope.tabCompleting) { + else if (keypressCode !== SHIFT && scope.tabCompleting) { scope.tabCompleting = false; scope.tabCompleteIndex = 0; } From 997a016122218dd51ee0d2fb2ddac1c61fce455b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 23 Sep 2014 13:01:12 +0100 Subject: [PATCH 03/92] fix NPE --- webclient/components/matrix/event-handler-service.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 21066e3d9..9c19f306c 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -260,8 +260,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence) { if (event.content.prev !== event.content.membership) { memberChanges = "membership"; } - else if (event.prev_content.displayname !== - event.content.displayname) { + else if (event.prev_content && (event.prev_content.displayname !== event.content.displayname)) { memberChanges = "displayname"; } From 4847045259a631a2f1e03fd9751df301e6589560 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 23 Sep 2014 13:36:58 +0100 Subject: [PATCH 04/92] send messages to users from the home page (SYWEB-19) --- webclient/home/home-controller.js | 30 ++++++++++++++++++++++++++++++ webclient/home/home.html | 13 ++++++++++--- webclient/user/user-controller.js | 3 ++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/webclient/home/home-controller.js b/webclient/home/home-controller.js index e35219beb..a535d439d 100644 --- a/webclient/home/home-controller.js +++ b/webclient/home/home-controller.js @@ -42,6 +42,10 @@ angular.module('HomeController', ['matrixService', 'eventHandlerService', 'Recen displayName: "", avatarUrl: "" }; + + $scope.newChat = { + user: "" + }; var refresh = function() { @@ -112,6 +116,32 @@ angular.module('HomeController', ['matrixService', 'eventHandlerService', 'Recen } ); }; + + // FIXME: factor this out between user-controller and home-controller etc. + $scope.messageUser = function() { + + // FIXME: create a new room every time, for now + + matrixService.create(null, 'private').then( + function(response) { + // This room has been created. Refresh the rooms list + var room_id = response.data.room_id; + console.log("Created room with id: "+ room_id); + + matrixService.invite(room_id, $scope.newChat.user).then( + function() { + $scope.feedback = "Invite sent successfully"; + $scope.$parent.goToPage("/room/" + room_id); + }, + function(reason) { + $scope.feedback = "Failure: " + JSON.stringify(reason); + }); + }, + function(error) { + $scope.feedback = "Failure: " + JSON.stringify(error.data); + }); + }; + $scope.onInit = function() { // Load profile data diff --git a/webclient/home/home.html b/webclient/home/home.html index 5a1e18e1d..0af382916 100644 --- a/webclient/home/home.html +++ b/webclient/home/home.html @@ -17,7 +17,7 @@
{{ config.user_id }}
- +

Recent conversations


@@ -52,17 +52,24 @@
- + private
- +
+
+
+ + +
+
+
{{ feedback }} diff --git a/webclient/user/user-controller.js b/webclient/user/user-controller.js index 3940db668..0dbfa325d 100644 --- a/webclient/user/user-controller.js +++ b/webclient/user/user-controller.js @@ -38,7 +38,8 @@ angular.module('UserController', ['matrixService']) $scope.user.avatar_url = response.data.avatar_url; } ); - + + // FIXME: factor this out between user-controller and home-controller etc. $scope.messageUser = function() { // FIXME: create a new room every time, for now From e9c88ae4f42ad520814b139474a876706da22037 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Tue, 23 Sep 2014 15:18:45 +0200 Subject: [PATCH 05/92] Partial fix of SYWEB-28: If members do not have last_active_ago, compare their presence state to order them --- webclient/app-filter.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/webclient/app-filter.js b/webclient/app-filter.js index ee9374668..0d059e162 100644 --- a/webclient/app-filter.js +++ b/webclient/app-filter.js @@ -70,7 +70,23 @@ angular.module('matrixWebClient') }); filtered.sort(function (a, b) { - return ((a["last_active_ago"] || 10e10) > (b["last_active_ago"] || 10e10) ? 1 : -1); + // Sort members on their last_active_ago value + if (undefined !== a.last_active_ago || undefined !== b.last_active_ago) { + return ((a.last_active_ago || 10e10) > (b.last_active_ago || 10e10) ? 1 : -1); + } + else { + // If they do not have last_active_ago, sort them according to their presence state + // Online users go first amongs members who do not have last_active_ago + var presenceLevels = { + offline: 1, + unavailable: 2, + online: 4, + free_for_chat: 3 + }; + var aPresence = (a.presence in presenceLevels) ? presenceLevels[a.presence] : 0; + var bPresence = (b.presence in presenceLevels) ? presenceLevels[b.presence] : 0; + return bPresence - aPresence; + } }); return filtered; }; From 3a8a94448af334e57c5cfa3583b2c20739aeb613 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 23 Sep 2014 14:29:08 +0100 Subject: [PATCH 06/92] Allow a (hidden undocumented) key to m.login.recaptcha to specify a shared secret to allow bots to bypass the ReCAPTCHA test (SYN-60) --- synapse/config/captcha.py | 7 ++++++- synapse/rest/register.py | 23 ++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 8ebcfc362..4ed9070b9 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -24,6 +24,7 @@ class CaptchaConfig(Config): self.captcha_ip_origin_is_x_forwarded = ( args.captcha_ip_origin_is_x_forwarded ) + self.captcha_bypass_secret = args.captcha_bypass_secret @classmethod def add_arguments(cls, parser): @@ -43,4 +44,8 @@ class CaptchaConfig(Config): "--captcha_ip_origin_is_x_forwarded", type=bool, default=False, help="When checking captchas, use the X-Forwarded-For (XFF) header" + " as the client IP and not the actual client IP." - ) \ No newline at end of file + ) + group.add_argument( + "--captcha_bypass_secret", type=str, + help="A secret key used to bypass the captcha test entirely." + ) diff --git a/synapse/rest/register.py b/synapse/rest/register.py index af528a44f..f1354e4b7 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -142,6 +142,24 @@ class RegisterRestServlet(RestServlet): if not self.hs.config.enable_registration_captcha: raise SynapseError(400, "Captcha not required.") + yield self._check_recaptcha(request, register_json) + + session[LoginType.RECAPTCHA] = True # mark captcha as done + self._save_session(session) + defer.returnValue({ + "next": [LoginType.PASSWORD, LoginType.EMAIL_IDENTITY] + }) + + @defer.inlineCallbacks + def _check_recaptcha(self, request, register_json): + if "captcha_bypass_secret" in register_json: + if (register_json["captcha_bypass_secret"] == + self.hs.config.captcha_bypass_secret): + defer.returnValue(None) + else: + raise SynapseError(400, "Captcha bypass secret incorrect", + errcode=Codes.CAPTCHA_NEEDED) + challenge = None user_response = None try: @@ -166,11 +184,6 @@ class RegisterRestServlet(RestServlet): challenge, user_response ) - session[LoginType.RECAPTCHA] = True # mark captcha as done - self._save_session(session) - defer.returnValue({ - "next": [LoginType.PASSWORD, LoginType.EMAIL_IDENTITY] - }) @defer.inlineCallbacks def _do_email_identity(self, request, register_json, session): From 5f16439752fa6ff9b452cac86fbbb07a12ae44f7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 23 Sep 2014 15:16:47 +0100 Subject: [PATCH 07/92] Make sure the config actually /has/ a captcha_bypass_secret set before trying to compare it --- synapse/rest/register.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/rest/register.py b/synapse/rest/register.py index f1354e4b7..3b07a127a 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -153,8 +153,9 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def _check_recaptcha(self, request, register_json): if "captcha_bypass_secret" in register_json: - if (register_json["captcha_bypass_secret"] == - self.hs.config.captcha_bypass_secret): + if (self.hs.config.captcha_bypass_secret is not None and + register_json["captcha_bypass_secret"] == + self.hs.config.captcha_bypass_secret): defer.returnValue(None) else: raise SynapseError(400, "Captcha bypass secret incorrect", From 537c7e1137684c97676bbb74d59cd93c5bf1aad1 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 23 Sep 2014 15:18:59 +0100 Subject: [PATCH 08/92] Config values are almost never 'None', but they might be empty string. Detect their presence by truth --- synapse/rest/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 3b07a127a..66cef26ad 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -153,7 +153,7 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def _check_recaptcha(self, request, register_json): if "captcha_bypass_secret" in register_json: - if (self.hs.config.captcha_bypass_secret is not None and + if (self.hs.config.captcha_bypass_secret and register_json["captcha_bypass_secret"] == self.hs.config.captcha_bypass_secret): defer.returnValue(None) From 78af6bbb981c41e5509c99454deb7205c31bf964 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 23 Sep 2014 15:28:32 +0100 Subject: [PATCH 09/92] Add m.room.deletion. If an event is deleted it will be returned to clients 'pruned', i.e. all client specified keys will be removed. --- synapse/api/events/__init__.py | 6 +++-- synapse/api/events/factory.py | 4 ++- synapse/api/events/room.py | 9 +++++++ synapse/rest/room.py | 38 ++++++++++++++++++++++++++++- synapse/storage/__init__.py | 29 +++++++++++++++++++--- synapse/storage/_base.py | 30 ++++++++++++++++++++--- synapse/storage/roommember.py | 13 +++++++--- synapse/storage/schema/delta/v4.sql | 6 +++-- synapse/storage/stream.py | 30 +++++++++++++++++++---- 9 files changed, 144 insertions(+), 21 deletions(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 0cee19685..910c990b3 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -22,7 +22,7 @@ def serialize_event(hs, e): if not isinstance(e, SynapseEvent): return e - d = e.get_dict() + d = {k: v for k, v in e.get_dict().items() if v is not None or v is not False} if "age_ts" in d: d["age"] = int(hs.get_clock().time_msec()) - d["age_ts"] del d["age_ts"] @@ -58,17 +58,19 @@ class SynapseEvent(JsonEncodedObject): "required_power_level", "age_ts", "prev_content", + "prev_state", + "pruned", ] internal_keys = [ "is_state", "prev_events", - "prev_state", "depth", "destinations", "origin", "outlier", "power_level", + "deleted", ] required_keys = [ diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index d3d96d73e..c65ea8372 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -17,7 +17,8 @@ from synapse.api.events.room import ( RoomTopicEvent, MessageEvent, RoomMemberEvent, FeedbackEvent, InviteJoinEvent, RoomConfigEvent, RoomNameEvent, GenericEvent, RoomPowerLevelsEvent, RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, - RoomCreateEvent, RoomAddStateLevelEvent, RoomSendEventLevelEvent + RoomCreateEvent, RoomAddStateLevelEvent, RoomSendEventLevelEvent, + RoomDeletionEvent, ) from synapse.util.stringutils import random_string @@ -39,6 +40,7 @@ class EventFactory(object): RoomAddStateLevelEvent, RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent, + RoomDeletionEvent, ] def __init__(self, hs): diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index 3a4dbc58c..986139555 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -180,3 +180,12 @@ class RoomAliasesEvent(SynapseStateEvent): def get_content_template(self): return {} + + +class RoomDeletionEvent(SynapseEvent): + TYPE = "m.room.deletion" + + valid_keys = SynapseEvent.valid_keys + ["deletes"] + + def get_content_template(self): + return {} diff --git a/synapse/rest/room.py b/synapse/rest/room.py index ecb1e346d..85a1d2eae 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -19,7 +19,7 @@ from twisted.internet import defer from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig -from synapse.api.events.room import RoomMemberEvent +from synapse.api.events.room import RoomMemberEvent, RoomDeletionEvent from synapse.api.constants import Membership import json @@ -430,6 +430,41 @@ class RoomMembershipRestServlet(RestServlet): self.txns.store_client_transaction(request, txn_id, response) defer.returnValue(response) +class RoomDeleteEventRestServlet(RestServlet): + def register(self, http_server): + PATTERN = ("/rooms/(?P[^/]*)/delete/(?P[^/]*)") + register_txn_path(self, PATTERN, http_server) + + @defer.inlineCallbacks + def on_POST(self, request, room_id, event_id): + user = yield self.auth.get_user_by_req(request) + content = _parse_json(request) + + event = self.event_factory.create_event( + etype=RoomDeletionEvent.TYPE, + room_id=urllib.unquote(room_id), + user_id=user.to_string(), + content=content, + deletes=event_id, + ) + + msg_handler = self.handlers.message_handler + yield msg_handler.send_message(event) + + defer.returnValue((200, {"event_id": event.event_id})) + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, event_id, txn_id): + try: + defer.returnValue(self.txns.get_client_transaction(request, txn_id)) + except KeyError: + pass + + response = yield self.on_POST(request, room_id, event_id) + + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) + def _parse_json(request): try: @@ -485,3 +520,4 @@ def register_servlets(hs, http_server): PublicRoomListRestServlet(hs).register(http_server) RoomStateRestServlet(hs).register(http_server) RoomInitialSyncRestServlet(hs).register(http_server) + RoomDeleteEventRestServlet(hs).register(http_server) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 66658f672..672ed6971 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -24,6 +24,7 @@ from synapse.api.events.room import ( RoomAddStateLevelEvent, RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent, + RoomDeletionEvent, ) from synapse.util.logutils import log_function @@ -61,7 +62,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 3 +SCHEMA_VERSION = 4 class _RollbackButIsFineException(Exception): @@ -182,6 +183,8 @@ class DataStore(RoomMemberStore, RoomStore, self._store_send_event_level(txn, event) elif event.type == RoomOpsPowerLevelsEvent.TYPE: self._store_ops_level(txn, event) + elif event.type == RoomDeletionEvent.TYPE: + self._store_deletion(txn, event) vals = { "topological_ordering": event.depth, @@ -203,7 +206,7 @@ class DataStore(RoomMemberStore, RoomStore, unrec = { k: v for k, v in event.get_full_dict().items() - if k not in vals.keys() + if k not in vals.keys() and k is not "deleted" } vals["unrecognized_keys"] = json.dumps(unrec) @@ -241,14 +244,32 @@ class DataStore(RoomMemberStore, RoomStore, } ) + def _store_deletion(self, txn, event): + event_id = event.event_id + deletes = event.deletes + + # We check if this new delete deletes an old delete or has been + # deleted by a previous delete that we received out of order. + sql = "SELECT * FROM deletions WHERE event_id = ? OR deletes = ?" + txn.execute(sql, (deletes, event_id)) + + if txn.fetchall(): + sql = "DELETE FROM deletions WHERE event_id = ? OR deletes = ?" + txn.execute(sql, (deletes, event_id, )) + else: + sql = "INSERT INTO deletions (event_id, deletes) VALUES (?,?)" + txn.execute(sql, (event_id, deletes)) + @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): sql = ( - "SELECT e.* FROM events as e " + "SELECT e.*, (%(deleted)s) AS deleted FROM events as e " "INNER JOIN current_state_events as c ON e.event_id = c.event_id " "INNER JOIN state_events as s ON e.event_id = s.event_id " "WHERE c.room_id = ? " - ) + ) % { + "deleted": "e.event_id IN (SELECT deletes FROM deletions)", + } if event_type: sql += " AND s.type = ? AND s.state_key = ? " diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 76ed7d06f..3aa610c85 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -17,6 +17,7 @@ import logging from twisted.internet import defer from synapse.api.errors import StoreError +from synapse.api.events.utils import prune_event from synapse.util.logutils import log_function import collections @@ -345,7 +346,7 @@ class SQLBaseStore(object): return self.runInteraction(func) def _parse_event_from_row(self, row_dict): - d = copy.deepcopy({k: v for k, v in row_dict.items() if v}) + d = copy.deepcopy({k: v for k, v in row_dict.items()}) d.pop("stream_ordering", None) d.pop("topological_ordering", None) @@ -373,8 +374,8 @@ class SQLBaseStore(object): sql = "SELECT * FROM events WHERE event_id = ?" for ev in events: - if hasattr(ev, "prev_state"): - # Load previous state_content. + if hasattr(ev, "prev_state"): + # Load previous state_content. # TODO: Should we be pulling this out above? cursor = txn.execute(sql, (ev.prev_state,)) prevs = self.cursor_to_dict(cursor) @@ -382,8 +383,31 @@ class SQLBaseStore(object): prev = self._parse_event_from_row(prevs[0]) ev.prev_content = prev.content + if not hasattr(ev, "deleted"): + logger.debug("Doesn't have deleted key: %s", ev) + ev.deleted = self._has_been_deleted_txn(txn, ev) + + if ev.deleted: + # Get the deletion event. + sql = "SELECT * FROM events WHERE event_id = ?" + txn.execute(sql, (ev.deleted,)) + + del_evs = self._parse_events_txn( + txn, self.cursor_to_dict(txn) + ) + + if del_evs: + prune_event(ev) + ev.pruned = del_evs[0] + return events + def _has_been_deleted_txn(self, txn, event): + sql = "SELECT * FROM deletions WHERE deletes = ?" + txn.execute(sql, (event.event_id,)) + return len(txn.fetchall()) > 0 + + class Table(object): """ A base class used to store information about a particular table. """ diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 04b4067d0..97222da57 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -182,14 +182,21 @@ class RoomMemberStore(SQLBaseStore): ) def _get_members_query_txn(self, txn, where_clause, where_values): + del_sql = ( + "SELECT event_id FROM deletions WHERE deletes = e.event_id" + ) + sql = ( - "SELECT e.* FROM events as e " + "SELECT e.*, (%(deleted)s) AS deleted FROM events as e " "INNER JOIN room_memberships as m " "ON e.event_id = m.event_id " "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " - "WHERE %s " - ) % (where_clause,) + "WHERE %(where)s " + ) % { + "deleted": del_sql, + "where": where_clause, + } txn.execute(sql, where_values) rows = self.cursor_to_dict(txn) diff --git a/synapse/storage/schema/delta/v4.sql b/synapse/storage/schema/delta/v4.sql index 1652ef292..2e2635317 100644 --- a/synapse/storage/schema/delta/v4.sql +++ b/synapse/storage/schema/delta/v4.sql @@ -1,5 +1,7 @@ CREATE TABLE IF NOT EXISTS deletions ( event_id TEXT NOT NULL, - deletes TEXT NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) + deletes TEXT NOT NULL ); + +CREATE INDEX IF NOT EXISTS deletions_event_id ON deletions (event_id); +CREATE INDEX IF NOT EXISTS deletions_deletes ON deletions (deletes); diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index a76fecf24..aaac0aae3 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -157,6 +157,10 @@ class StreamStore(SQLBaseStore): "WHERE m.user_id = ? " ) + del_sql = ( + "SELECT event_id FROM deletions WHERE deletes = e.event_id" + ) + if limit: limit = max(limit, MAX_STREAM_SIZE) else: @@ -171,13 +175,14 @@ class StreamStore(SQLBaseStore): return sql = ( - "SELECT * FROM events as e WHERE " + "SELECT *, (%(deleted)s) AS deleted FROM events AS e WHERE " "((room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "AND e.outlier = 0 " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { + "deleted": del_sql, "current": current_room_membership_sql, "invites": membership_sql, "limit": limit @@ -224,11 +229,20 @@ class StreamStore(SQLBaseStore): else: limit_str = "" + del_sql = ( + "SELECT event_id FROM deletions WHERE deletes = events.event_id" + ) + sql = ( - "SELECT * FROM events " + "SELECT *, (%(deleted)s) AS deleted FROM events " "WHERE outlier = 0 AND room_id = ? AND %(bounds)s " "ORDER BY topological_ordering %(order)s, stream_ordering %(order)s %(limit)s " - ) % {"bounds": bounds, "order": order, "limit": limit_str} + ) % { + "deleted": del_sql, + "bounds": bounds, + "order": order, + "limit": limit_str + } rows = yield self._execute_and_decode( sql, @@ -257,11 +271,17 @@ class StreamStore(SQLBaseStore): with_feedback=False): # TODO (erikj): Handle compressed feedback + del_sql = ( + "SELECT event_id FROM deletions WHERE deletes = events.event_id" + ) + sql = ( - "SELECT * FROM events " + "SELECT *, (%(deleted)s) AS deleted FROM events " "WHERE room_id = ? AND stream_ordering <= ? " "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) + ) % { + "deleted": del_sql, + } rows = yield self._execute_and_decode( sql, From b99f6eb904c9614aacd3532196be410341d0640b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 23 Sep 2014 15:29:27 +0100 Subject: [PATCH 10/92] Make sure we don't persist the 'pruned' key --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 672ed6971..46d420b63 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -206,7 +206,7 @@ class DataStore(RoomMemberStore, RoomStore, unrec = { k: v for k, v in event.get_full_dict().items() - if k not in vals.keys() and k is not "deleted" + if k not in vals.keys() and k not in ["deleted", "pruned"] } vals["unrecognized_keys"] = json.dumps(unrec) From 0c4ae63ad52a0bbe6a7a0cc519686b57f655e254 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 23 Sep 2014 15:35:58 +0100 Subject: [PATCH 11/92] Implemented /rooms/$roomid/state API. --- synapse/handlers/message.py | 16 ++++++++++++++++ synapse/rest/room.py | 13 +++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 14fae689f..317ef2c80 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -232,6 +232,22 @@ class MessageHandler(BaseHandler): # store message in db yield self._on_new_room_event(event, snapshot) + @defer.inlineCallbacks + def get_state_events(self, user_id, room_id): + """Retrieve all state events for a given room. + + Args: + user_id(str): The user requesting state events. + room_id(str): The room ID to get all state events from. + Returns: + A list of dicts representing state events. [{}, {}, {}] + """ + yield self.auth.check_joined_room(room_id, user_id) + + # TODO: This is duplicating logic from snapshot_all_rooms + current_state = yield self.store.get_current_state(room_id) + defer.returnValue([self.hs.serialize_event(c) for c in current_state]) + @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, feedback=False): diff --git a/synapse/rest/room.py b/synapse/rest/room.py index ecb1e346d..cf2e7af2e 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -329,12 +329,13 @@ class RoomStateRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): user = yield self.auth.get_user_by_req(request) - # TODO: Get all the current state for this room and return in the same - # format as initial sync, that is: - # [ - # { state event }, { state event } - # ] - defer.returnValue((200, [])) + handler = self.handlers.message_handler + # Get all the current state for this room + events = yield handler.get_state_events( + room_id=urllib.unquote(room_id), + user_id=user.to_string(), + ) + defer.returnValue((200, events)) # TODO: Needs unit testing From 932b376b4e3d1992268274f25e2343f6e81d93f8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 23 Sep 2014 15:37:32 +0100 Subject: [PATCH 12/92] Add prune_event method --- synapse/api/events/utils.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 synapse/api/events/utils.py diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py new file mode 100644 index 000000000..dfefb2662 --- /dev/null +++ b/synapse/api/events/utils.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .room import RoomMemberEvent + +def prune_event(event): + """ Prunes the given event of all keys we don't know about or think could + potentially be dodgy. + + This is used when we "delete" an event. We want to remove all fields that + the user has specified, but we do want to keep necessary information like + type, state_key etc. + """ + + # Remove all extraneous fields. + event.unrecognized_keys = {} + + if event.type == RoomMemberEvent.TYPE: + new_content = { + "membership": event.content["membership"] + } + else: + new_content = {} + + event.content = new_content + + return event From 2771efb51c9cfca06cd2906724ee7a1fa4fa01d1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 23 Sep 2014 15:39:04 +0100 Subject: [PATCH 13/92] Update API docs to include notes on /rooms/$roomid/state --- docs/client-server/swagger_matrix/api-docs-rooms | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/client-server/swagger_matrix/api-docs-rooms b/docs/client-server/swagger_matrix/api-docs-rooms index 0e1fa452a..b941e5813 100644 --- a/docs/client-server/swagger_matrix/api-docs-rooms +++ b/docs/client-server/swagger_matrix/api-docs-rooms @@ -639,7 +639,7 @@ { "method": "GET", "summary": "Get a list of all the current state events for this room.", - "notes": "NOT YET IMPLEMENTED.", + "notes": "This is equivalent to the events returned under the 'state' key for this room in /initialSync.", "type": "array", "items": { "$ref": "Event" From c03176af59adfe0d60ffd8beb8bf262b4563d20b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 23 Sep 2014 15:58:44 +0100 Subject: [PATCH 14/92] Send an HMAC(SHA1) protecting the User ID for the ReCAPTCHA bypass, rather than simply the secret itself, so it's useless if that HMAC leaks --- synapse/rest/register.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 66cef26ad..14d1ab018 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -21,6 +21,8 @@ from synapse.api.constants import LoginType from base import RestServlet, client_path_pattern import synapse.util.stringutils as stringutils +from hashlib import sha1 +import hmac import json import logging import urllib @@ -142,7 +144,7 @@ class RegisterRestServlet(RestServlet): if not self.hs.config.enable_registration_captcha: raise SynapseError(400, "Captcha not required.") - yield self._check_recaptcha(request, register_json) + yield self._check_recaptcha(request, register_json, session) session[LoginType.RECAPTCHA] = True # mark captcha as done self._save_session(session) @@ -151,14 +153,27 @@ class RegisterRestServlet(RestServlet): }) @defer.inlineCallbacks - def _check_recaptcha(self, request, register_json): - if "captcha_bypass_secret" in register_json: - if (self.hs.config.captcha_bypass_secret and - register_json["captcha_bypass_secret"] == - self.hs.config.captcha_bypass_secret): + def _check_recaptcha(self, request, register_json, session): + if ("captcha_bypass_hmac" in register_json and + self.hs.config.captcha_bypass_secret): + if "user" not in register_json: + raise SynapseError(400, "Captcha bypass needs 'user'") + + want = hmac.new( + key=self.hs.config.captcha_bypass_secret, + msg=register_json["user"], + digestmod=sha1, + ).hexdigest() + + # str() because otherwise hmac complains that 'unicode' does not + # have the buffer interface + got = str(register_json["captcha_bypass_hmac"]) + + if hmac.compare_digest(want, got): + session["user"] = register_json["user"] defer.returnValue(None) else: - raise SynapseError(400, "Captcha bypass secret incorrect", + raise SynapseError(400, "Captcha bypass HMAC incorrect", errcode=Codes.CAPTCHA_NEEDED) challenge = None @@ -209,6 +224,10 @@ class RegisterRestServlet(RestServlet): # captcha should've been done by this stage! raise SynapseError(400, "Captcha is required.") + if ("user" in session and "user" in register_json and + session["user"] != register_json["user"]): + raise SynapseError(400, "Cannot change user ID during registration") + password = register_json["password"].encode("utf-8") desired_user_id = (register_json["user"].encode("utf-8") if "user" in register_json else None) From c9f73bd3258e40f5728a9910f9f5051240ca682e Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 23 Sep 2014 16:11:12 +0100 Subject: [PATCH 15/92] fix one cause of SYWEB-53 --- webclient/components/matrix/matrix-filter.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/webclient/components/matrix/matrix-filter.js b/webclient/components/matrix/matrix-filter.js index 328e3a708..1a64dc126 100644 --- a/webclient/components/matrix/matrix-filter.js +++ b/webclient/components/matrix/matrix-filter.js @@ -49,15 +49,12 @@ angular.module('matrixFilter', []) if (member.state_key !== user_id) { if (member.state_key in $rootScope.presence) { - // If the user is available in presence, use the displayname there + // If the user is listed in presence, use the displayname there // as it is the most uptodate - roomName = $rootScope.presence[member.state_key].content.displayname; + roomName = $rootScope.presence[member.state_key].content.displayname || member.state_key; } - else if (member.content.displayname) { - roomName = member.content.displayname; - } - else { - roomName = member.state_key; + else { + roomName = member.content.displayname || member.state_key; } } } From 7d94913efb799654a8e51676bcb9813f2e453514 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 23 Sep 2014 16:12:06 +0100 Subject: [PATCH 16/92] remove old commented-out code --- webclient/components/matrix/matrix-service.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index 069e02e93..79781f5d8 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -697,11 +697,10 @@ angular.module('matrixService', []) createRoomIdToAliasMapping: function(roomId, alias) { roomIdToAlias[roomId] = alias; aliasToRoomId[alias] = roomId; - // localStorage.setItem(MAPPING_PREFIX+roomId, alias); }, getRoomIdToAliasMapping: function(roomId) { - var alias = roomIdToAlias[roomId]; // was localStorage.getItem(MAPPING_PREFIX+roomId) + var alias = roomIdToAlias[roomId]; //console.log("looking for alias for " + roomId + "; found: " + alias); return alias; }, From e4e8ad6780abd63f766db13974ddc3d3c4f7528f Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Tue, 23 Sep 2014 17:25:37 +0200 Subject: [PATCH 17/92] SYWEB-28: Fixed weird members list ordering: sort members on their last activity absolute time --- webclient/app-filter.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/webclient/app-filter.js b/webclient/app-filter.js index 0d059e162..9443446c9 100644 --- a/webclient/app-filter.js +++ b/webclient/app-filter.js @@ -70,9 +70,16 @@ angular.module('matrixWebClient') }); filtered.sort(function (a, b) { - // Sort members on their last_active_ago value - if (undefined !== a.last_active_ago || undefined !== b.last_active_ago) { - return ((a.last_active_ago || 10e10) > (b.last_active_ago || 10e10) ? 1 : -1); + // Sort members on their last_active absolute time + var aLastActiveTS = 0, bLastActiveTS = 0; + if (undefined !== a.last_active_ago) { + aLastActiveTS = a.last_updated - a.last_active_ago; + } + if (undefined !== b.last_active_ago) { + bLastActiveTS = b.last_updated - b.last_active_ago; + } + if (aLastActiveTS || bLastActiveTS) { + return bLastActiveTS - aLastActiveTS; } else { // If they do not have last_active_ago, sort them according to their presence state From a7420ff2b5b2880bded6d13b548607028113397f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 23 Sep 2014 16:53:27 +0100 Subject: [PATCH 18/92] Fix SYWEB-72 : Improve performance when typing. Swapped ng-keydown to a directive, which does the same thing (check if up/down arrow then call history.goUp/goDown). This has *dramatically* improved performance when typing in rooms which have lots (>100) of messages loaded. --- webclient/room/room-directive.js | 15 +++++++++++++++ webclient/room/room.html | 3 +-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/webclient/room/room-directive.js b/webclient/room/room-directive.js index 8db4cb5d9..d5108b5f1 100644 --- a/webclient/room/room-directive.js +++ b/webclient/room/room-directive.js @@ -135,6 +135,21 @@ angular.module('RoomController') }); }; }]) +.directive('commandHistory', [ function() { + return function (scope, element, attrs) { + element.bind("keydown keypress", function (event) { + var keycodePressed = event.which; + var UP_ARROW = 38; + var DOWN_ARROW = 40; + if (keycodePressed === UP_ARROW) { + scope.history.goUp(event); + } + else if (keycodePressed === DOWN_ARROW) { + scope.history.goDown(event); + } + }); + } +}]) // A directive to anchor the scroller position at the bottom when the browser is resizing. // When the screen resizes, the bottom of the element remains the same, not the top. diff --git a/webclient/room/room.html b/webclient/room/room.html index c807e2afe..278624623 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -163,8 +163,7 @@