From 3dfa84bec81e20cfde887fd87114607f58505edf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Aug 2014 12:52:39 +0100 Subject: [PATCH 001/112] Convert im schema to a 'one' table structure --- synapse/storage/schema/im.sql | 75 +++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 77096546b..e09ff6b64 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -12,43 +12,48 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +CREATE TABLE IF NOT EXISTS events( + ordering INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL, + event_type TEXT NOT NULL, + sender TEXT, + room_id TEXT, + content TEXT, + unrecognized_keys TEXT +); + +CREATE TABLE IF NOT EXISTS state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + event_type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT +); + +CREATE TABLE IF NOT EXISTS current_state( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, +); + +CREATE TABLE IF NOT EXISTS room_memberships( + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS feedback( + event_id TEXT NOT NULL, + feedback_type TEXT, + fb_sender_id TEXT, + room_id TEXT, + content TEXT +); + CREATE TABLE IF NOT EXISTS rooms( room_id TEXT PRIMARY KEY NOT NULL, is_public INTEGER, creator TEXT ); - -CREATE TABLE IF NOT EXISTS room_memberships( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id TEXT NOT NULL, -- no foreign key to users table, it could be an id belonging to another home server - sender TEXT NOT NULL, - room_id TEXT NOT NULL, - membership TEXT NOT NULL, - content TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS messages( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id TEXT, - room_id TEXT, - msg_id TEXT, - content TEXT -); - -CREATE TABLE IF NOT EXISTS feedback( - id INTEGER PRIMARY KEY AUTOINCREMENT, - content TEXT, - feedback_type TEXT, - fb_sender_id TEXT, - msg_id TEXT, - room_id TEXT, - msg_sender_id TEXT -); - -CREATE TABLE IF NOT EXISTS room_data( - id INTEGER PRIMARY KEY AUTOINCREMENT, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - content TEXT -); From 336987bb8debec9dcdbe27d59ce3889b39f86dbb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Aug 2014 16:27:14 +0100 Subject: [PATCH 002/112] Initial stab at refactoring the SQL tables, including rejigging some of the storage layer. --- synapse/storage/__init__.py | 111 ++++++++++++++++++++++------------ synapse/storage/_base.py | 17 +++++- synapse/storage/feedback.py | 67 ++++++-------------- synapse/storage/message.py | 81 ------------------------- synapse/storage/roomdata.py | 85 -------------------------- synapse/storage/schema/im.sql | 11 ++-- 6 files changed, 113 insertions(+), 259 deletions(-) delete mode 100644 synapse/storage/message.py delete mode 100644 synapse/storage/roomdata.py diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 3c27428c0..4fcef45e9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -46,50 +46,83 @@ class DataStore(RoomDataStore, RoomMemberStore, MessageStore, RoomStore, self.event_factory = hs.get_event_factory() self.hs = hs + @defer.inlineCallbacks def persist_event(self, event): - if event.type == MessageEvent.TYPE: - return self.store_message( - user_id=event.user_id, - room_id=event.room_id, - msg_id=event.msg_id, - content=json.dumps(event.content) - ) - elif event.type == RoomMemberEvent.TYPE: - return self.store_room_member( - user_id=event.target_user_id, - sender=event.user_id, - room_id=event.room_id, - content=event.content, - membership=event.content["membership"] - ) + if event.type == RoomMemberEvent.TYPE: + yield self._store_room_member(event) elif event.type == FeedbackEvent.TYPE: - return self.store_feedback( - room_id=event.room_id, - msg_id=event.msg_id, - msg_sender_id=event.msg_sender_id, - fb_sender_id=event.user_id, - fb_type=event.feedback_type, - content=json.dumps(event.content) - ) - elif event.type == RoomTopicEvent.TYPE: - return self.store_room_data( - room_id=event.room_id, - etype=event.type, - state_key=event.state_key, - content=json.dumps(event.content) - ) + yield self._store_feedback(event) elif event.type == RoomConfigEvent.TYPE: - if "visibility" in event.content: - visibility = event.content["visibility"] - return self.store_room_config( - room_id=event.room_id, - visibility=visibility - ) + yield self._store_room_config(event) + self._store_event(event) + + @defer.inlineCallbacks + def get_event(self, event_id): + events_dict = yield self._simple_select_one( + "events", + {"event_id": event_id}, + [ + "event_id", + "type", + "sender", + "room_id", + "content", + "unrecognized_keys" + ], + ) + + event = self._parse_event_from_row(events_dict) + defer.returnValue(event) + + @defer.inlineCallbacks + def _store_event(self, event): + vals = { + "event_id": event.event_id, + "event_type", event.type, + "sender": event.user_id, + "room_id": event.room_id, + "content": event.content, + } + + unrec = {k: v for k, v in event.get_full_dict() if k not in vals.keys()} + val["unrecognized_keys"] = unrec + + yield self._simple_insert("events", vals) + + if hasattr(event, "state_key"): + vals = { + "event_id": event.event_id, + "room_id": event.room_id, + "event_type": event.event_type, + "state_key": event.state_key, + } + + if hasattr(event, "prev_state"): + vals["prev_state"] = event.prev_state + + yield self._simple_insert("state_events", vals) + + # TODO (erikj): We also need to update the current state table? + + @defer.inlineCallbacks + def get_current_state(room_id, event_type=None, state_key="") + sql = ( + "SELECT e.* FROM events as e" + "INNER JOIN current_state as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) + + if event_type: + sql += " s.type = ? AND s.state_key = ? " + args = (room_id, event_type, state_key) else: - raise NotImplementedError( - "Don't know how to persist type=%s" % event.type - ) + args = (room_id, ) + + results = yield self._execute_query(sql, *args) + + defer.returnValue( def schema_path(schema): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 65f691ead..03537b7e3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging from twisted.internet import defer @@ -20,6 +19,8 @@ from twisted.internet import defer from synapse.api.errors import StoreError import collections +import json + logger = logging.getLogger(__name__) @@ -28,6 +29,7 @@ class SQLBaseStore(object): def __init__(self, hs): self._db_pool = hs.get_db_pool() + self.event_factory = hs.get_event_factory() def cursor_to_dict(self, cursor): """Converts a SQL cursor into an list of dicts. @@ -63,6 +65,9 @@ class SQLBaseStore(object): return decoder(cursor) return self._db_pool.runInteraction(interaction) + def _execut_query(self, query, *args): + return self._execute(self.cursor_to_dict, *args) + # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. @@ -279,6 +284,16 @@ class SQLBaseStore(object): return self._db_pool.runInteraction(func) + def _parse_event_from_row(self, row_dict): + d = copy.deepcopy({k: v for k, v in row.items() if v}) + d.update(json.loads(row["unrecognized_keys"])) + del d["unrecognized_keys"] + + return self.event_factory.create_event( + etype=d["type"], + **d + ) + class Table(object): """ A base class used to store information about a particular table. diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py index 9bd562c76..fc93f92e1 100644 --- a/synapse/storage/feedback.py +++ b/synapse/storage/feedback.py @@ -22,54 +22,27 @@ import json class FeedbackStore(SQLBaseStore): - def store_feedback(self, room_id, msg_id, msg_sender_id, - fb_sender_id, fb_type, content): - return self._simple_insert(FeedbackTable.table_name, dict( - room_id=room_id, - msg_id=msg_id, - msg_sender_id=msg_sender_id, - fb_sender_id=fb_sender_id, - fb_type=fb_type, - content=content, - )) + def _store_feedback(self, event): + return self._simple_insert("feedback", { + "event_id": event.event_id, + "feedback_type": event.feedback_type, + "room_id": event.room_id, + "target_event_id": event.target_event, + }) - def get_feedback(self, room_id=None, msg_id=None, msg_sender_id=None, - fb_sender_id=None, fb_type=None): - query = FeedbackTable.select_statement( - "msg_sender_id = ? AND room_id = ? AND msg_id = ? " + - "AND fb_sender_id = ? AND feedback_type = ? " + - "ORDER BY id DESC LIMIT 1") - return self._execute( - FeedbackTable.decode_single_result, - query, msg_sender_id, room_id, msg_id, fb_sender_id, fb_type, + @defer.inlineCallback + def get_feedback_for_event(self, event_id): + sql = ( + "SELECT events.* FROM events INNER JOIN feedback " + "ON events.event_id = feedback.event_id " + "WHERE feedback.target_event_id = ? " ) - def get_max_feedback_id(self): - return self._simple_max_id(FeedbackTable.table_name) + rows = yield self._execute_query(sql, event_id) - -class FeedbackTable(Table): - table_name = "feedback" - - fields = [ - "id", - "content", - "feedback_type", - "fb_sender_id", - "msg_id", - "room_id", - "msg_sender_id" - ] - - class EntryType(collections.namedtuple("FeedbackEntry", fields)): - - def as_event(self, event_factory): - return event_factory.create_event( - etype=FeedbackEvent.TYPE, - room_id=self.room_id, - msg_id=self.msg_id, - msg_sender_id=self.msg_sender_id, - user_id=self.fb_sender_id, - feedback_type=self.feedback_type, - content=json.loads(self.content), - ) + defer.returnValue( + [ + self._parse_event_from_row(r) + for r in rows + ] + ) diff --git a/synapse/storage/message.py b/synapse/storage/message.py deleted file mode 100644 index 7bb69c138..000000000 --- a/synapse/storage/message.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 matrix.org -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._base import SQLBaseStore, Table -from synapse.api.events.room import MessageEvent - -import collections -import json - - -class MessageStore(SQLBaseStore): - - def get_message(self, user_id, room_id, msg_id): - """Get a message from the store. - - Args: - user_id (str): The ID of the user who sent the message. - room_id (str): The room the message was sent in. - msg_id (str): The unique ID for this user/room combo. - """ - query = MessagesTable.select_statement( - "user_id = ? AND room_id = ? AND msg_id = ? " + - "ORDER BY id DESC LIMIT 1") - return self._execute( - MessagesTable.decode_single_result, - query, user_id, room_id, msg_id, - ) - - def store_message(self, user_id, room_id, msg_id, content): - """Store a message in the store. - - Args: - user_id (str): The ID of the user who sent the message. - room_id (str): The room the message was sent in. - msg_id (str): The unique ID for this user/room combo. - content (str): The content of the message (JSON) - """ - return self._simple_insert(MessagesTable.table_name, dict( - user_id=user_id, - room_id=room_id, - msg_id=msg_id, - content=content, - )) - - def get_max_message_id(self): - return self._simple_max_id(MessagesTable.table_name) - - -class MessagesTable(Table): - table_name = "messages" - - fields = [ - "id", - "user_id", - "room_id", - "msg_id", - "content" - ] - - class EntryType(collections.namedtuple("MessageEntry", fields)): - - def as_event(self, event_factory): - return event_factory.create_event( - etype=MessageEvent.TYPE, - room_id=self.room_id, - user_id=self.user_id, - msg_id=self.msg_id, - content=json.loads(self.content), - ) diff --git a/synapse/storage/roomdata.py b/synapse/storage/roomdata.py deleted file mode 100644 index cc04d1ba1..000000000 --- a/synapse/storage/roomdata.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 matrix.org -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._base import SQLBaseStore, Table - -import collections -import json - - -class RoomDataStore(SQLBaseStore): - - """Provides various CRUD operations for Room Events. """ - - def get_room_data(self, room_id, etype, state_key=""): - """Retrieve the data stored under this type and state_key. - - Args: - room_id (str) - etype (str) - state_key (str) - Returns: - namedtuple: Or None if nothing exists at this path. - """ - query = RoomDataTable.select_statement( - "room_id = ? AND type = ? AND state_key = ? " - "ORDER BY id DESC LIMIT 1" - ) - return self._execute( - RoomDataTable.decode_single_result, - query, room_id, etype, state_key, - ) - - def store_room_data(self, room_id, etype, state_key="", content=None): - """Stores room specific data. - - Args: - room_id (str) - etype (str) - state_key (str) - data (str)- The data to store for this path in JSON. - Returns: - The store ID for this data. - """ - return self._simple_insert(RoomDataTable.table_name, dict( - etype=etype, - state_key=state_key, - room_id=room_id, - content=content, - )) - - def get_max_room_data_id(self): - return self._simple_max_id(RoomDataTable.table_name) - - -class RoomDataTable(Table): - table_name = "room_data" - - fields = [ - "id", - "room_id", - "type", - "state_key", - "content" - ] - - class EntryType(collections.namedtuple("RoomDataEntry", fields)): - - def as_event(self, event_factory): - return event_factory.create_event( - etype=self.type, - room_id=self.room_id, - content=json.loads(self.content), - ) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index e09ff6b64..ad9770244 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS events( ordering INTEGER PRIMARY KEY AUTOINCREMENT, event_id TEXT NOT NULL, - event_type TEXT NOT NULL, - sender TEXT, + type TEXT NOT NULL, +-- sender TEXT, room_id TEXT, content TEXT, unrecognized_keys TEXT @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS events( CREATE TABLE IF NOT EXISTS state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, - event_type TEXT NOT NULL, + type TEXT NOT NULL, state_key TEXT NOT NULL, prev_state TEXT ); @@ -47,9 +47,8 @@ CREATE TABLE IF NOT EXISTS room_memberships( CREATE TABLE IF NOT EXISTS feedback( event_id TEXT NOT NULL, feedback_type TEXT, - fb_sender_id TEXT, - room_id TEXT, - content TEXT + target_event_id TEXT,sudo + room_id TEXT ); CREATE TABLE IF NOT EXISTS rooms( From beaf4384d906f7fb41d80619d0a46a3f593795db Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Aug 2014 17:43:34 +0100 Subject: [PATCH 003/112] Make feedback table also store sender. --- synapse/storage/feedback.py | 1 + synapse/storage/schema/im.sql | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py index fc93f92e1..b9e792c12 100644 --- a/synapse/storage/feedback.py +++ b/synapse/storage/feedback.py @@ -28,6 +28,7 @@ class FeedbackStore(SQLBaseStore): "feedback_type": event.feedback_type, "room_id": event.room_id, "target_event_id": event.target_event, + "sender": event.user_id, }) @defer.inlineCallback diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index ad9770244..37b7c6c74 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -47,7 +47,8 @@ CREATE TABLE IF NOT EXISTS room_memberships( CREATE TABLE IF NOT EXISTS feedback( event_id TEXT NOT NULL, feedback_type TEXT, - target_event_id TEXT,sudo + target_event_id TEXT, + sender TEXT, room_id TEXT ); From cbd5d55222fdc662b7253b9ec75c4ff42cfd92e4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Aug 2014 14:08:57 +0100 Subject: [PATCH 004/112] Change relative db paths to absolute paths in case we daemonize. --- synapse/app/homeserver.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 82afb04c7..2fd7e0ae4 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -30,6 +30,7 @@ import argparse import logging import logging.config import sqlite3 +import os logger = logging.getLogger(__name__) @@ -131,9 +132,15 @@ def setup(): verbosity = int(args.verbose) if args.verbose else None + # Because if/when we daemonize we change to root dir. + db_name = os.path.abspath(args.db) + log_file = args.log_file + if log_file: + log_file = os.path.abspath(log_file) + setup_logging( verbosity=verbosity, - filename=args.log_file, + filename=log_file, config_path=args.log_config, ) @@ -141,7 +148,7 @@ def setup(): hs = SynapseHomeServer( args.host, - db_name=args.db + db_name=db_name ) # This object doesn't need to be saved because it's set as the handler for From 6d6a1c3454ae787e3878202a8e41341ddcf7bee0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Aug 2014 14:30:25 +0100 Subject: [PATCH 005/112] Actually encode dicts as json in the DB --- synapse/storage/__init__.py | 4 ++-- synapse/storage/_base.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d38d61345..cd9acdc44 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -81,11 +81,11 @@ class DataStore(RoomDataStore, RoomMemberStore, MessageStore, RoomStore, "event_type", event.type, "sender": event.user_id, "room_id": event.room_id, - "content": event.content, + "content": json.dumps(event.content), } unrec = {k: v for k, v in event.get_full_dict() if k not in vals.keys()} - val["unrecognized_keys"] = unrec + val["unrecognized_keys"] = json.dumps(unrec) yield self._simple_insert("events", vals) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 489b6bd17..5cb26ad6d 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -288,7 +288,8 @@ class SQLBaseStore(object): def _parse_event_from_row(self, row_dict): d = copy.deepcopy({k: v for k, v in row.items() if v}) - d.update(json.loads(row["unrecognized_keys"])) + d.update(json.loads(json.loads(row["unrecognized_keys"]))) + d["content"] = json.loads(d["content"}) del d["unrecognized_keys"] return self.event_factory.create_event( From 937c175029be9bcf2ba26c27ecb879292e8c555a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Aug 2014 16:02:10 +0100 Subject: [PATCH 006/112] Fix up RoomMemberStore to work with the new schema. --- synapse/storage/_base.py | 6 +- synapse/storage/roommember.py | 166 ++++++++++++++-------------------- synapse/storage/schema/im.sql | 6 +- 3 files changed, 80 insertions(+), 98 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5cb26ad6d..befeb55b2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -64,7 +64,11 @@ class SQLBaseStore(object): def interaction(txn): cursor = txn.execute(query, args) - return decoder(cursor) + if decoder: + return decoder(cursor) + else: + return cursor + return self._db_pool.runInteraction(interaction) def _execut_query(self, query, *args): diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index ef73be4af..60296380e 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -31,6 +31,38 @@ logger = logging.getLogger(__name__) class RoomMemberStore(SQLBaseStore): + @defer.inlineCallbacks + def _store_room_member(self, event): + """Store a room member in the database. + """ + domain = self.hs.parse_userid(event.target_user_id).domain + + yield self._simple_insert( + "room_memberships", + { + "event_id": event.event_id, + "user_id": event.target_user_id, + "sender": event.user_id, + "room_id": event.room_id, + "membership": event.membership, + } + ) + + # Update room hosts table + if event.membership == Membership.JOIN: + sql = ( + "INSERT OR IGNORE INTO room_hosts (room_id, host) " + "VALUES (?, ?)" + ) + yield self._execute(None, sql, room_id, domain) + else: + sql = ( + "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" + ) + + yield self._execute(None, sql, room_id, domain) + + def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. @@ -38,36 +70,13 @@ class RoomMemberStore(SQLBaseStore): user_id (str): The member's user ID. room_id (str): The room the member is in. Returns: - namedtuple: The room member from the database, or None if this - member does not exist. + Deferred: Results in a MembershipEvent or None. """ - query = RoomMemberTable.select_statement( - "room_id = ? AND user_id = ? ORDER BY id DESC LIMIT 1") - return self._execute( - RoomMemberTable.decode_single_result, - query, room_id, user_id, + return self._get_members_by_dict( + room_id=room_id, + user_id=user_id ) - def store_room_member(self, user_id, sender, room_id, membership, content): - """Store a room member in the database. - - Args: - user_id (str): The member's user ID. - room_id (str): The room in relation to the member. - membership (synapse.api.constants.Membership): The new membership - state. - content (dict): The content of the membership (JSON). - """ - content_json = json.dumps(content) - return self._simple_insert(RoomMemberTable.table_name, dict( - user_id=user_id, - sender=sender, - room_id=room_id, - membership=membership, - content=content_json, - )) - - @defer.inlineCallbacks def get_room_members(self, room_id, membership=None): """Retrieve the current room member list for a room. @@ -79,17 +88,12 @@ class RoomMemberStore(SQLBaseStore): Returns: list of namedtuples representing the members in this room. """ - query = RoomMemberTable.select_statement( - "id IN (SELECT MAX(id) FROM " + RoomMemberTable.table_name - + " WHERE room_id = ? GROUP BY user_id)" - ) - res = yield self._execute( - RoomMemberTable.decode_results, query, room_id, - ) - # strip memberships which don't match + + where = {"room_id": room_id} if membership: - res = [entry for entry in res if entry.membership == membership] - defer.returnValue(res) + where["membership"] = membership + + return self._get_members_by_dict(**membership) def get_rooms_for_user_where_membership_is(self, user_id, membership_list): """ Get all the rooms for this user where the membership for this user @@ -106,67 +110,37 @@ class RoomMemberStore(SQLBaseStore): return defer.succeed(None) args = [user_id] - membership_placeholder = ["membership=?"] * len(membership_list) - where_membership = "(" + " OR ".join(membership_placeholder) + ")" - for membership in membership_list: - args.append(membership) + args.extend(membership_list) - query = ("SELECT room_id, membership FROM room_memberships" - + " WHERE user_id=? AND " + where_membership - + " GROUP BY room_id ORDER BY id DESC") - return self._execute( - self.cursor_to_dict, query, *args + where_clause "user_id = ? AND (%s)" % ( + " OR ".join(["membership = ?" for _ in membership_list]), ) + return self._get_members_query(where_clause, args) + + def get_joined_hosts_for_room(self, room_id): + return self._simple_select_onecol( + "room_hosts", + {"room_id": room_id}, + "host" + ) + + def _get_members_by_dict(self, where_dict): + clause = " AND ".join("%s = ?" % k for k in where.keys()) + vals = where.values() + return self._get_members_query(clause, vals) + @defer.inlineCallbacks - def get_joined_hosts_for_room(self, room_id): - query = RoomMemberTable.select_statement( - "id IN (SELECT MAX(id) FROM " + RoomMemberTable.table_name - + " WHERE room_id = ? GROUP BY user_id)" - ) + def _get_members_query(self, where_clause, where_values): + sql = ( + "SELECT e.* FROM events as e " + "INNER JOIN room_memberships as m " + "ON e.event_id = m.event_id " + "INNER JOIN current_state as c " + "ON m.event_id = c.event_id " + "WHERE %s " + ) % (where_clause,) - res = yield self._execute( - RoomMemberTable.decode_results, query, room_id, - ) - - def host_from_user_id_string(user_id): - domain = UserID.from_string(entry.user_id, self.hs).domain - return domain - - # strip memberships which don't match - hosts = [ - host_from_user_id_string(entry.user_id) - for entry in res - if entry.membership == Membership.JOIN - ] - - logger.debug("Returning hosts: %s from results: %s", hosts, res) - - defer.returnValue(hosts) - - def get_max_room_member_id(self): - return self._simple_max_id(RoomMemberTable.table_name) - - -class RoomMemberTable(Table): - table_name = "room_memberships" - - fields = [ - "id", - "user_id", - "sender", - "room_id", - "membership", - "content" - ] - - class EntryType(collections.namedtuple("RoomMemberEntry", fields)): - - def as_event(self, event_factory): - return event_factory.create_event( - etype=RoomMemberEvent.TYPE, - room_id=self.room_id, - target_user_id=self.user_id, - user_id=self.sender, - content=json.loads(self.content), - ) + rows = yield self._execute_query(sql, where_values) + results = [self._parse_event_from_row(r) for r in rows] + defer.returnValue(results) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 37b7c6c74..7f564c854 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -17,7 +17,6 @@ CREATE TABLE IF NOT EXISTS events( ordering INTEGER PRIMARY KEY AUTOINCREMENT, event_id TEXT NOT NULL, type TEXT NOT NULL, --- sender TEXT, room_id TEXT, content TEXT, unrecognized_keys TEXT @@ -57,3 +56,8 @@ CREATE TABLE IF NOT EXISTS rooms( is_public INTEGER, creator TEXT ); + +CREATE TABLE IF NOT EXISTS room_hosts( + room_id TEXT NOT NULL, + host TEXT NOT NULL +); From 2529f2bc01781314ecdedd69e272c737ba1a71f5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Aug 2014 16:58:51 +0100 Subject: [PATCH 007/112] Rename _execute_query --- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 2 +- synapse/storage/feedback.py | 2 +- synapse/storage/roommember.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index cd9acdc44..75d93f711 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -119,7 +119,7 @@ class DataStore(RoomDataStore, RoomMemberStore, MessageStore, RoomStore, else: args = (room_id, ) - results = yield self._execute_query(sql, *args) + results = yield self._execute_and_decode(sql, *args) defer.returnValue( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index befeb55b2..7fef8601e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -71,7 +71,7 @@ class SQLBaseStore(object): return self._db_pool.runInteraction(interaction) - def _execut_query(self, query, *args): + def _execute_and_decode(self, query, *args): return self._execute(self.cursor_to_dict, *args) # "Simple" SQL API methods that operate on a single table with no JOINs, diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py index b9e792c12..dd5f3fbc1 100644 --- a/synapse/storage/feedback.py +++ b/synapse/storage/feedback.py @@ -39,7 +39,7 @@ class FeedbackStore(SQLBaseStore): "WHERE feedback.target_event_id = ? " ) - rows = yield self._execute_query(sql, event_id) + rows = yield self._execute_and_decode(sql, event_id) defer.returnValue( [ diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 60296380e..c99cefbcf 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -141,6 +141,6 @@ class RoomMemberStore(SQLBaseStore): "WHERE %s " ) % (where_clause,) - rows = yield self._execute_query(sql, where_values) + rows = yield self._execute_and_decode(sql, where_values) results = [self._parse_event_from_row(r) for r in rows] defer.returnValue(results) From 78b501eba68f93c91f53ddf179ebde32f511894f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Aug 2014 17:09:28 +0100 Subject: [PATCH 008/112] Fix typo --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 75d93f711..afdd75f46 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -121,7 +121,7 @@ class DataStore(RoomDataStore, RoomMemberStore, MessageStore, RoomStore, results = yield self._execute_and_decode(sql, *args) - defer.returnValue( + defer.returnValue([self._parse_event_from_row(r) for r in results]) def schema_path(schema): From 661c7117659118ed977f56a092525dbdae9dc67c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Aug 2014 17:34:37 +0100 Subject: [PATCH 009/112] Start fixing places that use the data store. --- synapse/handlers/room.py | 17 ++++----------- synapse/rest/room.py | 39 +++++++++++++++++++---------------- synapse/storage/__init__.py | 8 +++---- synapse/storage/_base.py | 2 +- synapse/storage/feedback.py | 4 +++- synapse/storage/roommember.py | 2 +- 6 files changed, 33 insertions(+), 39 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index eae40765b..a9ff2d93f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -141,12 +141,7 @@ class MessageHandler(BaseHandler): yield self.state_handler.handle_new_event(event) # store in db - store_id = yield self.store.store_room_data( - room_id=event.room_id, - etype=event.type, - state_key=event.state_key, - content=json.dumps(event.content) - ) + store_id = yield self.store.persist_event(event) event.destinations = yield self.store.get_joined_hosts_for_room( event.room_id @@ -201,19 +196,15 @@ class MessageHandler(BaseHandler): raise RoomError( 403, "Member does not meet private room rules.") - data = yield self.store.get_room_data(room_id, event_type, state_key) + data = yield self.store.get_current_state(room_id, event_type, state_key) defer.returnValue(data) @defer.inlineCallbacks - def get_feedback(self, room_id=None, msg_sender_id=None, msg_id=None, - user_id=None, fb_sender_id=None, fb_type=None): + def get_feedback(self, event_id): yield self.auth.check_joined_room(room_id, user_id) # Pull out the feedback from the db - fb = yield self.store.get_feedback( - room_id=room_id, msg_id=msg_id, msg_sender_id=msg_sender_id, - fb_sender_id=fb_sender_id, fb_type=fb_type - ) + fb = yield self.store.get_feedback(event_id) if fb: defer.returnValue(fb) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 1fc0c996b..3f153df8e 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -285,25 +285,28 @@ class FeedbackRestServlet(RestServlet): feedback_type): user = yield (self.auth.get_user_by_req(request)) - if feedback_type not in Feedback.LIST: - raise SynapseError(400, "Bad feedback type.", - errcode=Codes.BAD_JSON) + # TODO (erikj): Implement this? + raise NotImplementedError("Getting feedback is not supported") - msg_handler = self.handlers.message_handler - feedback = yield msg_handler.get_feedback( - room_id=urllib.unquote(room_id), - msg_sender_id=msg_sender_id, - msg_id=msg_id, - user_id=user.to_string(), - fb_sender_id=fb_sender_id, - fb_type=feedback_type - ) - - if not feedback: - raise SynapseError(404, "Feedback not found.", - errcode=Codes.NOT_FOUND) - - defer.returnValue((200, json.loads(feedback.content))) +# if feedback_type not in Feedback.LIST: +# raise SynapseError(400, "Bad feedback type.", +# errcode=Codes.BAD_JSON) +# +# msg_handler = self.handlers.message_handler +# feedback = yield msg_handler.get_feedback( +# room_id=urllib.unquote(room_id), +# msg_sender_id=msg_sender_id, +# msg_id=msg_id, +# user_id=user.to_string(), +# fb_sender_id=fb_sender_id, +# fb_type=feedback_type +# ) +# +# if not feedback: +# raise SynapseError(404, "Feedback not found.", +# errcode=Codes.NOT_FOUND) +# +# defer.returnValue((200, json.loads(feedback.content))) @defer.inlineCallbacks def on_PUT(self, request, room_id, sender_id, msg_id, fb_sender_id, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index afdd75f46..182b6ebad 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -21,13 +21,11 @@ from synapse.api.events.room import ( from .directory import DirectoryStore from .feedback import FeedbackStore -from .message import MessageStore from .presence import PresenceStore from .profile import ProfileStore from .registration import RegistrationStore from .room import RoomStore from .roommember import RoomMemberStore -from .roomdata import RoomDataStore from .stream import StreamStore from .pdu import StatePduStore, PduStore from .transactions import TransactionStore @@ -36,7 +34,7 @@ import json import os -class DataStore(RoomDataStore, RoomMemberStore, MessageStore, RoomStore, +class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, PduStore, StatePduStore, TransactionStore, DirectoryStore): @@ -78,7 +76,7 @@ class DataStore(RoomDataStore, RoomMemberStore, MessageStore, RoomStore, def _store_event(self, event): vals = { "event_id": event.event_id, - "event_type", event.type, + "event_type": event.type, "sender": event.user_id, "room_id": event.room_id, "content": json.dumps(event.content), @@ -105,7 +103,7 @@ class DataStore(RoomDataStore, RoomMemberStore, MessageStore, RoomStore, # TODO (erikj): We also need to update the current state table? @defer.inlineCallbacks - def get_current_state(room_id, event_type=None, state_key="") + def get_current_state(room_id, event_type=None, state_key=""): sql = ( "SELECT e.* FROM events as e" "INNER JOIN current_state as c ON e.event_id = c.event_id " diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7fef8601e..533f50970 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -293,7 +293,7 @@ class SQLBaseStore(object): def _parse_event_from_row(self, row_dict): d = copy.deepcopy({k: v for k, v in row.items() if v}) d.update(json.loads(json.loads(row["unrecognized_keys"]))) - d["content"] = json.loads(d["content"}) + d["content"] = json.loads(d["content"]) del d["unrecognized_keys"] return self.event_factory.create_event( diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py index dd5f3fbc1..e60f98d1e 100644 --- a/synapse/storage/feedback.py +++ b/synapse/storage/feedback.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + from ._base import SQLBaseStore, Table from synapse.api.events.room import FeedbackEvent @@ -31,7 +33,7 @@ class FeedbackStore(SQLBaseStore): "sender": event.user_id, }) - @defer.inlineCallback + @defer.inlineCallbacks def get_feedback_for_event(self, event_id): sql = ( "SELECT events.* FROM events INNER JOIN feedback " diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index c99cefbcf..14c0152e8 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -112,7 +112,7 @@ class RoomMemberStore(SQLBaseStore): args = [user_id] args.extend(membership_list) - where_clause "user_id = ? AND (%s)" % ( + where_clause = "user_id = ? AND (%s)" % ( " OR ".join(["membership = ?" for _ in membership_list]), ) From 7e681ad778b18f47fc7d6f410c6b1e87a1a99f20 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Aug 2014 18:01:39 +0100 Subject: [PATCH 010/112] Update StreamStore --- synapse/storage/stream.py | 285 +++++--------------------------------- 1 file changed, 38 insertions(+), 247 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 1dedffac4..1456d216f 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -20,264 +20,55 @@ from .feedback import FeedbackTable from .roomdata import RoomDataTable from .roommember import RoomMemberTable +from synapse.api.constants import Membership + import json import logging + logger = logging.getLogger(__name__) +MAX_STREAM_SIZE = 1000 + + class StreamStore(SQLBaseStore): - def get_message_stream(self, user_id, from_key, to_key, room_id, limit=0, - with_feedback=False): - """Get all messages for this user between the given keys. + @defer.inlineCallbacks + def get_room_events_stream(self, user_id, from_key, to_key, room_id, + limit=0, with_feedback=False): - Args: - user_id (str): The user who is requesting messages. - from_key (int): The ID to start returning results from (exclusive). - to_key (int): The ID to stop returning results (exclusive). - room_id (str): Gets messages only for this room. Can be None, in - which case all room messages will be returned. - Returns: - A tuple of rows (list of namedtuples), new_id(int) - """ - if with_feedback and room_id: # with fb MUST specify a room ID - return self._db_pool.runInteraction( - self._get_message_rows_with_feedback, - user_id, from_key, to_key, room_id, limit - ) + current_room_membership_sql = ( + "SELECT m.room_id FROM room_memberships as m " + "INNER JOIN current_state as c ON m.event_id = c.event_id " + "WHERE m.user_id = ?" + ) + + invites_sql = ( + "SELECT m.event_id FROM room_membershipas as m " + "INNER JOIN current_state as c ON m.event_id = c.event_id " + "WHERE m.user_id = ? AND m.membership = ?" + ) + + if limit: + limit = max(limit, MAX_STREAM_SIZE) else: - return self._db_pool.runInteraction( - self._get_message_rows, - user_id, from_key, to_key, room_id, limit - ) + limit = 1000 - def _get_message_rows(self, txn, user_id, from_pkey, to_pkey, room_id, - limit): - # work out which rooms this user is joined in on and join them with - # the room id on the messages table, bounded by the specified pkeys + sql = ( + "SELECT * FROM events as e WHERE " + "(room_id IN (%(current)s)) OR " + "(event_id IN (%(invites)s)) " + "ORDER BY ordering ASC LIMIT %(limit)d" + ) % { + "current": current_room_membership_sql, + "invites": invites_sql, + "limit": limit, + } - # get all messages where the *current* membership state is 'join' for - # this user in that room. - query = ("SELECT messages.* FROM messages WHERE ? IN" - + " (SELECT membership from room_memberships WHERE user_id=?" - + " AND room_id = messages.room_id ORDER BY id DESC LIMIT 1)") - query_args = ["join", user_id] - - if room_id: - query += " AND messages.room_id=?" - query_args.append(room_id) - - (query, query_args) = self._append_stream_operations( - "messages", query, query_args, from_pkey, to_pkey, limit=limit + rows = yield self._execute_and_decode( + sql, + user_id, user_id, Membership.INVITE ) - logger.debug("[SQL] %s : %s", query, query_args) - cursor = txn.execute(query, query_args) - return self._as_events(cursor, MessagesTable, from_pkey) - - def _get_message_rows_with_feedback(self, txn, user_id, from_pkey, to_pkey, - room_id, limit): - # this col represents the compressed feedback JSON as per spec - compressed_feedback_col = ( - "'[' || group_concat('{\"sender_id\":\"' || f.fb_sender_id" - + " || '\",\"feedback_type\":\"' || f.feedback_type" - + " || '\",\"content\":' || f.content || '}') || ']'" - ) - - global_msg_id_join = ("f.room_id = messages.room_id" - + " and f.msg_id = messages.msg_id" - + " and messages.user_id = f.msg_sender_id") - - select_query = ( - "SELECT messages.*, f.content AS fb_content, f.fb_sender_id" - + ", " + compressed_feedback_col + " AS compressed_fb" - + " FROM messages LEFT JOIN feedback f ON " + global_msg_id_join) - - current_membership_sub_query = ( - "(SELECT membership from room_memberships rm" - + " WHERE user_id=? AND room_id = rm.room_id" - + " ORDER BY id DESC LIMIT 1)") - - where = (" WHERE ? IN " + current_membership_sub_query - + " AND messages.room_id=?") - - query = select_query + where - query_args = ["join", user_id, room_id] - - (query, query_args) = self._append_stream_operations( - "messages", query, query_args, from_pkey, to_pkey, - limit=limit, group_by=" GROUP BY messages.id " - ) - - logger.debug("[SQL] %s : %s", query, query_args) - cursor = txn.execute(query, query_args) - - # convert the result set into events - entries = self.cursor_to_dict(cursor) - events = [] - for entry in entries: - # TODO we should spec the cursor > event mapping somewhere else. - event = {} - straight_mappings = ["msg_id", "user_id", "room_id"] - for key in straight_mappings: - event[key] = entry[key] - event["content"] = json.loads(entry["content"]) - if entry["compressed_fb"]: - event["feedback"] = json.loads(entry["compressed_fb"]) - events.append(event) - - latest_pkey = from_pkey if len(entries) == 0 else entries[-1]["id"] - - return (events, latest_pkey) - - def get_room_member_stream(self, user_id, from_key, to_key): - """Get all room membership events for this user between the given keys. - - Args: - user_id (str): The user who is requesting membership events. - from_key (int): The ID to start returning results from (exclusive). - to_key (int): The ID to stop returning results (exclusive). - Returns: - A tuple of rows (list of namedtuples), new_id(int) - """ - return self._db_pool.runInteraction( - self._get_room_member_rows, user_id, from_key, to_key - ) - - def _get_room_member_rows(self, txn, user_id, from_pkey, to_pkey): - # get all room membership events for rooms which the user is - # *currently* joined in on, or all invite events for this user. - current_membership_sub_query = ( - "(SELECT membership FROM room_memberships" - + " WHERE user_id=? AND room_id = rm.room_id" - + " ORDER BY id DESC LIMIT 1)") - - query = ("SELECT rm.* FROM room_memberships rm " - # all membership events for rooms you've currently joined. - + " WHERE (? IN " + current_membership_sub_query - # all invite membership events for this user - + " OR rm.membership=? AND user_id=?)" - + " AND rm.id > ?") - query_args = ["join", user_id, "invite", user_id, from_pkey] - - if to_pkey != -1: - query += " AND rm.id < ?" - query_args.append(to_pkey) - - cursor = txn.execute(query, query_args) - return self._as_events(cursor, RoomMemberTable, from_pkey) - - def get_feedback_stream(self, user_id, from_key, to_key, room_id, limit=0): - return self._db_pool.runInteraction( - self._get_feedback_rows, - user_id, from_key, to_key, room_id, limit - ) - - def _get_feedback_rows(self, txn, user_id, from_pkey, to_pkey, room_id, - limit): - # work out which rooms this user is joined in on and join them with - # the room id on the feedback table, bounded by the specified pkeys - - # get all messages where the *current* membership state is 'join' for - # this user in that room. - query = ( - "SELECT feedback.* FROM feedback WHERE ? IN " - + "(SELECT membership from room_memberships WHERE user_id=?" - + " AND room_id = feedback.room_id ORDER BY id DESC LIMIT 1)") - query_args = ["join", user_id] - - if room_id: - query += " AND feedback.room_id=?" - query_args.append(room_id) - - (query, query_args) = self._append_stream_operations( - "feedback", query, query_args, from_pkey, to_pkey, limit=limit - ) - - logger.debug("[SQL] %s : %s", query, query_args) - cursor = txn.execute(query, query_args) - return self._as_events(cursor, FeedbackTable, from_pkey) - - def get_room_data_stream(self, user_id, from_key, to_key, room_id, - limit=0): - return self._db_pool.runInteraction( - self._get_room_data_rows, - user_id, from_key, to_key, room_id, limit - ) - - def _get_room_data_rows(self, txn, user_id, from_pkey, to_pkey, room_id, - limit): - # work out which rooms this user is joined in on and join them with - # the room id on the feedback table, bounded by the specified pkeys - - # get all messages where the *current* membership state is 'join' for - # this user in that room. - query = ( - "SELECT room_data.* FROM room_data WHERE ? IN " - + "(SELECT membership from room_memberships WHERE user_id=?" - + " AND room_id = room_data.room_id ORDER BY id DESC LIMIT 1)") - query_args = ["join", user_id] - - if room_id: - query += " AND room_data.room_id=?" - query_args.append(room_id) - - (query, query_args) = self._append_stream_operations( - "room_data", query, query_args, from_pkey, to_pkey, limit=limit - ) - - logger.debug("[SQL] %s : %s", query, query_args) - cursor = txn.execute(query, query_args) - return self._as_events(cursor, RoomDataTable, from_pkey) - - def _append_stream_operations(self, table_name, query, query_args, - from_pkey, to_pkey, limit=None, - group_by=""): - LATEST_ROW = -1 - order_by = "" - if to_pkey > from_pkey: - if from_pkey != LATEST_ROW: - # e.g. from=5 to=9 >> from 5 to 9 >> id>5 AND id<9 - query += (" AND %s.id > ? AND %s.id < ?" % - (table_name, table_name)) - query_args.append(from_pkey) - query_args.append(to_pkey) - else: - # e.g. from=-1 to=5 >> from now to 5 >> id>5 ORDER BY id DESC - query += " AND %s.id > ? " % table_name - order_by = "ORDER BY id DESC" - query_args.append(to_pkey) - elif from_pkey > to_pkey: - if to_pkey != LATEST_ROW: - # from=9 to=5 >> from 9 to 5 >> id>5 AND id<9 ORDER BY id DESC - query += (" AND %s.id > ? AND %s.id < ? " % - (table_name, table_name)) - order_by = "ORDER BY id DESC" - query_args.append(to_pkey) - query_args.append(from_pkey) - else: - # from=5 to=-1 >> from 5 to now >> id>5 - query += " AND %s.id > ?" % table_name - query_args.append(from_pkey) - - query += group_by + order_by - - if limit and limit > 0: - query += " LIMIT ?" - query_args.append(str(limit)) - - return (query, query_args) - - def _as_events(self, cursor, table, from_pkey): - data_entries = table.decode_results(cursor) - last_pkey = from_pkey - if data_entries: - last_pkey = data_entries[-1].id - - events = [ - entry.as_event(self.event_factory).get_dict() - for entry in data_entries - ] - - return (events, last_pkey) + defer.returnValue([self._parse_event_from_row(r) for r in results]) From 2c46bb620828efaebdbae37e5212a28b505ee72d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 14 Aug 2014 18:40:50 +0100 Subject: [PATCH 011/112] Fix up typos and correct sql queries --- synapse/handlers/room.py | 10 ++-------- synapse/storage/__init__.py | 20 ++++++++++---------- synapse/storage/_base.py | 9 +++++---- synapse/storage/roommember.py | 26 +++++++++++++------------- synapse/storage/schema/im.sql | 4 ++-- synapse/storage/stream.py | 11 ++++------- 6 files changed, 36 insertions(+), 44 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a9ff2d93f..9b55206e4 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -201,7 +201,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_feedback(self, event_id): - yield self.auth.check_joined_room(room_id, user_id) + # yield self.auth.check_joined_room(room_id, user_id) # Pull out the feedback from the db fb = yield self.store.get_feedback(event_id) @@ -690,13 +690,7 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _do_local_membership_update(self, event, membership, broadcast_msg): # store membership - store_id = yield self.store.store_room_member( - user_id=event.target_user_id, - sender=event.user_id, - room_id=event.room_id, - content=event.content, - membership=membership - ) + store_id = yield self.store.persist_event(event) # Send a PDU to all hosts who have joined the room. destinations = yield self.store.get_joined_hosts_for_room( diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 182b6ebad..f41c21dcd 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer from synapse.api.events.room import ( RoomMemberEvent, MessageEvent, RoomTopicEvent, FeedbackEvent, @@ -52,7 +53,7 @@ class DataStore(RoomMemberStore, RoomStore, elif event.type == RoomConfigEvent.TYPE: yield self._store_room_config(event) - self._store_event(event) + yield self._store_event(event) @defer.inlineCallbacks def get_event(self, event_id): @@ -76,14 +77,13 @@ class DataStore(RoomMemberStore, RoomStore, def _store_event(self, event): vals = { "event_id": event.event_id, - "event_type": event.type, - "sender": event.user_id, + "type": event.type, "room_id": event.room_id, "content": json.dumps(event.content), } - unrec = {k: v for k, v in event.get_full_dict() if k not in vals.keys()} - val["unrecognized_keys"] = json.dumps(unrec) + unrec = {k: v for k, v in event.get_full_dict().items() if k not in vals.keys()} + vals["unrecognized_keys"] = json.dumps(unrec) yield self._simple_insert("events", vals) @@ -91,7 +91,7 @@ class DataStore(RoomMemberStore, RoomStore, vals = { "event_id": event.event_id, "room_id": event.room_id, - "event_type": event.event_type, + "type": event.type, "state_key": event.state_key, } @@ -103,16 +103,16 @@ class DataStore(RoomMemberStore, RoomStore, # TODO (erikj): We also need to update the current state table? @defer.inlineCallbacks - def get_current_state(room_id, event_type=None, state_key=""): + def get_current_state(self, room_id, event_type=None, state_key=""): sql = ( - "SELECT e.* FROM events as e" - "INNER JOIN current_state as c ON e.event_id = c.event_id " + "SELECT e.* FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " "INNER JOIN state_events as s ON e.event_id = s.event_id " "WHERE c.room_id = ? " ) if event_type: - sql += " s.type = ? AND s.state_key = ? " + sql += " AND s.type = ? AND s.state_key = ? " args = (room_id, event_type, state_key) else: args = (room_id, ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 533f50970..c8ec63f30 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError import collections +import copy import json @@ -59,7 +60,7 @@ class SQLBaseStore(object): The result of decoder(results) """ logger.debug( - "[SQL] %s Args=%s Func=%s", query, args, decoder.__name__ + "[SQL] %s Args=%s Func=%s", query, args, decoder.__name__ if decoder else None ) def interaction(txn): @@ -72,7 +73,7 @@ class SQLBaseStore(object): return self._db_pool.runInteraction(interaction) def _execute_and_decode(self, query, *args): - return self._execute(self.cursor_to_dict, *args) + return self._execute(self.cursor_to_dict, query, *args) # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. @@ -291,8 +292,8 @@ class SQLBaseStore(object): return self._db_pool.runInteraction(func) def _parse_event_from_row(self, row_dict): - d = copy.deepcopy({k: v for k, v in row.items() if v}) - d.update(json.loads(json.loads(row["unrecognized_keys"]))) + d = copy.deepcopy({k: v for k, v in row_dict.items() if v}) + d.update(json.loads(json.loads(row_dict["unrecognized_keys"]))) d["content"] = json.loads(d["content"]) del d["unrecognized_keys"] diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 14c0152e8..8c4b04f19 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -54,13 +54,13 @@ class RoomMemberStore(SQLBaseStore): "INSERT OR IGNORE INTO room_hosts (room_id, host) " "VALUES (?, ?)" ) - yield self._execute(None, sql, room_id, domain) + yield self._execute(None, sql, event.room_id, domain) else: sql = ( "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" ) - yield self._execute(None, sql, room_id, domain) + yield self._execute(None, sql, event.room_id, domain) def get_room_member(self, user_id, room_id): @@ -72,10 +72,10 @@ class RoomMemberStore(SQLBaseStore): Returns: Deferred: Results in a MembershipEvent or None. """ - return self._get_members_by_dict( - room_id=room_id, - user_id=user_id - ) + return self._get_members_by_dict({ + "e.room_id": room_id, + "m.user_id": user_id, + }) def get_room_members(self, room_id, membership=None): """Retrieve the current room member list for a room. @@ -89,11 +89,11 @@ class RoomMemberStore(SQLBaseStore): list of namedtuples representing the members in this room. """ - where = {"room_id": room_id} + where = {"m.room_id": room_id} if membership: - where["membership"] = membership + where["m.membership"] = membership - return self._get_members_by_dict(**membership) + return self._get_members_by_dict(where) def get_rooms_for_user_where_membership_is(self, user_id, membership_list): """ Get all the rooms for this user where the membership for this user @@ -126,8 +126,8 @@ class RoomMemberStore(SQLBaseStore): ) def _get_members_by_dict(self, where_dict): - clause = " AND ".join("%s = ?" % k for k in where.keys()) - vals = where.values() + clause = " AND ".join("%s = ?" % k for k in where_dict.keys()) + vals = where_dict.values() return self._get_members_query(clause, vals) @defer.inlineCallbacks @@ -136,11 +136,11 @@ class RoomMemberStore(SQLBaseStore): "SELECT e.* FROM events as e " "INNER JOIN room_memberships as m " "ON e.event_id = m.event_id " - "INNER JOIN current_state as c " + "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " "WHERE %s " ) % (where_clause,) - rows = yield self._execute_and_decode(sql, where_values) + rows = yield self._execute_and_decode(sql, *where_values) results = [self._parse_event_from_row(r) for r in rows] defer.returnValue(results) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 7f564c854..85c0c7119 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -30,9 +30,9 @@ CREATE TABLE IF NOT EXISTS state_events( prev_state TEXT ); -CREATE TABLE IF NOT EXISTS current_state( +CREATE TABLE IF NOT EXISTS current_state_events( event_id TEXT NOT NULL, - room_id TEXT NOT NULL, + room_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS room_memberships( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 1456d216f..9937239c2 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -13,12 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer from ._base import SQLBaseStore -from .message import MessagesTable -from .feedback import FeedbackTable -from .roomdata import RoomDataTable -from .roommember import RoomMemberTable from synapse.api.constants import Membership @@ -40,13 +37,13 @@ class StreamStore(SQLBaseStore): current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " - "INNER JOIN current_state as c ON m.event_id = c.event_id " + "INNER JOIN current_state_events as c ON m.event_id = c.event_id " "WHERE m.user_id = ?" ) invites_sql = ( "SELECT m.event_id FROM room_membershipas as m " - "INNER JOIN current_state as c ON m.event_id = c.event_id " + "INNER JOIN current_state_events as c ON m.event_id = c.event_id " "WHERE m.user_id = ? AND m.membership = ?" ) @@ -71,4 +68,4 @@ class StreamStore(SQLBaseStore): user_id, user_id, Membership.INVITE ) - defer.returnValue([self._parse_event_from_row(r) for r in results]) + defer.returnValue([self._parse_event_from_row(r) for r in rows]) From 5002efa31bb57a92b87b9d7319641d9b5a2a6047 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 10:26:35 +0100 Subject: [PATCH 012/112] Reimplement the get public rooms api to work with new DB schema --- synapse/api/events/factory.py | 3 +- synapse/api/events/room.py | 23 +++++++++ synapse/handlers/room.py | 2 +- synapse/storage/__init__.py | 6 ++- synapse/storage/_base.py | 2 +- synapse/storage/room.py | 88 ++++++++++++++++++++++------------- synapse/storage/schema/im.sql | 12 +++++ 7 files changed, 100 insertions(+), 36 deletions(-) diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 12aa04fc6..23d2b0401 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -15,7 +15,7 @@ from synapse.api.events.room import ( RoomTopicEvent, MessageEvent, RoomMemberEvent, FeedbackEvent, - InviteJoinEvent, RoomConfigEvent + InviteJoinEvent, RoomConfigEvent, RoomNameEvent, ) from synapse.util.stringutils import random_string @@ -25,6 +25,7 @@ class EventFactory(object): _event_classes = [ RoomTopicEvent, + RoomNameEvent, MessageEvent, RoomMemberEvent, FeedbackEvent, diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index f3df849af..8136d495d 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -19,14 +19,37 @@ from . import SynapseEvent class RoomTopicEvent(SynapseEvent): TYPE = "m.room.topic" + internal_keys = SynapseEvent.internal_keys + [ + "topic", + ] + def __init__(self, **kwargs): kwargs["state_key"] = "" + if "topic" in kwargs["content"]: + kwargs["topic"] = kwargs["content"]["topic"] super(RoomTopicEvent, self).__init__(**kwargs) def get_content_template(self): return {"topic": u"string"} +class RoomNameEvent(SynapseEvent): + TYPE = "m.room.name" + + internal_keys = SynapseEvent.internal_keys + [ + "name", + ] + + def __init__(self, **kwargs): + kwargs["state_key"] = "" + if "name" in kwargs["content"]: + kwargs["name"] = kwargs["content"]["name"] + super(RoomNameEvent, self).__init__(**kwargs) + + def get_content_template(self): + return {"name": u"string"} + + class RoomMemberEvent(SynapseEvent): TYPE = "m.room.member" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 9b55206e4..5c1b59dbc 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -790,5 +790,5 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def get_public_room_list(self): - chunk = yield self.store.get_rooms(is_public=True, with_topics=True) + chunk = yield self.store.get_rooms(is_public=True) defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f41c21dcd..f62cee3c3 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -17,7 +17,7 @@ from twisted.internet import defer from synapse.api.events.room import ( RoomMemberEvent, MessageEvent, RoomTopicEvent, FeedbackEvent, - RoomConfigEvent + RoomConfigEvent, RoomNameEvent, ) from .directory import DirectoryStore @@ -52,6 +52,10 @@ class DataStore(RoomMemberStore, RoomStore, yield self._store_feedback(event) elif event.type == RoomConfigEvent.TYPE: yield self._store_room_config(event) + elif event.type == RoomNameEvent.TYPE: + yield self._store_room_name(event) + elif event.type == RoomTopicEvent.TYPE: + yield self._store_room_topic(event) yield self._store_event(event) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c8ec63f30..c26e9a0f9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -68,7 +68,7 @@ class SQLBaseStore(object): if decoder: return decoder(cursor) else: - return cursor + return cursor.fetchall() return self._db_pool.runInteraction(interaction) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index a97162831..8f44b67d8 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -76,49 +76,73 @@ class RoomStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_rooms(self, is_public, with_topics): + def get_rooms(self, is_public): """Retrieve a list of all public rooms. Args: is_public (bool): True if the rooms returned should be public. - with_topics (bool): True to include the current topic for the room - in the response. Returns: - A list of room dicts containing at least a "room_id" key, and a - "topic" key if one is set and with_topic=True. + A list of room dicts containing at least a "room_id" key, a + "topic" key if one is set, and a "name" key if one is set """ - room_data_type = RoomTopicEvent.TYPE - public = 1 if is_public else 0 - latest_topic = ("SELECT max(room_data.id) FROM room_data WHERE " - + "room_data.type = ? GROUP BY room_id") - - query = ("SELECT rooms.*, room_data.content, room_alias FROM rooms " - + "LEFT JOIN " - + "room_aliases ON room_aliases.room_id = rooms.room_id " - + "LEFT JOIN " - + "room_data ON rooms.room_id = room_data.room_id WHERE " - + "(room_data.id IN (" + latest_topic + ") " - + "OR room_data.id IS NULL) AND rooms.is_public = ?") - - res = yield self._execute( - self.cursor_to_dict, query, room_data_type, public + topic_subquery = ( + "SELECT topics.event_id as event_id, topics.room_id as room_id, topic FROM topics " + "INNER JOIN current_state_events as c " + "ON c.event_id = topics.event_id " ) - # return only the keys the specification expects - ret_keys = ["room_id", "topic", "room_alias"] + name_subquery = ( + "SELECT room_names.event_id as event_id, room_names.room_id as room_id, name FROM room_names " + "INNER JOIN current_state_events as c " + "ON c.event_id = room_names.event_id " + ) - # extract topic from the json (icky) FIXME - for i, room_row in enumerate(res): - try: - content_json = json.loads(room_row["content"]) - room_row["topic"] = content_json["topic"] - except: - pass # no topic set - # filter the dict based on ret_keys - res[i] = {k: v for k, v in room_row.iteritems() if k in ret_keys} + sql = ( + "SELECT r.room_id, n.name, t.topic, group_concat(a.room_alias) FROM rooms AS r " + "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " + "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " + "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " + "WHERE r.is_public = ? " + "GROUP BY r.room_id " + ) % { + "topic": topic_subquery, + "name": name_subquery, + } - defer.returnValue(res) + rows = yield self._execute(None, sql, is_public) + + ret = [ + { + "room_id": r[0], + "name": r[1], + "topic": r[2], + "aliases": r[3].split(","), + } + for r in rows + ] + + defer.returnValue(ret) + + def _store_room_topic(self, event): + return self._simple_insert( + "topics", + { + "event_id": event.event_id, + "room_id": event.room_id, + "topic": event.topic, + } + ) + + def _store_room_name(self, event): + return self._simple_insert( + "room_names", + { + "event_id": event.event_id, + "room_id": event.room_id, + "name": event.name, + } + ) class RoomsTable(Table): diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 85c0c7119..9a0f2145d 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -51,6 +51,18 @@ CREATE TABLE IF NOT EXISTS feedback( room_id TEXT ); +CREATE TABLE IF NOT EXISTS topics( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + topic TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS room_names( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + name TEXT NOT NULL +); + CREATE TABLE IF NOT EXISTS rooms( room_id TEXT PRIMARY KEY NOT NULL, is_public INTEGER, From 114984a2361ee41005a769f6dc127c470ee08aee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 13:58:28 +0100 Subject: [PATCH 013/112] Start chagning the events stream to work with the new DB schema --- synapse/api/streams/event.py | 77 +++------------------------------- synapse/handlers/events.py | 8 +--- synapse/handlers/room.py | 79 +++++++++++++++++++---------------- synapse/storage/__init__.py | 10 ++++- synapse/storage/_base.py | 2 +- synapse/storage/roommember.py | 8 +++- synapse/storage/schema/im.sql | 5 ++- synapse/storage/stream.py | 31 ++++++++++++++ 8 files changed, 102 insertions(+), 118 deletions(-) diff --git a/synapse/api/streams/event.py b/synapse/api/streams/event.py index 4b6d739e5..427363cad 100644 --- a/synapse/api/streams/event.py +++ b/synapse/api/streams/event.py @@ -28,17 +28,17 @@ import logging logger = logging.getLogger(__name__) -class MessagesStreamData(StreamData): - EVENT_TYPE = MessageEvent.TYPE +class EventsStreamData(StreamData): + EVENT_TYPE = "EventsStream" def __init__(self, hs, room_id=None, feedback=False): - super(MessagesStreamData, self).__init__(hs) + super(EventsStreamData, self).__init__(hs) self.room_id = room_id self.with_feedback = feedback @defer.inlineCallbacks def get_rows(self, user_id, from_key, to_key, limit): - (data, latest_ver) = yield self.store.get_message_stream( + data, latest_ver = yield self.store.get_room_events_stream( user_id=user_id, from_key=from_key, to_key=to_key, @@ -50,74 +50,7 @@ class MessagesStreamData(StreamData): @defer.inlineCallbacks def max_token(self): - val = yield self.store.get_max_message_id() - defer.returnValue(val) - - -class RoomMemberStreamData(StreamData): - EVENT_TYPE = RoomMemberEvent.TYPE - - @defer.inlineCallbacks - def get_rows(self, user_id, from_key, to_key, limit): - (data, latest_ver) = yield self.store.get_room_member_stream( - user_id=user_id, - from_key=from_key, - to_key=to_key - ) - - defer.returnValue((data, latest_ver)) - - @defer.inlineCallbacks - def max_token(self): - val = yield self.store.get_max_room_member_id() - defer.returnValue(val) - - -class FeedbackStreamData(StreamData): - EVENT_TYPE = FeedbackEvent.TYPE - - def __init__(self, hs, room_id=None): - super(FeedbackStreamData, self).__init__(hs) - self.room_id = room_id - - @defer.inlineCallbacks - def get_rows(self, user_id, from_key, to_key, limit): - (data, latest_ver) = yield self.store.get_feedback_stream( - user_id=user_id, - from_key=from_key, - to_key=to_key, - limit=limit, - room_id=self.room_id - ) - defer.returnValue((data, latest_ver)) - - @defer.inlineCallbacks - def max_token(self): - val = yield self.store.get_max_feedback_id() - defer.returnValue(val) - - -class RoomDataStreamData(StreamData): - EVENT_TYPE = RoomTopicEvent.TYPE # TODO need multiple event types - - def __init__(self, hs, room_id=None): - super(RoomDataStreamData, self).__init__(hs) - self.room_id = room_id - - @defer.inlineCallbacks - def get_rows(self, user_id, from_key, to_key, limit): - (data, latest_ver) = yield self.store.get_room_data_stream( - user_id=user_id, - from_key=from_key, - to_key=to_key, - limit=limit, - room_id=self.room_id - ) - defer.returnValue((data, latest_ver)) - - @defer.inlineCallbacks - def max_token(self): - val = yield self.store.get_max_room_data_id() + val = yield self.store.get_room_events_max_id() defer.returnValue(val) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 3af7d824a..6bb797caf 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -17,8 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.streams.event import ( - EventStream, MessagesStreamData, RoomMemberStreamData, FeedbackStreamData, - RoomDataStreamData + EventStream, EventsStreamData ) from synapse.handlers.presence import PresenceStreamData @@ -26,10 +25,7 @@ from synapse.handlers.presence import PresenceStreamData class EventStreamHandler(BaseHandler): stream_data_classes = [ - MessagesStreamData, - RoomMemberStreamData, - FeedbackStreamData, - RoomDataStreamData, + EventsStreamData, PresenceStreamData, ] diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 432d13982..345125000 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -23,7 +23,7 @@ from synapse.api.events.room import ( RoomTopicEvent, MessageEvent, InviteJoinEvent, RoomMemberEvent, RoomConfigEvent ) -from synapse.api.streams.event import EventStream, MessagesStreamData +from synapse.api.streams.event import EventStream, EventsStreamData from synapse.util import stringutils from ._base import BaseHandler @@ -97,30 +97,30 @@ class MessageHandler(BaseHandler): self.notifier.on_new_room_event(event, store_id) yield self.hs.get_federation().handle_new_event(event) - - @defer.inlineCallbacks - def get_messages(self, user_id=None, room_id=None, pagin_config=None, - feedback=False): - """Get messages in a room. - - Args: - user_id (str): The user requesting messages. - room_id (str): The room they want messages from. - pagin_config (synapse.api.streams.PaginationConfig): The pagination - config rules to apply, if any. - feedback (bool): True to get compressed feedback with the messages - Returns: - dict: Pagination API results - """ - yield self.auth.check_joined_room(room_id, user_id) - - data_source = [MessagesStreamData(self.hs, room_id=room_id, - feedback=feedback)] - event_stream = EventStream(user_id, data_source) - pagin_config = yield event_stream.fix_tokens(pagin_config) - data_chunk = yield event_stream.get_chunk(config=pagin_config) - defer.returnValue(data_chunk) - +# +# @defer.inlineCallbacks +# def get_messages(self, user_id=None, room_id=None, pagin_config=None, +# feedback=False): +# """Get messages in a room. +# +# Args: +# user_id (str): The user requesting messages. +# room_id (str): The room they want messages from. +# pagin_config (synapse.api.streams.PaginationConfig): The pagination +# config rules to apply, if any. +# feedback (bool): True to get compressed feedback with the messages +# Returns: +# dict: Pagination API results +# """ +# yield self.auth.check_joined_room(room_id, user_id) +# +# data_source = [MessagesStreamData(self.hs, room_id=room_id, +# feedback=feedback)] +# event_stream = EventStream(user_id, data_source) +# pagin_config = yield event_stream.fix_tokens(pagin_config) +# data_chunk = yield event_stream.get_chunk(config=pagin_config) +# defer.returnValue(data_chunk) +# @defer.inlineCallbacks def store_room_data(self, event=None, stamp_event=True): """ Stores data for a room. @@ -251,20 +251,27 @@ class MessageHandler(BaseHandler): user_id=user_id, membership_list=[Membership.INVITE, Membership.JOIN] ) - for room_info in room_list: - if room_info["membership"] != Membership.JOIN: + + ret = [] + + for event in room_list: + d = event.get_dict() + ret.append(d) + + if event.membership != Membership.JOIN: continue try: - event_chunk = yield self.get_messages( - user_id=user_id, - pagin_config=pagin_config, - feedback=feedback, - room_id=room_info["room_id"] + messages = yield self.store.get_recent_events_for_room( + event.room_id, + limit=50, ) - room_info["messages"] = event_chunk + d["messages"] = [m.get_dict() for m in messages] except: pass - defer.returnValue(room_list) + + logger.debug("snapshot_all_rooms returning: %s", ret) + + defer.returnValue(ret) class RoomCreationHandler(BaseHandler): @@ -442,7 +449,7 @@ class RoomMemberHandler(BaseHandler): member_list = yield self.store.get_room_members(room_id=room_id) event_list = [ - entry.as_event(self.event_factory).get_dict() + entry.get_dict() for entry in member_list ] chunk_data = { @@ -685,7 +692,7 @@ class RoomMemberHandler(BaseHandler): user_id=user.to_string(), membership_list=membership_list ) - defer.returnValue([r["room_id"] for r in rooms]) + defer.returnValue([r.room_id for r in rooms]) @defer.inlineCallbacks def _do_local_membership_update(self, event, membership, broadcast_msg): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f62cee3c3..46b9dbcbb 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -104,7 +104,15 @@ class DataStore(RoomMemberStore, RoomStore, yield self._simple_insert("state_events", vals) - # TODO (erikj): We also need to update the current state table? + yield self._simple_insert( + "current_state_events", + { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + ) @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c26e9a0f9..413838f79 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -293,7 +293,7 @@ class SQLBaseStore(object): def _parse_event_from_row(self, row_dict): d = copy.deepcopy({k: v for k, v in row_dict.items() if v}) - d.update(json.loads(json.loads(row_dict["unrecognized_keys"]))) + d.update(json.loads(row_dict["unrecognized_keys"])) d["content"] = json.loads(d["content"]) del d["unrecognized_keys"] diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 8c4b04f19..a0620677b 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -63,6 +63,7 @@ class RoomMemberStore(SQLBaseStore): yield self._execute(None, sql, event.room_id, domain) + @defer.inlineCallbacks def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. @@ -72,11 +73,13 @@ class RoomMemberStore(SQLBaseStore): Returns: Deferred: Results in a MembershipEvent or None. """ - return self._get_members_by_dict({ + rows = yield self._get_members_by_dict({ "e.room_id": room_id, "m.user_id": user_id, }) + defer.returnValue(rows[0] if rows else None) + def get_room_members(self, room_id, membership=None): """Retrieve the current room member list for a room. @@ -142,5 +145,8 @@ class RoomMemberStore(SQLBaseStore): ) % (where_clause,) rows = yield self._execute_and_decode(sql, *where_values) + + logger.debug("_get_members_query Got rows %s", rows) + results = [self._parse_event_from_row(r) for r in rows] defer.returnValue(results) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 9a0f2145d..2452890ea 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -32,7 +32,10 @@ CREATE TABLE IF NOT EXISTS state_events( CREATE TABLE IF NOT EXISTS current_state_events( event_id TEXT NOT NULL, - room_id TEXT NOT NULL + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + CONSTRAINT uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE ); CREATE TABLE IF NOT EXISTS room_memberships( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 9937239c2..c5c3770a4 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -34,6 +34,7 @@ class StreamStore(SQLBaseStore): @defer.inlineCallbacks def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): + # TODO (erikj): Handle compressed feedback current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " @@ -69,3 +70,33 @@ class StreamStore(SQLBaseStore): ) defer.returnValue([self._parse_event_from_row(r) for r in rows]) + + @defer.inlineCallbacks + def get_recent_events_for_room(self, room_id, limit, with_feedback=False): + # TODO (erikj): Handle compressed feedback + + sql = ( + "SELECT * FROM events WHERE room_id = ? " + "ORDER BY ordering DESC LIMIT ? " + ) + + rows = yield self._execute_and_decode( + sql, + room_id, limit + ) + + rows.reverse() # As we selected with reverse ordering + + defer.returnValue([self._parse_event_from_row(r) for r in rows]) + + @defer.inlineCallbacks + def get_room_events_max_id(self): + res = yield self._execute_and_decode( + "SELECT MAX(ordering) as m FROM events" + ) + + if not res: + defer.returnValue(0) + return + + defer.returnValue(res[0]["m"]) From 01f089d9fbb9b89fa143ac44e51529fa8ed7ec12 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 15:28:54 +0100 Subject: [PATCH 014/112] Correctly return new token when returning events. Serialize events correctly. --- synapse/api/notifier.py | 3 ++- synapse/api/streams/event.py | 2 +- synapse/handlers/room.py | 5 ++++- synapse/storage/__init__.py | 6 +++++- synapse/storage/stream.py | 18 +++++++++++++----- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/synapse/api/notifier.py b/synapse/api/notifier.py index 65b5a4ebb..9f622df6b 100644 --- a/synapse/api/notifier.py +++ b/synapse/api/notifier.py @@ -15,6 +15,7 @@ from synapse.api.constants import Membership from synapse.api.events.room import RoomMemberEvent +from synapse.api.streams.event import EventsStreamData from twisted.internet import defer from twisted.internet import reactor @@ -66,7 +67,7 @@ class Notifier(object): self._notify_and_callback( user_id=user_id, event_data=event.get_dict(), - stream_type=event.type, + stream_type=EventsStreamData.EVENT_TYPE, store_id=store_id) def on_new_user_event(self, user_id, event_data, stream_type, store_id): diff --git a/synapse/api/streams/event.py b/synapse/api/streams/event.py index 427363cad..895a96b5b 100644 --- a/synapse/api/streams/event.py +++ b/synapse/api/streams/event.py @@ -160,7 +160,7 @@ class EventStream(PaginationStream): self.user_id, from_pkey, to_pkey, limit ) - chunk += event_chunk + chunk += [e.get_dict() for e in event_chunk] next_ver.append(str(max_pkey)) defer.returnValue((chunk, EventStream.SEPARATOR.join(next_ver))) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 345125000..9261984b7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -255,7 +255,10 @@ class MessageHandler(BaseHandler): ret = [] for event in room_list: - d = event.get_dict() + d = { + "room_id": event.room_id, + "membership": event.membership, + } ret.append(d) if event.membership != Membership.JOIN: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 46b9dbcbb..750e86040 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -57,7 +57,8 @@ class DataStore(RoomMemberStore, RoomStore, elif event.type == RoomTopicEvent.TYPE: yield self._store_room_topic(event) - yield self._store_event(event) + ret = yield self._store_event(event) + defer.returnValue(ret) @defer.inlineCallbacks def get_event(self, event_id): @@ -114,6 +115,9 @@ class DataStore(RoomMemberStore, RoomStore, } ) + latest = yield self.get_room_events_max_id() + defer.returnValue(latest) + @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): sql = ( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index c5c3770a4..1300aee8b 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -43,7 +43,7 @@ class StreamStore(SQLBaseStore): ) invites_sql = ( - "SELECT m.event_id FROM room_membershipas as m " + "SELECT m.event_id FROM room_memberships as m " "INNER JOIN current_state_events as c ON m.event_id = c.event_id " "WHERE m.user_id = ? AND m.membership = ?" ) @@ -55,8 +55,9 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events as e WHERE " - "(room_id IN (%(current)s)) OR " - "(event_id IN (%(invites)s)) " + "((room_id IN (%(current)s)) OR " + "(event_id IN (%(invites)s))) " + " AND e.ordering > ? AND e.ordering < ? " "ORDER BY ordering ASC LIMIT %(limit)d" ) % { "current": current_room_membership_sql, @@ -66,10 +67,17 @@ class StreamStore(SQLBaseStore): rows = yield self._execute_and_decode( sql, - user_id, user_id, Membership.INVITE + user_id, user_id, Membership.INVITE, from_key, to_key ) - defer.returnValue([self._parse_event_from_row(r) for r in rows]) + ret = [self._parse_event_from_row(r) for r in rows] + + if ret: + max_id = max([r["ordering"] for r in rows]) + else: + max_id = to_key + + defer.returnValue((ret, max_id)) @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, with_feedback=False): From 8d1f7632095b949d7726dd72fce10224764f3c11 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 15:53:06 +0100 Subject: [PATCH 015/112] Fix pagination to work with new db schema --- synapse/handlers/room.py | 48 +++++++++++++++++++-------------------- synapse/storage/stream.py | 31 +++++++++++++++++++------ 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 9261984b7..b0b2441b9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -97,30 +97,30 @@ class MessageHandler(BaseHandler): self.notifier.on_new_room_event(event, store_id) yield self.hs.get_federation().handle_new_event(event) -# -# @defer.inlineCallbacks -# def get_messages(self, user_id=None, room_id=None, pagin_config=None, -# feedback=False): -# """Get messages in a room. -# -# Args: -# user_id (str): The user requesting messages. -# room_id (str): The room they want messages from. -# pagin_config (synapse.api.streams.PaginationConfig): The pagination -# config rules to apply, if any. -# feedback (bool): True to get compressed feedback with the messages -# Returns: -# dict: Pagination API results -# """ -# yield self.auth.check_joined_room(room_id, user_id) -# -# data_source = [MessagesStreamData(self.hs, room_id=room_id, -# feedback=feedback)] -# event_stream = EventStream(user_id, data_source) -# pagin_config = yield event_stream.fix_tokens(pagin_config) -# data_chunk = yield event_stream.get_chunk(config=pagin_config) -# defer.returnValue(data_chunk) -# + + @defer.inlineCallbacks + def get_messages(self, user_id=None, room_id=None, pagin_config=None, + feedback=False): + """Get messages in a room. + + Args: + user_id (str): The user requesting messages. + room_id (str): The room they want messages from. + pagin_config (synapse.api.streams.PaginationConfig): The pagination + config rules to apply, if any. + feedback (bool): True to get compressed feedback with the messages + Returns: + dict: Pagination API results + """ + yield self.auth.check_joined_room(room_id, user_id) + + data_source = [EventsStreamData(self.hs, room_id=room_id, + feedback=feedback)] + event_stream = EventStream(user_id, data_source) + pagin_config = yield event_stream.fix_tokens(pagin_config) + data_chunk = yield event_stream.get_chunk(config=pagin_config) + defer.returnValue(data_chunk) + @defer.inlineCallbacks def store_room_data(self, event=None, stamp_event=True): """ Stores data for a room. diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 1300aee8b..6bfa00d59 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -53,18 +53,35 @@ class StreamStore(SQLBaseStore): else: limit = 1000 + # From and to keys should be integers from ordering. + from_key = int(from_key) + to_key = int(to_key) + + if from_key == to_key: + defer.returnValue(([], to_key)) + return + + sql = ( "SELECT * FROM events as e WHERE " "((room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " - " AND e.ordering > ? AND e.ordering < ? " - "ORDER BY ordering ASC LIMIT %(limit)d" ) % { "current": current_room_membership_sql, "invites": invites_sql, - "limit": limit, } + if from_key < to_key: + sql += ( + "AND e.ordering > ? AND e.ordering < ? " + "ORDER BY ordering ASC LIMIT %(limit)d " + ) % {"limit": limit} + else: + sql += ( + "AND e.ordering < ? AND e.ordering > ? " + "ORDER BY ordering DESC LIMIT %(limit)d " + ) % {"limit": int(limit)} + rows = yield self._execute_and_decode( sql, user_id, user_id, Membership.INVITE, from_key, to_key @@ -72,12 +89,12 @@ class StreamStore(SQLBaseStore): ret = [self._parse_event_from_row(r) for r in rows] - if ret: - max_id = max([r["ordering"] for r in rows]) + if from_key < to_key: + key = max([r["ordering"] for r in rows]) else: - max_id = to_key + key = min([r["ordering"] for r in rows]) - defer.returnValue((ret, max_id)) + defer.returnValue((ret, key)) @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, with_feedback=False): From 86be66c34e3d9d3f6b8e6a40ed239f4803550e55 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 16:04:54 +0100 Subject: [PATCH 016/112] Actually use MAX_STREAM_SIZE constant. --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 6bfa00d59..47f05a41b 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -51,7 +51,7 @@ class StreamStore(SQLBaseStore): if limit: limit = max(limit, MAX_STREAM_SIZE) else: - limit = 1000 + limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. from_key = int(from_key) From cd2967d271d8127b6f6ebf4aa7a671d3eeca3c59 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 16:05:46 +0100 Subject: [PATCH 017/112] Fix bug when generating a key when get_room_events_stream returned zero rows --- synapse/storage/stream.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 47f05a41b..bb56f0763 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -89,10 +89,14 @@ class StreamStore(SQLBaseStore): ret = [self._parse_event_from_row(r) for r in rows] - if from_key < to_key: - key = max([r["ordering"] for r in rows]) + + if rows: + if from_key < to_key: + key = max([r["ordering"] for r in rows]) + else: + key = min([r["ordering"] for r in rows]) else: - key = min([r["ordering"] for r in rows]) + key = to_key defer.returnValue((ret, key)) From 19946509a4ddbdd647261f9f19358e6f1f0e337b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 16:06:08 +0100 Subject: [PATCH 018/112] Support generic events. --- synapse/api/events/factory.py | 9 ++++----- synapse/api/events/room.py | 4 ++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 23d2b0401..b61dac7ac 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -15,7 +15,7 @@ from synapse.api.events.room import ( RoomTopicEvent, MessageEvent, RoomMemberEvent, FeedbackEvent, - InviteJoinEvent, RoomConfigEvent, RoomNameEvent, + InviteJoinEvent, RoomConfigEvent, RoomNameEvent, GenericEvent, ) from synapse.util.stringutils import random_string @@ -43,10 +43,9 @@ class EventFactory(object): if "event_id" not in kwargs: kwargs["event_id"] = random_string(10) - try: + if etype in self._event_list: handler = self._event_list[etype] - except KeyError: # unknown event type - # TODO allow custom event types. - raise NotImplementedError("Unknown etype=%s" % etype) + else: + handler = GenericEvent return handler(**kwargs) diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index 8136d495d..d9b3d572f 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -16,6 +16,10 @@ from . import SynapseEvent +class GenericEvent(SynapseEvent): + def get_content_template(self): + return {} + class RoomTopicEvent(SynapseEvent): TYPE = "m.room.topic" From 8fa3cc37f93cc90d8bf34dadbdf207eb8e2fdebd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 16:11:25 +0100 Subject: [PATCH 019/112] Comment. --- synapse/storage/stream.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index bb56f0763..7c2c45e0f 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -71,6 +71,7 @@ class StreamStore(SQLBaseStore): "invites": invites_sql, } + # Constraints and ordering depend on direction. if from_key < to_key: sql += ( "AND e.ordering > ? AND e.ordering < ? " From d260a42ca279fbca46f85b2c96bddc4f814ecef3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 16:17:36 +0100 Subject: [PATCH 020/112] PEP8 cleanups --- synapse/api/events/room.py | 1 + synapse/handlers/room.py | 17 ++++++++++------- synapse/storage/__init__.py | 6 +++++- synapse/storage/_base.py | 3 ++- synapse/storage/room.py | 15 +++++++++++---- synapse/storage/roommember.py | 1 - synapse/storage/stream.py | 2 -- 7 files changed, 29 insertions(+), 16 deletions(-) diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index d9b3d572f..dbf537fb8 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -20,6 +20,7 @@ class GenericEvent(SynapseEvent): def get_content_template(self): return {} + class RoomTopicEvent(SynapseEvent): TYPE = "m.room.topic" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b0b2441b9..4ecfb278b 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -114,8 +114,9 @@ class MessageHandler(BaseHandler): """ yield self.auth.check_joined_room(room_id, user_id) - data_source = [EventsStreamData(self.hs, room_id=room_id, - feedback=feedback)] + data_source = [ + EventsStreamData(self.hs, room_id=room_id, feedback=feedback) + ] event_stream = EventStream(user_id, data_source) pagin_config = yield event_stream.fix_tokens(pagin_config) data_chunk = yield event_stream.get_chunk(config=pagin_config) @@ -196,7 +197,9 @@ class MessageHandler(BaseHandler): raise RoomError( 403, "Member does not meet private room rules.") - data = yield self.store.get_current_state(room_id, event_type, state_key) + data = yield self.store.get_current_state( + room_id, event_type, state_key + ) defer.returnValue(data) @defer.inlineCallbacks @@ -496,7 +499,7 @@ class RoomMemberHandler(BaseHandler): SynapseError if there was a problem changing the membership. """ - #broadcast_msg = False + # broadcast_msg = False prev_state = yield self.store.get_room_member( event.target_user_id, event.room_id @@ -570,7 +573,8 @@ class RoomMemberHandler(BaseHandler): defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks - def _do_join(self, event, room_host=None, do_auth=True, broadcast_msg=True): + def _do_join(self, event, room_host=None, do_auth=True, + broadcast_msg=True): joinee = self.hs.parse_userid(event.target_user_id) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -621,7 +625,6 @@ class RoomMemberHandler(BaseHandler): broadcast_msg=broadcast_msg, ) - if should_do_dance: yield self._do_invite_join_dance( room_id=room_id, @@ -755,7 +758,7 @@ class RoomMemberHandler(BaseHandler): room_id, "", is_public=False ) - #yield self.state_handler.handle_new_event(event) + # yield self.state_handler.handle_new_event(event) yield federation.handle_new_event(new_event) yield federation.get_state_for_room( target_host, room_id diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 750e86040..ad36d14a3 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -87,7 +87,11 @@ class DataStore(RoomMemberStore, RoomStore, "content": json.dumps(event.content), } - unrec = {k: v for k, v in event.get_full_dict().items() if k not in vals.keys()} + unrec = { + k: v + for k, v in event.get_full_dict().items() + if k not in vals.keys() + } vals["unrecognized_keys"] = json.dumps(unrec) yield self._simple_insert("events", vals) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 413838f79..36cc57c1b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -60,7 +60,8 @@ class SQLBaseStore(object): The result of decoder(results) """ logger.debug( - "[SQL] %s Args=%s Func=%s", query, args, decoder.__name__ if decoder else None + "[SQL] %s Args=%s Func=%s", + query, args, decoder.__name__ if decoder else None ) def interaction(txn): diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 8f44b67d8..22f2dcca4 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -87,19 +87,26 @@ class RoomStore(SQLBaseStore): """ topic_subquery = ( - "SELECT topics.event_id as event_id, topics.room_id as room_id, topic FROM topics " + "SELECT topics.event_id as event_id, " + "topics.room_id as room_id, topic " + "FROM topics " "INNER JOIN current_state_events as c " "ON c.event_id = topics.event_id " ) name_subquery = ( - "SELECT room_names.event_id as event_id, room_names.room_id as room_id, name FROM room_names " + "SELECT room_names.event_id as event_id, " + "room_names.room_id as room_id, name " + "FROM room_names " "INNER JOIN current_state_events as c " "ON c.event_id = room_names.event_id " ) + # We use non printing ascii character US () as a seperator sql = ( - "SELECT r.room_id, n.name, t.topic, group_concat(a.room_alias) FROM rooms AS r " + "SELECT r.room_id, n.name, t.topic, " + "group_concat(a.room_alias, '') " + "FROM rooms AS r " "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " @@ -117,7 +124,7 @@ class RoomStore(SQLBaseStore): "room_id": r[0], "name": r[1], "topic": r[2], - "aliases": r[3].split(","), + "aliases": r[3].split(""), } for r in rows ] diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index a0620677b..89c87290c 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -62,7 +62,6 @@ class RoomMemberStore(SQLBaseStore): yield self._execute(None, sql, event.room_id, domain) - @defer.inlineCallbacks def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 7c2c45e0f..cf4b1682b 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -61,7 +61,6 @@ class StreamStore(SQLBaseStore): defer.returnValue(([], to_key)) return - sql = ( "SELECT * FROM events as e WHERE " "((room_id IN (%(current)s)) OR " @@ -90,7 +89,6 @@ class StreamStore(SQLBaseStore): ret = [self._parse_event_from_row(r) for r in rows] - if rows: if from_key < to_key: key = max([r["ordering"] for r in rows]) From 506711749f674c2b408f4034662240883e0e2764 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 16:45:16 +0100 Subject: [PATCH 021/112] We no longer need to special case room config events. --- synapse/storage/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index ad36d14a3..841ad8f13 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -50,8 +50,8 @@ class DataStore(RoomMemberStore, RoomStore, yield self._store_room_member(event) elif event.type == FeedbackEvent.TYPE: yield self._store_feedback(event) - elif event.type == RoomConfigEvent.TYPE: - yield self._store_room_config(event) +# elif event.type == RoomConfigEvent.TYPE: +# yield self._store_room_config(event) elif event.type == RoomNameEvent.TYPE: yield self._store_room_name(event) elif event.type == RoomTopicEvent.TYPE: From 6efc688917cac03f91c7337015741f2c6e82e0f1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 16:47:09 +0100 Subject: [PATCH 022/112] Fix typo of key name --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 4ecfb278b..14ffddc63 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -603,7 +603,7 @@ class RoomMemberHandler(BaseHandler): if prev_state and prev_state.membership == Membership.INVITE: room = yield self.store.get_room(room_id) inviter = UserID.from_string( - prev_state.sender, self.hs + prev_state.user_id, self.hs ) should_do_dance = not inviter.is_mine and not room From 0e938b1ff75bdeffefb3f3a3260972a19d99d4d9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 16:47:48 +0100 Subject: [PATCH 023/112] Rename method name to not clash with other ones in storage. --- synapse/state.py | 2 +- synapse/storage/pdu.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 4f8b4d976..ca8e1ca63 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -86,7 +86,7 @@ class StateHandler(object): else: event.depth = 0 - current_state = yield self.store.get_current_state( + current_state = yield self.store.get_current_state_pdu( key.context, key.type, key.state_key ) diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 13adc581e..a24ce7ab7 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -580,7 +580,7 @@ class StatePduStore(SQLBaseStore): txn.execute(query, query_args) - def get_current_state(self, context, pdu_type, state_key): + def get_current_state_pdu(self, context, pdu_type, state_key): """For a given context, pdu_type, state_key 3-tuple, return what is currently considered the current state. @@ -595,10 +595,10 @@ class StatePduStore(SQLBaseStore): """ return self._db_pool.runInteraction( - self._get_current_state, context, pdu_type, state_key + self._get_current_state_pdu, context, pdu_type, state_key ) - def _get_current_state(self, txn, context, pdu_type, state_key): + def _get_current_state_pdu(self, txn, context, pdu_type, state_key): return self._get_current_interaction(txn, context, pdu_type, state_key) def _get_current_interaction(self, txn, context, pdu_type, state_key): From f5fca6f78789e1f9711924fd4dad29c8c12b692f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Aug 2014 17:42:21 +0100 Subject: [PATCH 024/112] Fix some of the tests to reflect changes in the storage layer. --- tests/handlers/test_room.py | 22 +++++++--------------- tests/test_state.py | 4 ++-- tests/utils.py | 24 ++---------------------- 3 files changed, 11 insertions(+), 39 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index fd2d66db3..bfdde6135 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -40,7 +40,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.hostname, db_pool=None, datastore=NonCallableMock(spec_set=[ - "store_room_member", + "persist_event", "get_joined_hosts_for_room", "get_room_member", "get_room", @@ -97,7 +97,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ) store_id = "store_id_fooo" - self.datastore.store_room_member.return_value = defer.succeed(store_id) + self.datastore.persist_event.return_value = defer.succeed(store_id) # Actual invocation yield self.room_member_handler.change_membership(event) @@ -110,12 +110,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): set(event.destinations) ) - self.datastore.store_room_member.assert_called_once_with( - user_id=target_user_id, - sender=user_id, - room_id=room_id, - content=content, - membership=Membership.INVITE, + self.datastore.persist_event.assert_called_once_with( + event ) self.notifier.on_new_room_event.assert_called_once_with( event, store_id) @@ -149,7 +145,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ) store_id = "store_id_fooo" - self.datastore.store_room_member.return_value = defer.succeed(store_id) + self.datastore.persist_event.return_value = defer.succeed(store_id) self.datastore.get_room.return_value = defer.succeed(1) # Not None. prev_state = NonCallableMock() @@ -171,12 +167,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): set(event.destinations) ) - self.datastore.store_room_member.assert_called_once_with( - user_id=target_user_id, - sender=user_id, - room_id=room_id, - content=content, - membership=Membership.JOIN, + self.datastore.persist_event.assert_called_once_with( + event ) self.notifier.on_new_room_event.assert_called_once_with( event, store_id) diff --git a/tests/test_state.py b/tests/test_state.py index aaf873a85..e64d15a3a 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -36,7 +36,7 @@ class StateTestCase(unittest.TestCase): "get_unresolved_state_tree", "update_current_state", "get_latest_pdus_in_context", - "get_current_state", + "get_current_state_pdu", "get_pdu", ]) self.replication = Mock(spec=["get_pdu"]) @@ -247,7 +247,7 @@ class StateTestCase(unittest.TestCase): pdus = [tup] self.persistence.get_latest_pdus_in_context.return_value = pdus - self.persistence.get_current_state.return_value = state_pdu + self.persistence.get_current_state_pdu.return_value = state_pdu yield self.state.handle_new_event(event) diff --git a/tests/utils.py b/tests/utils.py index 20a63316f..717b81e6e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -221,34 +221,14 @@ class MemoryDataStore(object): content=content) self.paths_to_content[path] = data - def get_message_stream(self, user_id=None, from_key=None, to_key=None, + def get_room_events_stream(self, user_id=None, from_key=None, to_key=None, room_id=None, limit=0, with_feedback=False): return ([], from_key) # TODO - def get_room_member_stream(self, user_id=None, from_key=None, to_key=None): - return ([], from_key) # TODO - - def get_feedback_stream(self, user_id=None, from_key=None, to_key=None, - room_id=None, limit=0): - return ([], from_key) # TODO - - def get_room_data_stream(self, user_id=None, from_key=None, to_key=None, - room_id=None, limit=0): - return ([], from_key) # TODO - def to_events(self, data_store_list): return data_store_list # TODO - def get_max_message_id(self): - return 0 # TODO - - def get_max_feedback_id(self): - return 0 # TODO - - def get_max_room_member_id(self): - return 0 # TODO - - def get_max_room_data_id(self): + def get_room_events_max_id(self): return 0 # TODO def get_joined_hosts_for_room(self, room_id): From dccb2f57be566c4cbf8cc413c78eed79036d7049 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 10:59:04 +0100 Subject: [PATCH 025/112] Disable the ability to GET individualy messages. We need to think about the correct API to do this, as the current one doesn't make much sense. --- synapse/handlers/room.py | 12 ++++--- tests/rest/test_rooms.py | 72 ++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 14ffddc63..cdc98d2b0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -59,12 +59,14 @@ class MessageHandler(BaseHandler): yield self.auth.check_joined_room(room_id, user_id) # Pull out the message from the db - msg = yield self.store.get_message(room_id=room_id, - msg_id=msg_id, - user_id=sender_id) +# msg = yield self.store.get_message( +# room_id=room_id, +# msg_id=msg_id, +# user_id=sender_id +# ) + + # TODO (erikj): Once we work out the correct c-s api we need to think on how to do this. - if msg: - defer.returnValue(msg) defer.returnValue(None) @defer.inlineCallbacks diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 86025103d..3ac2bbdd0 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -104,36 +104,36 @@ class RoomPermissionsTestCase(RestTestCase): def tearDown(self): pass - @defer.inlineCallbacks - def test_get_message(self): - # get message in uncreated room, expect 403 - (code, response) = yield self.mock_server.trigger_get( - "/rooms/noroom/messages/someid/m1") - self.assertEquals(403, code, msg=str(response)) - - # get message in created room not joined (no state), expect 403 - (code, response) = yield self.mock_server.trigger_get( - self.created_rmid_msg_path) - self.assertEquals(403, code, msg=str(response)) - - # get message in created room and invited, expect 403 - yield self.invite(room=self.created_rmid, src=self.rmcreator_id, - targ=self.user_id) - (code, response) = yield self.mock_server.trigger_get( - self.created_rmid_msg_path) - self.assertEquals(403, code, msg=str(response)) - - # get message in created room and joined, expect 200 - yield self.join(room=self.created_rmid, user=self.user_id) - (code, response) = yield self.mock_server.trigger_get( - self.created_rmid_msg_path) - self.assertEquals(200, code, msg=str(response)) - - # get message in created room and left, expect 403 - yield self.leave(room=self.created_rmid, user=self.user_id) - (code, response) = yield self.mock_server.trigger_get( - self.created_rmid_msg_path) - self.assertEquals(403, code, msg=str(response)) +# @defer.inlineCallbacks +# def test_get_message(self): +# # get message in uncreated room, expect 403 +# (code, response) = yield self.mock_server.trigger_get( +# "/rooms/noroom/messages/someid/m1") +# self.assertEquals(403, code, msg=str(response)) +# +# # get message in created room not joined (no state), expect 403 +# (code, response) = yield self.mock_server.trigger_get( +# self.created_rmid_msg_path) +# self.assertEquals(403, code, msg=str(response)) +# +# # get message in created room and invited, expect 403 +# yield self.invite(room=self.created_rmid, src=self.rmcreator_id, +# targ=self.user_id) +# (code, response) = yield self.mock_server.trigger_get( +# self.created_rmid_msg_path) +# self.assertEquals(403, code, msg=str(response)) +# +# # get message in created room and joined, expect 200 +# yield self.join(room=self.created_rmid, user=self.user_id) +# (code, response) = yield self.mock_server.trigger_get( +# self.created_rmid_msg_path) +# self.assertEquals(200, code, msg=str(response)) +# +# # get message in created room and left, expect 403 +# yield self.leave(room=self.created_rmid, user=self.user_id) +# (code, response) = yield self.mock_server.trigger_get( +# self.created_rmid_msg_path) +# self.assertEquals(403, code, msg=str(response)) @defer.inlineCallbacks def test_send_message(self): @@ -913,9 +913,9 @@ class RoomMessagesTestCase(RestTestCase): (code, response) = yield self.mock_server.trigger("PUT", path, content) self.assertEquals(200, code, msg=str(response)) - (code, response) = yield self.mock_server.trigger("GET", path, None) - self.assertEquals(200, code, msg=str(response)) - self.assert_dict(json.loads(content), response) + # (code, response) = yield self.mock_server.trigger("GET", path, None) + # self.assertEquals(200, code, msg=str(response)) + # self.assert_dict(json.loads(content), response) # m.text message type path = "/rooms/%s/messages/%s/mid2" % ( @@ -925,9 +925,9 @@ class RoomMessagesTestCase(RestTestCase): (code, response) = yield self.mock_server.trigger("PUT", path, content) self.assertEquals(200, code, msg=str(response)) - (code, response) = yield self.mock_server.trigger("GET", path, None) - self.assertEquals(200, code, msg=str(response)) - self.assert_dict(json.loads(content), response) + # (code, response) = yield self.mock_server.trigger("GET", path, None) + # self.assertEquals(200, code, msg=str(response)) + # self.assert_dict(json.loads(content), response) # trying to send message in different user path path = "/rooms/%s/messages/%s/mid2" % ( From bc2512fa9515a447f97e79b398413b5f3eda36c5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 10:59:34 +0100 Subject: [PATCH 026/112] Don't bother generating png's --- graph/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph/graph.py b/graph/graph.py index da829c388..220f5eb1d 100644 --- a/graph/graph.py +++ b/graph/graph.py @@ -113,7 +113,7 @@ def make_graph(pdus, room, filename_prefix): graph.add_edge(state_edge) graph.write('%s.dot' % filename_prefix, format='raw', prog='dot') - graph.write_png("%s.png" % filename_prefix, prog='dot') +# graph.write_png("%s.png" % filename_prefix, prog='dot') graph.write_svg("%s.svg" % filename_prefix, prog='dot') From 1a1e0384efb213acd74aeb8d669b1d43f18de667 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 10:59:57 +0100 Subject: [PATCH 027/112] Ensure we have a 'membership' key in RoomMemberEvents --- synapse/api/events/room.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index dbf537fb8..42459f3f2 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -66,6 +66,8 @@ class RoomMemberEvent(SynapseEvent): def __init__(self, **kwargs): if "target_user_id" in kwargs: kwargs["state_key"] = kwargs["target_user_id"] + if "membership" not in kwargs: + kwargs["membership"] = kwargs.get("content", {}).get("membership") super(RoomMemberEvent, self).__init__(**kwargs) def get_content_template(self): From 2f91d16033a87f7f52aa722c91765cc488853795 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 11:00:22 +0100 Subject: [PATCH 028/112] We don't need to do a json.loads here --- synapse/rest/room.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 303759b71..fe0953c69 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -115,7 +115,7 @@ class RoomTopicRestServlet(RestServlet): if not data: raise SynapseError(404, "Topic not found.", errcode=Codes.NOT_FOUND) - defer.returnValue((200, json.loads(data.content))) + defer.returnValue((200, data.content)) @defer.inlineCallbacks def on_PUT(self, request, room_id): @@ -175,7 +175,7 @@ class RoomMemberRestServlet(RestServlet): if not member: raise SynapseError(404, "Member not found.", errcode=Codes.NOT_FOUND) - defer.returnValue((200, json.loads(member.content))) + defer.returnValue((200, member.content)) @defer.inlineCallbacks def on_DELETE(self, request, roomid, target_user_id): From 291010f100f20ef56b731ae51341abdc4d4a7835 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 11:06:59 +0100 Subject: [PATCH 029/112] Not all event streams returns SynapseEvents --- synapse/api/streams/event.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/api/streams/event.py b/synapse/api/streams/event.py index 895a96b5b..414b05be3 100644 --- a/synapse/api/streams/event.py +++ b/synapse/api/streams/event.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.errors import EventStreamError +from synapse.api.events import SynapseEvent from synapse.api.events.room import ( RoomMemberEvent, MessageEvent, FeedbackEvent, RoomTopicEvent ) @@ -160,7 +161,10 @@ class EventStream(PaginationStream): self.user_id, from_pkey, to_pkey, limit ) - chunk += [e.get_dict() for e in event_chunk] + chunk.extend([ + e.get_dict() if isinstance(e, SynapseEvent) else e + for e in event_chunk + ]) next_ver.append(str(max_pkey)) defer.returnValue((chunk, EventStream.SEPARATOR.join(next_ver))) From 663a259d64a29248824935dc0c53aabd433e0ee2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 11:08:03 +0100 Subject: [PATCH 030/112] Change the MemoryDataStore to implement new storage api --- tests/utils.py | 149 +++++++++++++++++-------------------------------- 1 file changed, 50 insertions(+), 99 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 717b81e6e..990380fb1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -111,35 +111,20 @@ class MockClock(object): class MemoryDataStore(object): - class RoomMember(namedtuple( - "RoomMember", - ["room_id", "user_id", "sender", "membership", "content"] - )): - def as_event(self, event_factory): - return event_factory.create_event( - etype=RoomMemberEvent.TYPE, - room_id=self.room_id, - target_user_id=self.user_id, - user_id=self.sender, - content=json.loads(self.content), - ) - - PathData = namedtuple("PathData", - ["room_id", "path", "content"]) - - Message = namedtuple("Message", - ["room_id", "msg_id", "user_id", "content"]) - - Room = namedtuple("Room", - ["room_id", "is_public", "creator"]) + Room = namedtuple( + "Room", + ["room_id", "is_public", "creator"] + ) def __init__(self): self.tokens_to_users = {} self.paths_to_content = {} + self.members = {} - self.messages = {} self.rooms = {} - self.room_members = {} + + self.current_state = {} + self.events = [] def register(self, user_id, token, password_hash): if user_id in self.tokens_to_users.values(): @@ -162,100 +147,66 @@ class MemoryDataStore(object): if room_id in self.rooms: raise StoreError(409, "Conflicting room!") - room = MemoryDataStore.Room(room_id=room_id, is_public=is_public, - creator=room_creator_user_id) + room = MemoryDataStore.Room( + room_id=room_id, + is_public=is_public, + creator=room_creator_user_id + ) self.rooms[room_id] = room - #self.store_room_member(user_id=room_creator_user_id, room_id=room_id, - #membership=Membership.JOIN, - #content={"membership": Membership.JOIN}) - def get_message(self, user_id=None, room_id=None, msg_id=None): - try: - return self.messages[user_id + room_id + msg_id] - except: - return None + def get_room_member(self, user_id, room_id): + return self.members.get(room_id, {}).get(user_id) - def store_message(self, user_id=None, room_id=None, msg_id=None, - content=None): - msg = MemoryDataStore.Message(room_id=room_id, msg_id=msg_id, - user_id=user_id, content=content) - self.messages[user_id + room_id + msg_id] = msg - - def get_room_member(self, user_id=None, room_id=None): - try: - return self.members[user_id + room_id] - except: - return None - - def get_room_members(self, room_id=None, membership=None): - try: - return self.room_members[room_id] - except: - return None + def get_room_members(self, room_id, membership=None): + if membership: + return [ + v for k, v in self.members.get(room_id, {}).items() + if v.membership == membership + ] + else: + return self.members.get(room_id, {}).values() def get_rooms_for_user_where_membership_is(self, user_id, membership_list): - return [r for r in self.room_members - if user_id in self.room_members[r]] - - def store_room_member(self, user_id=None, sender=None, room_id=None, - membership=None, content=None): - member = MemoryDataStore.RoomMember(room_id=room_id, user_id=user_id, - sender=sender, membership=membership, content=json.dumps(content)) - self.members[user_id + room_id] = member - - # TODO should be latest state - if room_id not in self.room_members: - self.room_members[room_id] = [] - self.room_members[room_id].append(member) - - def get_room_data(self, room_id, etype, state_key=""): - path = "%s-%s-%s" % (room_id, etype, state_key) - try: - return self.paths_to_content[path] - except: - return None - - def store_room_data(self, room_id, etype, state_key="", content=None): - path = "%s-%s-%s" % (room_id, etype, state_key) - data = MemoryDataStore.PathData(path=path, room_id=room_id, - content=content) - self.paths_to_content[path] = data + return [ + r for r in self.members + if self.members[r].get(user_id).membership in membership_list + ] def get_room_events_stream(self, user_id=None, from_key=None, to_key=None, room_id=None, limit=0, with_feedback=False): return ([], from_key) # TODO - def to_events(self, data_store_list): - return data_store_list # TODO - - def get_room_events_max_id(self): - return 0 # TODO - def get_joined_hosts_for_room(self, room_id): return defer.succeed([]) def persist_event(self, event): - if event.type == MessageEvent.TYPE: - return self.store_message( - user_id=event.user_id, - room_id=event.room_id, - msg_id=event.msg_id, - content=json.dumps(event.content) - ) - elif event.type == RoomMemberEvent.TYPE: - return self.store_room_member( - user_id=event.target_user_id, - room_id=event.room_id, - content=event.content, - membership=event.content["membership"] - ) + if event.type == RoomMemberEvent.TYPE: + room_id = event.room_id + user = event.target_user_id + membership = event.membership + self.members.setdefault(room_id, {})[user] = event + + if hasattr(event, "state_key"): + key = (event.room_id, event.type, event.state_key) + self.current_state[key] = event + + self.events.append(event) + + def get_current_state(self, room_id, event_type=None, state_key=""): + if event_type: + key = (room_id, event_type, state_key) + return self.current_state.get(key) else: - raise NotImplementedError( - "Don't know how to persist type=%s" % event.type - ) + return [ + e for e in self.current_state + if e[0] == room_id + ] def set_presence_state(self, user_localpart, state): return defer.succeed({"state": 0}) def get_presence_list(self, user_localpart, accepted): return [] + + def get_room_events_max_id(self): + return 0 # TODO (erikj) From fc26275bb34206f48d70c7effcbc6f5d0bf86ecb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 15:50:41 +0100 Subject: [PATCH 031/112] Add two different columns for ordering the events table, one which can be used for pagination and one which can be as tokens for notifying clients. Also add a 'processed' field which is currently always set to True --- synapse/federation/handler.py | 4 ++-- synapse/storage/__init__.py | 18 ++++++++++++++---- synapse/storage/schema/im.sql | 13 ++++++++----- synapse/storage/stream.py | 17 +++++++++-------- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/synapse/federation/handler.py b/synapse/federation/handler.py index 580e591ac..68243d31d 100644 --- a/synapse/federation/handler.py +++ b/synapse/federation/handler.py @@ -63,7 +63,7 @@ class FederationEventHandler(object): Deferred: Resolved when it has successfully been queued for processing. """ - yield self._fill_out_prev_events(event) + yield self.fill_out_prev_events(event) pdu = self.pdu_codec.pdu_from_event(event) @@ -129,7 +129,7 @@ class FederationEventHandler(object): yield self.event_handler.on_receive(new_state_event) @defer.inlineCallbacks - def _fill_out_prev_events(self, event): + def fill_out_prev_events(self, event): if hasattr(event, "prev_events"): return diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 841ad8f13..9f78f3f9b 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -43,9 +43,10 @@ class DataStore(RoomMemberStore, RoomStore, def __init__(self, hs): super(DataStore, self).__init__(hs) self.event_factory = hs.get_event_factory() + self.hs = hs @defer.inlineCallbacks - def persist_event(self, event): + def persist_event(self, event, backfilled=False): if event.type == RoomMemberEvent.TYPE: yield self._store_room_member(event) elif event.type == FeedbackEvent.TYPE: @@ -57,7 +58,7 @@ class DataStore(RoomMemberStore, RoomStore, elif event.type == RoomTopicEvent.TYPE: yield self._store_room_topic(event) - ret = yield self._store_event(event) + ret = yield self._store_event(event, backfilled) defer.returnValue(ret) @defer.inlineCallbacks @@ -79,14 +80,23 @@ class DataStore(RoomMemberStore, RoomStore, defer.returnValue(event) @defer.inlineCallbacks - def _store_event(self, event): + def _store_event(self, event, backfilled): + # FIXME (erikj): This should be removed when we start amalgamating + # event and pdu storage. + yield self.hs.get_federation().fill_out_prev_events(event) + vals = { + "topological_ordering": event.depth, "event_id": event.event_id, "type": event.type, "room_id": event.room_id, "content": json.dumps(event.content), + "processed": True, } + if backfilled: + vals["token_ordering"] = "-1" + unrec = { k: v for k, v in event.get_full_dict().items() @@ -96,7 +106,7 @@ class DataStore(RoomMemberStore, RoomStore, yield self._simple_insert("events", vals) - if hasattr(event, "state_key"): + if not backfilled and hasattr(event, "state_key"): vals = { "event_id": event.event_id, "room_id": event.room_id, diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 2452890ea..b0240e39a 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -14,12 +14,15 @@ */ CREATE TABLE IF NOT EXISTS events( - ordering INTEGER PRIMARY KEY AUTOINCREMENT, + token_ordering INTEGER AUTOINCREMENT, + topological_ordering INTEGER NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, - room_id TEXT, - content TEXT, - unrecognized_keys TEXT + room_id TEXT NOT NULL, + content TEXT NOT NULL, + unrecognized_keys TEXT, + processed BOOL NOT NULL, + CONSTRAINT ev_uniq UNIQUE (event_id) ); CREATE TABLE IF NOT EXISTS state_events( @@ -35,7 +38,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( room_id TEXT NOT NULL, type TEXT NOT NULL, state_key TEXT NOT NULL, - CONSTRAINT uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE + CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE ); CREATE TABLE IF NOT EXISTS room_memberships( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index cf4b1682b..f7968f576 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -73,13 +73,14 @@ class StreamStore(SQLBaseStore): # Constraints and ordering depend on direction. if from_key < to_key: sql += ( - "AND e.ordering > ? AND e.ordering < ? " - "ORDER BY ordering ASC LIMIT %(limit)d " + "AND e.token_ordering > ? AND e.token_ordering < ? " + "ORDER BY token_ordering, rowid ASC LIMIT %(limit)d " ) % {"limit": limit} else: sql += ( - "AND e.ordering < ? AND e.ordering > ? " - "ORDER BY ordering DESC LIMIT %(limit)d " + "AND e.token_ordering < ? " + "AND e.token_ordering > ? " + "ORDER BY e.token_ordering, rowid DESC LIMIT %(limit)d " ) % {"limit": int(limit)} rows = yield self._execute_and_decode( @@ -91,9 +92,9 @@ class StreamStore(SQLBaseStore): if rows: if from_key < to_key: - key = max([r["ordering"] for r in rows]) + key = max([r["token_ordering"] for r in rows]) else: - key = min([r["ordering"] for r in rows]) + key = min([r["token_ordering"] for r in rows]) else: key = to_key @@ -105,7 +106,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events WHERE room_id = ? " - "ORDER BY ordering DESC LIMIT ? " + "ORDER BY token_ordering, rowid DESC LIMIT ? " ) rows = yield self._execute_and_decode( @@ -120,7 +121,7 @@ class StreamStore(SQLBaseStore): @defer.inlineCallbacks def get_room_events_max_id(self): res = yield self._execute_and_decode( - "SELECT MAX(ordering) as m FROM events" + "SELECT MAX(token_ordering) as m FROM events" ) if not res: From 709a92cee89709811c51cac7d8c66922093be673 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 16:00:46 +0100 Subject: [PATCH 032/112] SQL doesn't allow AUTOINCREMENT on non PRIMARY KEY columns. --- synapse/storage/__init__.py | 21 +++++++++++++++++++-- synapse/storage/schema/im.sql | 2 +- tests/rest/test_presence.py | 2 ++ tests/rest/test_profile.py | 1 + 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9f78f3f9b..b846081d4 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -45,6 +45,9 @@ class DataStore(RoomMemberStore, RoomStore, self.event_factory = hs.get_event_factory() self.hs = hs + self.min_token_deferred = self._get_min_token() + self.min_token = None + @defer.inlineCallbacks def persist_event(self, event, backfilled=False): if event.type == RoomMemberEvent.TYPE: @@ -82,7 +85,7 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks def _store_event(self, event, backfilled): # FIXME (erikj): This should be removed when we start amalgamating - # event and pdu storage. + # event and pdu storage yield self.hs.get_federation().fill_out_prev_events(event) vals = { @@ -95,7 +98,10 @@ class DataStore(RoomMemberStore, RoomStore, } if backfilled: - vals["token_ordering"] = "-1" + if not self.min_token_deferred.called: + yield self.min_token_deferred + self.min_token -= 1 + vals["token_ordering"] = self.min_token unrec = { k: v @@ -151,6 +157,17 @@ class DataStore(RoomMemberStore, RoomStore, defer.returnValue([self._parse_event_from_row(r) for r in results]) + @defer.inlineCallbacks + def _get_min_token(self): + row = yield self._execute( + None, + "SELECT MIN(token_ordering) FROM events" + ) + + self.min_token = rows[0][0] if rows and rows[0] else 0 + + defer.returnValue(self.min_token) + def schema_path(schema): """ Get a filesystem path for the named database schema diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index b0240e39a..0fb3dbee5 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - token_ordering INTEGER AUTOINCREMENT, + token_ordering INTEGER PRIMARY KEY AUTOINCREMENT, topological_ordering INTEGER NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, diff --git a/tests/rest/test_presence.py b/tests/rest/test_presence.py index 91d4d1ff6..99287823e 100644 --- a/tests/rest/test_presence.py +++ b/tests/rest/test_presence.py @@ -51,6 +51,7 @@ class PresenceStateTestCase(unittest.TestCase): hs = HomeServer("test", db_pool=None, http_client=None, + datastore=None, resource_for_client=self.mock_server, resource_for_federation=self.mock_server, ) @@ -109,6 +110,7 @@ class PresenceListTestCase(unittest.TestCase): hs = HomeServer("test", db_pool=None, http_client=None, + datastore=None, resource_for_client=self.mock_server, resource_for_federation=self.mock_server ) diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py index ff1e92805..ee47daf4c 100644 --- a/tests/rest/test_profile.py +++ b/tests/rest/test_profile.py @@ -46,6 +46,7 @@ class ProfileTestCase(unittest.TestCase): resource_for_client=self.mock_server, federation=Mock(), replication_layer=Mock(), + datastore=None, ) def _get_user_by_token(token=None): From 4eb8f84aa8fc735e228f66d11c355625bb14c1cf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 16:20:21 +0100 Subject: [PATCH 033/112] Make snapshot_all_rooms return results in the correct form, including start and end tokens. --- synapse/handlers/room.py | 9 +++++++-- synapse/storage/stream.py | 25 +++++++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index cdc98d2b0..9e10235fa 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -269,11 +269,16 @@ class MessageHandler(BaseHandler): if event.membership != Membership.JOIN: continue try: - messages = yield self.store.get_recent_events_for_room( + messages, token = yield self.store.get_recent_events_for_room( event.room_id, limit=50, ) - d["messages"] = [m.get_dict() for m in messages] + + d["messages"] = { + "chunk": [m.get_dict() for m in messages], + "start": token[0], + "end": token[1], + } except: pass diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index f7968f576..6728a4b5e 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -104,19 +104,36 @@ class StreamStore(SQLBaseStore): def get_recent_events_for_room(self, room_id, limit, with_feedback=False): # TODO (erikj): Handle compressed feedback + end_token = yield self.get_room_events_max_id() + sql = ( - "SELECT * FROM events WHERE room_id = ? " - "ORDER BY token_ordering, rowid DESC LIMIT ? " + "SELECT * FROM events WHERE " + "WHERE room_id = ? AND token_ordering <= ? " + "ORDER BY topological_ordering, rowid DESC LIMIT ? " ) rows = yield self._execute_and_decode( sql, - room_id, limit + room_id, end_token, limit ) rows.reverse() # As we selected with reverse ordering - defer.returnValue([self._parse_event_from_row(r) for r in rows]) + if rows: + topo = rows[0]["topological_ordering"] + row_id = rows[0]["rowid"] + start_token = "p%s-%s" % (topo, row_id) + + token = (start_token, end_token) + else: + token = ("START", end_token) + + defer.returnValue( + ( + [self._parse_event_from_row(r) for r in rows], + token + ) + ) @defer.inlineCallbacks def get_room_events_max_id(self): From 1422a22970e4ab3f7a37fe672af2f1d1de901c10 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Aug 2014 16:25:18 +0100 Subject: [PATCH 034/112] Fix typos in SQL and where we still had rowid's (which no longer exist) --- synapse/handlers/room.py | 2 +- synapse/storage/stream.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 9e10235fa..e5dd2e245 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -280,7 +280,7 @@ class MessageHandler(BaseHandler): "end": token[1], } except: - pass + logger.exception("Failed to get snapshot") logger.debug("snapshot_all_rooms returning: %s", ret) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 6728a4b5e..c03c983e1 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -74,13 +74,13 @@ class StreamStore(SQLBaseStore): if from_key < to_key: sql += ( "AND e.token_ordering > ? AND e.token_ordering < ? " - "ORDER BY token_ordering, rowid ASC LIMIT %(limit)d " + "ORDER BY token_ordering ASC LIMIT %(limit)d " ) % {"limit": limit} else: sql += ( "AND e.token_ordering < ? " "AND e.token_ordering > ? " - "ORDER BY e.token_ordering, rowid DESC LIMIT %(limit)d " + "ORDER BY e.token_ordering DESC LIMIT %(limit)d " ) % {"limit": int(limit)} rows = yield self._execute_and_decode( @@ -107,9 +107,9 @@ class StreamStore(SQLBaseStore): end_token = yield self.get_room_events_max_id() sql = ( - "SELECT * FROM events WHERE " + "SELECT * FROM events " "WHERE room_id = ? AND token_ordering <= ? " - "ORDER BY topological_ordering, rowid DESC LIMIT ? " + "ORDER BY topological_ordering, token_ordering DESC LIMIT ? " ) rows = yield self._execute_and_decode( @@ -121,8 +121,8 @@ class StreamStore(SQLBaseStore): if rows: topo = rows[0]["topological_ordering"] - row_id = rows[0]["rowid"] - start_token = "p%s-%s" % (topo, row_id) + toke = rows[0]["token_ordering"] + start_token = "p%s-%s" % (topo, toke) token = (start_token, end_token) else: From 598a1d8ff953c70f9f54564225d693a1bcf42144 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 14:19:48 +0100 Subject: [PATCH 035/112] Change the way pagination works to support out of order events. --- synapse/api/streams/__init__.py | 19 +- synapse/api/streams/event.py | 90 +++++---- synapse/handlers/presence.py | 2 +- synapse/handlers/room.py | 3 +- synapse/storage/schema/im.sql | 2 +- synapse/storage/stream.py | 186 +++++++++++++++--- .../components/matrix/event-stream-service.js | 1 - webclient/components/matrix/matrix-service.js | 4 +- 8 files changed, 226 insertions(+), 81 deletions(-) diff --git a/synapse/api/streams/__init__.py b/synapse/api/streams/__init__.py index 989e63f9e..44f4cc607 100644 --- a/synapse/api/streams/__init__.py +++ b/synapse/api/streams/__init__.py @@ -20,23 +20,23 @@ class PaginationConfig(object): """A configuration object which stores pagination parameters.""" - def __init__(self, from_tok=None, to_tok=None, limit=0): + def __init__(self, from_tok=None, to_tok=None, direction='f', limit=0): self.from_tok = from_tok self.to_tok = to_tok + self.direction = direction self.limit = limit @classmethod def from_request(cls, request, raise_invalid_params=True): params = { - "from_tok": PaginationStream.TOK_START, - "to_tok": PaginationStream.TOK_END, - "limit": 0 + "direction": 'f', } query_param_mappings = [ # 3-tuple of qp_key, attribute, rules ("from", "from_tok", lambda x: type(x) == str), ("to", "to_tok", lambda x: type(x) == str), - ("limit", "limit", lambda x: x.isdigit()) + ("limit", "limit", lambda x: x.isdigit()), + ("dir", "direction", lambda x: x == 'f' or x == 'b'), ] for qp, attr, is_valid in query_param_mappings: @@ -48,12 +48,17 @@ class PaginationConfig(object): return PaginationConfig(**params) + def __str__(self): + return ( + "" + ) % (self.from_tok, self.to_tok, self.direction, self.limit) + class PaginationStream(object): """ An interface for streaming data as chunks. """ - TOK_START = "START" TOK_END = "END" def get_chunk(self, config=None): @@ -76,7 +81,7 @@ class StreamData(object): self.hs = hs self.store = hs.get_datastore() - def get_rows(self, user_id, from_pkey, to_pkey, limit): + def get_rows(self, user_id, from_pkey, to_pkey, limit, direction): """ Get event stream data between the specified pkeys. Args: diff --git a/synapse/api/streams/event.py b/synapse/api/streams/event.py index 414b05be3..a5c8b2b31 100644 --- a/synapse/api/streams/event.py +++ b/synapse/api/streams/event.py @@ -38,8 +38,8 @@ class EventsStreamData(StreamData): self.with_feedback = feedback @defer.inlineCallbacks - def get_rows(self, user_id, from_key, to_key, limit): - data, latest_ver = yield self.store.get_room_events_stream( + def get_rows(self, user_id, from_key, to_key, limit, direction): + data, latest_ver = yield self.store.get_room_events( user_id=user_id, from_key=from_key, to_key=to_key, @@ -70,6 +70,15 @@ class EventStream(PaginationStream): pagination_config.from_tok) pagination_config.to_tok = yield self.fix_token( pagination_config.to_tok) + + if ( + not pagination_config.to_tok + and pagination_config.direction == 'f' + ): + pagination_config.to_tok = yield self.get_current_max_token() + + logger.debug("pagination_config: %s", pagination_config) + defer.returnValue(pagination_config) @defer.inlineCallbacks @@ -81,39 +90,42 @@ class EventStream(PaginationStream): Returns: The fixed-up token, which may == token. """ - # replace TOK_START and TOK_END with 0_0_0 or -1_-1_-1 depending. - replacements = [ - (PaginationStream.TOK_START, "0"), - (PaginationStream.TOK_END, "-1") - ] - for magic_token, key in replacements: - if magic_token == token: - token = EventStream.SEPARATOR.join( - [key] * len(self.stream_data) - ) + if token == PaginationStream.TOK_END: + new_token = yield self.get_current_max_token() - # replace -1 values with an actual pkey - token_segments = self._split_token(token) - for i, tok in enumerate(token_segments): - if tok == -1: - # add 1 to the max token because results are EXCLUSIVE from the - # latest version. - token_segments[i] = 1 + (yield self.stream_data[i].max_token()) - defer.returnValue(EventStream.SEPARATOR.join( - str(x) for x in token_segments - )) + logger.debug("fix_token: From %s to %s", token, new_token) + + token = new_token + + defer.returnValue(token) @defer.inlineCallbacks - def get_chunk(self, config=None): + def get_current_max_token(self): + new_token_parts = [] + for s in self.stream_data: + mx = yield s.max_token() + new_token_parts.append(str(mx)) + + new_token = EventStream.SEPARATOR.join(new_token_parts) + + logger.debug("get_current_max_token: %s", new_token) + + defer.returnValue(new_token) + + @defer.inlineCallbacks + def get_chunk(self, config): # no support for limit on >1 streams, makes no sense. if config.limit and len(self.stream_data) > 1: raise EventStreamError( 400, "Limit not supported on multiplexed streams." ) - (chunk_data, next_tok) = yield self._get_chunk_data(config.from_tok, - config.to_tok, - config.limit) + chunk_data, next_tok = yield self._get_chunk_data( + config.from_tok, + config.to_tok, + config.limit, + config.direction, + ) defer.returnValue({ "chunk": chunk_data, @@ -122,7 +134,7 @@ class EventStream(PaginationStream): }) @defer.inlineCallbacks - def _get_chunk_data(self, from_tok, to_tok, limit): + def _get_chunk_data(self, from_tok, to_tok, limit, direction): """ Get event data between the two tokens. Tokens are SEPARATOR separated values representing pkey values of @@ -140,11 +152,12 @@ class EventStream(PaginationStream): EventStreamError if something went wrong. """ # sanity check - if (from_tok.count(EventStream.SEPARATOR) != - to_tok.count(EventStream.SEPARATOR) or - (from_tok.count(EventStream.SEPARATOR) + 1) != - len(self.stream_data)): - raise EventStreamError(400, "Token lengths don't match.") + if to_tok is not None: + if (from_tok.count(EventStream.SEPARATOR) != + to_tok.count(EventStream.SEPARATOR) or + (from_tok.count(EventStream.SEPARATOR) + 1) != + len(self.stream_data)): + raise EventStreamError(400, "Token lengths don't match.") chunk = [] next_ver = [] @@ -158,7 +171,7 @@ class EventStream(PaginationStream): continue (event_chunk, max_pkey) = yield self.stream_data[i].get_rows( - self.user_id, from_pkey, to_pkey, limit + self.user_id, from_pkey, to_pkey, limit, direction, ) chunk.extend([ @@ -177,9 +190,8 @@ class EventStream(PaginationStream): Returns: A list of ints. """ - segments = token.split(EventStream.SEPARATOR) - try: - int_segments = [int(x) for x in segments] - except ValueError: - raise EventStreamError(400, "Bad token: %s" % token) - return int_segments + if token: + segments = token.split(EventStream.SEPARATOR) + else: + segments = [None] * len(self.stream_data) + return segments diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index e8cb83edd..f140dc527 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -684,7 +684,7 @@ class PresenceStreamData(StreamData): super(PresenceStreamData, self).__init__(hs) self.presence = hs.get_handlers().presence_handler - def get_rows(self, user_id, from_key, to_key, limit): + def get_rows(self, user_id, from_key, to_key, limit, direction): cachemap = self.presence._user_cachemap # TODO(paul): limit, and filter by visibility diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index e5dd2e245..40867ae2e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -466,7 +466,7 @@ class RoomMemberHandler(BaseHandler): for entry in member_list ] chunk_data = { - "start": "START", + "start": "START", # FIXME (erikj): START is no longer a valid value "end": "END", "chunk": event_list } @@ -811,4 +811,5 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def get_public_room_list(self): chunk = yield self.store.get_rooms(is_public=True) + # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 0fb3dbee5..ea04261ff 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - token_ordering INTEGER PRIMARY KEY AUTOINCREMENT, + stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, topological_ordering INTEGER NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index c03c983e1..87fc97881 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -16,8 +16,9 @@ from twisted.internet import defer from ._base import SQLBaseStore - +from synapse.api.errors import SynapseError from synapse.api.constants import Membership +from synapse.util.logutils import log_function import json import logging @@ -29,9 +30,96 @@ logger = logging.getLogger(__name__) MAX_STREAM_SIZE = 1000 +_STREAM_TOKEN = "stream" +_TOPOLOGICAL_TOKEN = "topological" + + +def _parse_stream_token(string): + try: + if string[0] != 's': + raise + return int(string[1:]) + except: + logger.debug("Not stream token: %s", string) + raise SynapseError(400, "Invalid token") + + +def _parse_topological_token(string): + try: + if string[0] != 't': + raise + parts = string[1:].split('-', 1) + return (int(parts[0]), int(parts[1])) + except: + logger.debug("Not topological token: %s", string) + raise SynapseError(400, "Invalid token") + + +def is_stream_token(string): + try: + _parse_stream_token(string) + return True + except: + return False + + +def is_topological_token(string): + try: + _parse_topological_token(string) + return True + except: + return False + + +def _get_token_bound(token, comparison): + try: + s = _parse_stream_token(token) + return "%s %s %d" % ("stream_ordering", comparison, s) + except: + pass + + try: + top, stream = _parse_topological_token(token) + return "%s %s %d AND %s %s %d" % ( + "topological_ordering", comparison, top, + "stream_ordering", comparison, stream, + ) + except: + pass + + raise SynapseError(400, "Invalid token") + + class StreamStore(SQLBaseStore): + @log_function + def get_room_events(self, user_id, from_key, to_key, room_id, limit=0, + direction='f', with_feedback=False): + is_events = ( + direction == 'f' + and is_stream_token(from_key) + and to_key and is_stream_token(to_key) + ) + + if is_events: + return self.get_room_events_stream( + user_id=user_id, + from_key=from_key, + to_key=to_key, + room_id=room_id, + limit=limit, + with_feedback=with_feedback, + ) + else: + return self.paginate_room_events( + from_key=from_key, + to_key=to_key, + room_id=room_id, + limit=limit, + with_feedback=with_feedback, + ) @defer.inlineCallbacks + @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): # TODO (erikj): Handle compressed feedback @@ -54,8 +142,8 @@ class StreamStore(SQLBaseStore): limit = MAX_STREAM_SIZE # From and to keys should be integers from ordering. - from_key = int(from_key) - to_key = int(to_key) + from_id = _parse_stream_token(from_key) + to_id = _parse_stream_token(to_key) if from_key == to_key: defer.returnValue(([], to_key)) @@ -65,41 +153,78 @@ class StreamStore(SQLBaseStore): "SELECT * FROM events as e WHERE " "((room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " + "AND e.stream_ordering > ? AND e.stream_ordering < ? " + "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "current": current_room_membership_sql, "invites": invites_sql, + "limit": limit } - # Constraints and ordering depend on direction. - if from_key < to_key: - sql += ( - "AND e.token_ordering > ? AND e.token_ordering < ? " - "ORDER BY token_ordering ASC LIMIT %(limit)d " - ) % {"limit": limit} - else: - sql += ( - "AND e.token_ordering < ? " - "AND e.token_ordering > ? " - "ORDER BY e.token_ordering DESC LIMIT %(limit)d " - ) % {"limit": int(limit)} - rows = yield self._execute_and_decode( sql, - user_id, user_id, Membership.INVITE, from_key, to_key + user_id, user_id, Membership.INVITE, from_id, to_id ) ret = [self._parse_event_from_row(r) for r in rows] if rows: - if from_key < to_key: - key = max([r["token_ordering"] for r in rows]) - else: - key = min([r["token_ordering"] for r in rows]) + key = "s%d" % max([r["stream_ordering"] for r in rows]) else: + # Assume we didn't get anything because there was nothing to get. key = to_key defer.returnValue((ret, key)) + @defer.inlineCallbacks + @log_function + def paginate_room_events(self, room_id, from_key, to_key=None, + direction='b', limit=-1, + with_feedback=False): + # TODO (erikj): Handle compressed feedback + + from_comp = '<' if direction =='b' else '>' + to_comp = '>' if direction =='b' else '<' + order = "DESC" if direction == 'b' else "ASC" + + args = [room_id] + + bounds = _get_token_bound(from_key, from_comp) + if to_key: + bounds = "%s AND %s" % (bounds, _get_token_bound(to_key, to_comp)) + + if int(limit) > 0: + args.append(int(limit)) + limit_str = " LIMIT ?" + else: + limit_str = "" + + sql = ( + "SELECT * FROM events " + "WHERE room_id = ? AND %(bounds)s " + "ORDER BY topological_ordering %(order)s, stream_ordering %(order)s %(limit)s " + ) % {"bounds": bounds, "order": order, "limit": limit_str} + + rows = yield self._execute_and_decode( + sql, + *args + ) + + if rows: + topo = rows[-1]["topological_ordering"] + toke = rows[-1]["stream_ordering"] + next_token = "t%s-%s" % (topo, toke) + else: + # TODO (erikj): We should work out what to do here instead. + next_token = to_key if to_key else from_key + + defer.returnValue( + ( + [self._parse_event_from_row(r) for r in rows], + next_token + ) + ) + @defer.inlineCallbacks def get_recent_events_for_room(self, room_id, limit, with_feedback=False): # TODO (erikj): Handle compressed feedback @@ -108,8 +233,8 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events " - "WHERE room_id = ? AND token_ordering <= ? " - "ORDER BY topological_ordering, token_ordering DESC LIMIT ? " + "WHERE room_id = ? AND stream_ordering <= ? " + "ORDER BY topological_ordering, stream_ordering DESC LIMIT ? " ) rows = yield self._execute_and_decode( @@ -121,12 +246,12 @@ class StreamStore(SQLBaseStore): if rows: topo = rows[0]["topological_ordering"] - toke = rows[0]["token_ordering"] + toke = rows[0]["stream_ordering"] start_token = "p%s-%s" % (topo, toke) token = (start_token, end_token) else: - token = ("START", end_token) + token = (end_token, end_token) defer.returnValue( ( @@ -138,11 +263,14 @@ class StreamStore(SQLBaseStore): @defer.inlineCallbacks def get_room_events_max_id(self): res = yield self._execute_and_decode( - "SELECT MAX(token_ordering) as m FROM events" + "SELECT MAX(stream_ordering) as m FROM events" ) - if not res: - defer.returnValue(0) + logger.debug("get_room_events_max_id: %s", res) + + if not res or not res[0] or not res[0]["m"]: + defer.returnValue("s1") return - defer.returnValue(res[0]["m"]) + key = res[0]["m"] + 1 + defer.returnValue("s%d" % (key,)) diff --git a/webclient/components/matrix/event-stream-service.js b/webclient/components/matrix/event-stream-service.js index c94cf0fe7..a446fad5d 100644 --- a/webclient/components/matrix/event-stream-service.js +++ b/webclient/components/matrix/event-stream-service.js @@ -25,7 +25,6 @@ the eventHandlerService. angular.module('eventStreamService', []) .factory('eventStreamService', ['$q', '$timeout', 'matrixService', 'eventHandlerService', function($q, $timeout, matrixService, eventHandlerService) { var END = "END"; - var START = "START"; var TIMEOUT_MS = 30000; var ERR_TIMEOUT_MS = 5000; diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index c52c94c31..3cd0aa674 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -230,8 +230,8 @@ angular.module('matrixService', []) path = path.replace("$room_id", room_id); var params = { from: from_token, - to: "START", - limit: limit + limit: limit, + dir: 'b' }; return doRequest("GET", path, params); }, From 75b6d982a01a431a89d2ab76d91a09159630d059 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 14:20:03 +0100 Subject: [PATCH 036/112] Add a 'backfill room' button --- synapse/federation/handler.py | 22 +++++++++++++++------- synapse/federation/replication.py | 9 ++++++--- synapse/handlers/federation.py | 21 ++++++++++++++++++--- synapse/rest/room.py | 16 ++++++++++++++++ synapse/storage/__init__.py | 22 ++++++++++++++++++---- synapse/storage/pdu.py | 16 ++++++++-------- 6 files changed, 81 insertions(+), 25 deletions(-) diff --git a/synapse/federation/handler.py b/synapse/federation/handler.py index 68243d31d..984c1558e 100644 --- a/synapse/federation/handler.py +++ b/synapse/federation/handler.py @@ -74,10 +74,18 @@ class FederationEventHandler(object): @log_function @defer.inlineCallbacks - def backfill(self, room_id, limit): - # TODO: Work out which destinations to ask for backfill - # self.replication_layer.backfill(dest, room_id, limit) - pass + def backfill(self, dest, room_id, limit): + pdus = yield self.replication_layer.backfill(dest, room_id, limit) + + if not pdus: + defer.returnValue([]) + + events = [ + self.pdu_codec.event_from_pdu(pdu) + for pdu in pdus + ] + + defer.returnValue(events) @log_function def get_state_for_room(self, destination, room_id): @@ -87,7 +95,7 @@ class FederationEventHandler(object): @log_function @defer.inlineCallbacks - def on_receive_pdu(self, pdu): + def on_receive_pdu(self, pdu, backfilled): """ Called by the ReplicationLayer when we have a new pdu. We need to do auth checks and put it throught the StateHandler. """ @@ -95,7 +103,7 @@ class FederationEventHandler(object): try: with (yield self.lock_manager.lock(pdu.context)): - if event.is_state: + if event.is_state and not backfilled: is_new_state = yield self.state_handler.handle_new_state( pdu ) @@ -104,7 +112,7 @@ class FederationEventHandler(object): else: is_new_state = False - yield self.event_handler.on_receive(event, is_new_state) + yield self.event_handler.on_receive(event, is_new_state, backfilled) except AuthError: # TODO: Implement something in federation that allows us to diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index bc9df2f21..3e5f1a410 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -208,7 +208,7 @@ class ReplicationLayer(object): pdus = [Pdu(outlier=False, **p) for p in transaction.pdus] for pdu in pdus: - yield self._handle_new_pdu(pdu) + yield self._handle_new_pdu(pdu, backfilled=True) defer.returnValue(pdus) @@ -415,7 +415,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def _handle_new_pdu(self, pdu): + def _handle_new_pdu(self, pdu, backfilled=False): # We reprocess pdus when we have seen them only as outliers existing = yield self._get_persisted_pdu(pdu.pdu_id, pdu.origin) @@ -451,7 +451,10 @@ class ReplicationLayer(object): # Persist the Pdu, but don't mark it as processed yet. yield self.pdu_actions.persist_received(pdu) - ret = yield self.handler.on_receive_pdu(pdu) + if not backfilled: + ret = yield self.handler.on_receive_pdu(pdu, backfilled=backfilled) + else: + ret = None yield self.pdu_actions.mark_as_processed(pdu) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7026df90a..ef9ed274d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -35,7 +35,7 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def on_receive(self, event, is_new_state): + def on_receive(self, event, is_new_state, backfilled): if hasattr(event, "state_key") and not is_new_state: logger.debug("Ignoring old state.") return @@ -70,6 +70,21 @@ class FederationHandler(BaseHandler): else: with (yield self.room_lock.lock(event.room_id)): - store_id = yield self.store.persist_event(event) + store_id = yield self.store.persist_event(event, backfilled) - yield self.notifier.on_new_room_event(event, store_id) + if not backfilled: + yield self.notifier.on_new_room_event(event, store_id) + + + @log_function + @defer.inlineCallbacks + def backfill(self, dest, room_id, limit): + events = yield self.hs.get_federation().backfill(dest, room_id, limit) + + for event in events: + try: + yield self.store.persist_event(event, backfilled=True) + except: + logger.debug("Failed to persiste event: %s", event) + + defer.returnValue(events) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index dfb2aabe7..89ea9f0d2 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -383,6 +383,21 @@ class RoomMessageListRestServlet(RestServlet): defer.returnValue((200, msgs)) +class RoomTriggerBackfill(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/backfill$") + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + remote_server = urllib.unquote(request.args["remote"][0]) + room_id = urllib.unquote(room_id) + limit = int(request.args["limit"][0]) + + handler = self.handlers.federation_handler + events = yield handler.backfill(remote_server, room_id, limit) + + res = [event.get_dict() for event in events] + defer.returnValue((200, res)) + def _parse_json(request): try: content = json.loads(request.content.read()) @@ -403,3 +418,4 @@ def register_servlets(hs, http_server): RoomMemberListRestServlet(hs).register(http_server) RoomMessageListRestServlet(hs).register(http_server) JoinRoomAliasServlet(hs).register(http_server) + RoomTriggerBackfill(hs).register(http_server) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b846081d4..2243a710d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -20,6 +20,8 @@ from synapse.api.events.room import ( RoomConfigEvent, RoomNameEvent, ) +from synapse.util.logutils import log_function + from .directory import DirectoryStore from .feedback import FeedbackStore from .presence import PresenceStore @@ -32,9 +34,13 @@ from .pdu import StatePduStore, PduStore from .transactions import TransactionStore import json +import logging import os +logger = logging.getLogger(__name__) + + class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, PduStore, StatePduStore, TransactionStore, @@ -49,6 +55,7 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token = None @defer.inlineCallbacks + @log_function def persist_event(self, event, backfilled=False): if event.type == RoomMemberEvent.TYPE: yield self._store_room_member(event) @@ -83,6 +90,7 @@ class DataStore(RoomMemberStore, RoomStore, defer.returnValue(event) @defer.inlineCallbacks + @log_function def _store_event(self, event, backfilled): # FIXME (erikj): This should be removed when we start amalgamating # event and pdu storage @@ -101,7 +109,7 @@ class DataStore(RoomMemberStore, RoomStore, if not self.min_token_deferred.called: yield self.min_token_deferred self.min_token -= 1 - vals["token_ordering"] = self.min_token + vals["stream_ordering"] = self.min_token unrec = { k: v @@ -110,7 +118,11 @@ class DataStore(RoomMemberStore, RoomStore, } vals["unrecognized_keys"] = json.dumps(unrec) - yield self._simple_insert("events", vals) + try: + yield self._simple_insert("events", vals) + except: + logger.exception("Failed to persist, probably duplicate") + return if not backfilled and hasattr(event, "state_key"): vals = { @@ -161,10 +173,12 @@ class DataStore(RoomMemberStore, RoomStore, def _get_min_token(self): row = yield self._execute( None, - "SELECT MIN(token_ordering) FROM events" + "SELECT MIN(stream_ordering) FROM events" ) - self.min_token = rows[0][0] if rows and rows[0] else 0 + self.min_token = min(row[0][0], -1) if row and row[0] else -1 + + logger.debug("min_token is: %s", self.min_token) defer.returnValue(self.min_token) diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index a24ce7ab7..7655f43ed 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + from ._base import SQLBaseStore, Table, JoinHelper from synapse.util.logutils import log_function @@ -319,6 +321,7 @@ class PduStore(SQLBaseStore): return [(row[0], row[1], row[2]) for row in results] + @defer.inlineCallbacks def get_oldest_pdus_in_context(self, context): """Get a list of Pdus that we haven't backfilled beyond yet (and haven't seen). This list is used when we want to backfill backwards and is the @@ -331,17 +334,14 @@ class PduStore(SQLBaseStore): Returns: list: A list of PduIdTuple. """ - return self._db_pool.runInteraction( - self._get_oldest_pdus_in_context, context - ) - - def _get_oldest_pdus_in_context(self, txn, context): - txn.execute( + results = yield self._execute( + None, "SELECT pdu_id, origin FROM %(back)s WHERE context = ?" % {"back": PduBackwardExtremitiesTable.table_name, }, - (context,) + context ) - return [PduIdTuple(i, o) for i, o in txn.fetchall()] + + defer.returnValue([PduIdTuple(i, o) for i, o in results]) def is_pdu_new(self, pdu_id, origin, context, depth): """For a given Pdu, try and figure out if it's 'new', i.e., if it's From 234128586bd210a496bea7aef7045cd5905b8b5c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 14:30:28 +0100 Subject: [PATCH 037/112] Print out stacktrace when we failed to persist event. --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ef9ed274d..0430a8307 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -85,6 +85,6 @@ class FederationHandler(BaseHandler): try: yield self.store.persist_event(event, backfilled=True) except: - logger.debug("Failed to persiste event: %s", event) + logger.exception("Failed to persist event: %s", event) defer.returnValue(events) From 840771190fc11e51ee0810749a99d87186ccb78b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 14:32:47 +0100 Subject: [PATCH 038/112] Fix bug where we sometimes set min_token to None. --- synapse/storage/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2243a710d..470b7b766 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -176,7 +176,8 @@ class DataStore(RoomMemberStore, RoomStore, "SELECT MIN(stream_ordering) FROM events" ) - self.min_token = min(row[0][0], -1) if row and row[0] else -1 + self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 + self.min_token = min(self.min_token, -1) logger.debug("min_token is: %s", self.min_token) From 22dd0b37c4215a001e801ebe4bdfa36153ea1324 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 15:02:47 +0100 Subject: [PATCH 039/112] Fix typo in merge conflict --- tests/rest/test_presence.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/rest/test_presence.py b/tests/rest/test_presence.py index 80b5972b4..0ba72addf 100644 --- a/tests/rest/test_presence.py +++ b/tests/rest/test_presence.py @@ -57,7 +57,6 @@ class PresenceStateTestCase(unittest.TestCase): "set_presence_state", ]), http_client=None, - datastore=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource, ) @@ -135,7 +134,6 @@ class PresenceListTestCase(unittest.TestCase): "get_presence_list", ]), http_client=None, - datastore=None, resource_for_client=self.mock_resource, resource_for_federation=self.mock_resource ) From 5c00614aaba881c354cb9eecf024aa3a84838c4f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 15:51:10 +0100 Subject: [PATCH 040/112] PresenceStreamData was expecting *_key to be ints --- synapse/handlers/presence.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 60684f17d..319e3c7c8 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -689,6 +689,9 @@ class PresenceStreamData(StreamData): self.presence = hs.get_handlers().presence_handler def get_rows(self, user_id, from_key, to_key, limit, direction): + from_key = int(from_key) + to_key = int(to_key) + cachemap = self.presence._user_cachemap # TODO(paul): limit, and filter by visibility From 7c60905ee7bed94e600018146fe309a295673af8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 15:52:10 +0100 Subject: [PATCH 041/112] Default from param to 'END' --- synapse/api/streams/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/api/streams/__init__.py b/synapse/api/streams/__init__.py index 44f4cc607..d831eafba 100644 --- a/synapse/api/streams/__init__.py +++ b/synapse/api/streams/__init__.py @@ -29,6 +29,7 @@ class PaginationConfig(object): @classmethod def from_request(cls, request, raise_invalid_params=True): params = { + "from_tok": "END", "direction": 'f', } From 41333452e53f72a0f1c9f63fd8c46117d70eea3a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 15:52:20 +0100 Subject: [PATCH 042/112] Update tests --- tests/handlers/test_federation.py | 8 ++++---- tests/rest/test_presence.py | 9 +-------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index ab9c24257..cb45169dd 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -70,9 +70,9 @@ class FederationTestCase(unittest.TestCase): store_id = "ASD" self.datastore.persist_event.return_value = defer.succeed(store_id) - yield self.handlers.federation_handler.on_receive(event, False) + yield self.handlers.federation_handler.on_receive(event, False, False) - self.datastore.persist_event.assert_called_once_with(event) + self.datastore.persist_event.assert_called_once_with(event, False) self.notifier.on_new_room_event.assert_called_once_with( event, store_id) @@ -89,7 +89,7 @@ class FederationTestCase(unittest.TestCase): content={}, ) - yield self.handlers.federation_handler.on_receive(event, False) + yield self.handlers.federation_handler.on_receive(event, False, False) mem_handler = self.handlers.room_member_handler self.assertEquals(1, mem_handler.change_membership.call_count) @@ -115,7 +115,7 @@ class FederationTestCase(unittest.TestCase): content={}, ) - yield self.handlers.federation_handler.on_receive(event, False) + yield self.handlers.federation_handler.on_receive(event, False, False) mem_handler = self.handlers.room_member_handler self.assertEquals(0, mem_handler.change_membership.call_count) diff --git a/tests/rest/test_presence.py b/tests/rest/test_presence.py index 0ba72addf..8ac246b4d 100644 --- a/tests/rest/test_presence.py +++ b/tests/rest/test_presence.py @@ -287,14 +287,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): # all be ours # I'll already get my own presence state change - self.assertEquals({"start": "0", "end": "1", "chunk": [ - {"type": "m.presence", - "content": { - "user_id": "@apple:test", - "state": ONLINE, - "mtime_age": 0, - }}, - ]}, response) + self.assertEquals({"start": "1", "end": "1", "chunk": []}, response) self.mock_datastore.set_presence_state.return_value = defer.succeed( {"state": ONLINE}) From cc48e920d6df00000e569cf4caf508cd9ea3268b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 15:54:07 +0100 Subject: [PATCH 043/112] Don't expect a reflection from events stream --- tests/rest/test_events.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 1ab92395f..4025e1458 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -190,9 +190,7 @@ class EventStreamPermissionsTestCase(RestTestCase): "/events?access_token=%s&timeout=0" % (self.token)) self.assertEquals(200, code, msg=str(response)) - # First message is a reflection of my own presence status change - self.assertEquals(1, len(response["chunk"])) - self.assertEquals("m.presence", response["chunk"][0]["type"]) + self.assertEquals(0, len(response["chunk"])) # joined room (expect all content for room) yield self.join(room=room_id, user=self.user_id, tok=self.token) From d94765999d26fd38be4eb37b9be8a3da112b5b82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 16:40:25 +0100 Subject: [PATCH 044/112] Add comment about what strorage.stream does --- synapse/storage/stream.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 87fc97881..18c1002e2 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -13,6 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" This module is responsible for getting events from the DB for pagination +and event streaming. + +The order it returns events in depend on whether we are streaming forwards or +are paginating backwards. We do this because we want to handle out of order +messages nicely, while still returning them in the correct order when we +paginate bacwards. + +This is implemented by keeping two ordering columns: stream_ordering and +topological_ordering. Stream ordering is basically insertion/received order +(except for events from backfill requests). The topolgical_ordering is a +weak ordering of events based on the pdu graph. + +This means that we have to have two different types of tokens, depending on +what sort order was used: + - stream tokens are of the form: "s%d", which maps directly to the column + - topological tokems: "t%d-%d", where the integers map to the topological + and stream ordering columns respectively. +""" + from twisted.internet import defer from ._base import SQLBaseStore From eea2dc7dde1b06af927ff34c723a9414fef5255e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 16:40:38 +0100 Subject: [PATCH 045/112] Remove debug logging from token parsing funcs. --- synapse/storage/stream.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 18c1002e2..3a67baa26 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -60,7 +60,6 @@ def _parse_stream_token(string): raise return int(string[1:]) except: - logger.debug("Not stream token: %s", string) raise SynapseError(400, "Invalid token") @@ -71,7 +70,6 @@ def _parse_topological_token(string): parts = string[1:].split('-', 1) return (int(parts[0]), int(parts[1])) except: - logger.debug("Not topological token: %s", string) raise SynapseError(400, "Invalid token") From ae493c9418b9fa2ad2e9686ff29117ae271e8cfd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 16:45:55 +0100 Subject: [PATCH 046/112] Fix token to correct format --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3a67baa26..895a61fbc 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -265,7 +265,7 @@ class StreamStore(SQLBaseStore): if rows: topo = rows[0]["topological_ordering"] toke = rows[0]["stream_ordering"] - start_token = "p%s-%s" % (topo, toke) + start_token = "t%s-%s" % (topo, toke) token = (start_token, end_token) else: From d4fb1c8a924911ea7a24b927157f8ae21087f631 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Aug 2014 17:18:19 +0100 Subject: [PATCH 047/112] Only hit get_room_events_stream if we have a valid user_id --- synapse/storage/stream.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 895a61fbc..f2be27564 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -112,8 +112,11 @@ class StreamStore(SQLBaseStore): @log_function def get_room_events(self, user_id, from_key, to_key, room_id, limit=0, direction='f', with_feedback=False): + # We deal with events request in two different ways depending on if + # this looks like an /events request or a pagination request. is_events = ( direction == 'f' + and user_id and is_stream_token(from_key) and to_key and is_stream_token(to_key) ) From 849627b82e751071f80c96f62c9e59a2565cd85c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Aug 2014 11:50:16 +0100 Subject: [PATCH 048/112] Don't generate room membership messages. Include previous state of in membership messages. --- synapse/handlers/room.py | 17 ++++++++++------- tests/rest/test_rooms.py | 7 ++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 40867ae2e..7ab881847 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -506,18 +506,21 @@ class RoomMemberHandler(BaseHandler): SynapseError if there was a problem changing the membership. """ - # broadcast_msg = False + broadcast_msg = False prev_state = yield self.store.get_room_member( event.target_user_id, event.room_id ) - if prev_state and prev_state.membership == event.membership: - # treat this event as a NOOP. - if do_auth: # This is mainly to fix a unit test. - yield self.auth.check(event, raises=True) - defer.returnValue({}) - return + if prev_state: + event.content["prev"] = prev_state.membership + +# if prev_state and prev_state.membership == event.membership: +# # treat this event as a NOOP. +# if do_auth: # This is mainly to fix a unit test. +# yield self.auth.check(event, raises=True) +# defer.returnValue({}) +# return room_id = event.room_id diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index e87318104..a9b66df91 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -794,7 +794,12 @@ class RoomMemberStateTestCase(RestTestCase): (code, response) = yield self.mock_resource.trigger("GET", path, None) self.assertEquals(200, code, msg=str(response)) - self.assertEquals(json.loads(content), response) + + expected_response = { + "membership": Membership.JOIN, + "prev": Membership.JOIN, + } + self.assertEquals(expected_response, response) @defer.inlineCallbacks def test_rooms_members_other(self): From 955662d64c971487a3c49887226a418d1aa60599 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Wed, 20 Aug 2014 13:43:31 +0200 Subject: [PATCH 049/112] Disabled sending buttons while a message is being sent. Useful on bad Internet connection. --- webclient/room/room-controller.js | 18 +++++++++++++++--- webclient/room/room.html | 6 +++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 364ca4151..2fa7fe34b 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -30,7 +30,8 @@ angular.module('RoomController', ['ngSanitize']) earliest_token: "END", // stores how far back we've paginated. can_paginate: true, // this is toggled off when we run out of items paginating: false, // used to avoid concurrent pagination requests pulling in dup contents - stream_failure: undefined // the response when the stream fails + stream_failure: undefined, // the response when the stream fails + sending: false // true when a message is being sent. It helps to disable the UI when a process is running }; $scope.members = {}; $scope.autoCompleting = false; @@ -232,7 +233,9 @@ angular.module('RoomController', ['ngSanitize']) if ($scope.textInput == "") { return; } - + + $scope.state.sending = true; + // Send the text message var promise; // FIXME: handle other commands too @@ -247,10 +250,12 @@ angular.module('RoomController', ['ngSanitize']) function() { console.log("Sent message"); $scope.textInput = ""; + $scope.state.sending = false; }, function(error) { $scope.feedback = "Failed to send: " + error.data.error; - }); + $scope.state.sending = false; + }); }; $scope.onInit = function() { @@ -362,18 +367,24 @@ angular.module('RoomController', ['ngSanitize']) }; $scope.sendImage = function(url) { + $scope.state.sending = true; + matrixService.sendImageMessage($scope.room_id, url).then( function() { console.log("Image sent"); }, function(error) { $scope.feedback = "Failed to send image: " + error.data.error; + $scope.state.sending = false; }); }; $scope.imageFileToSend; $scope.$watch("imageFileToSend", function(newValue, oldValue) { if ($scope.imageFileToSend) { + + $scope.state.sending = true; + // First download the image to the Internet console.log("Uploading image..."); mFileUpload.uploadFile($scope.imageFileToSend).then( @@ -383,6 +394,7 @@ angular.module('RoomController', ['ngSanitize']) }, function(error) { $scope.feedback = "Can't upload image"; + $scope.state.sending = false; } ); } diff --git a/webclient/room/room.html b/webclient/room/room.html index 8cae7ee51..1ca28b20f 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -63,10 +63,10 @@ {{ state.user_id }} - + - + @@ -79,7 +79,7 @@ - + - From 5ef0948eaa48d44822345efe04ec1612a96a4d37 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Aug 2014 14:42:36 +0100 Subject: [PATCH 051/112] Better handle the edge cases of trying to remote join rooms --- synapse/handlers/federation.py | 78 ++++++++++++++++++++++++++++++++++ synapse/handlers/room.py | 47 +++++--------------- 2 files changed, 88 insertions(+), 37 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0430a8307..aa3bf273f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -72,6 +72,34 @@ class FederationHandler(BaseHandler): with (yield self.room_lock.lock(event.room_id)): store_id = yield self.store.persist_event(event, backfilled) + room = yield self.store.get_room(event.room_id) + + if not room: + # Huh, let's try and get the current state + try: + federation = self.hs.get_federation() + yield federation.get_state_for_room( + event.origin, event.room_id + ) + + hosts = yield self.store.get_joined_hosts_for_room( + event.room_id + ) + if self.hs.hostname in hosts: + try: + yield self.store.store_room( + event.room_id, + "", + is_public=False + ) + except: + pass + except: + logger.exception( + "Failed to get current state for room %s", + event.room_id + ) + if not backfilled: yield self.notifier.on_new_room_event(event, store_id) @@ -88,3 +116,53 @@ class FederationHandler(BaseHandler): logger.exception("Failed to persist event: %s", event) defer.returnValue(events) + + @log_function + @defer.inlineCallbacks + def do_invite_join(self, target_host, room_id, joinee, content): + federation = self.hs.get_federation() + + hosts = yield self.store.get_joined_hosts_for_room(room_id) + if self.hs.hostname in hosts: + # We are already in the room. + logger.debug("We're already in the room apparently") + defer.returnValue(False) + + # First get current state to see if we are already joined. + try: + yield federation.get_state_for_room(target_host, room_id) + + hosts = yield self.store.get_joined_hosts_for_room(room_id) + if self.hs.hostname in hosts: + # Oh, we were actually in the room already. + logger.debug("We're already in the room apparently") + defer.returnValue(False) + except Exception: + logger.exception("Failed to get current state") + + new_event = self.event_factory.create_event( + etype=InviteJoinEvent.TYPE, + target_host=target_host, + room_id=room_id, + user_id=joinee, + content=content + ) + + new_event.destinations = [target_host] + + yield federation.handle_new_event(new_event) + + store_id = yield self.store.persist_event(new_event) + self.notifier.on_new_room_event(new_event, store_id) + + try: + yield self.store.store_room( + event.room_id, + "", + is_public=False + ) + except: + pass + + + defer.returnValue(True) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 7ab881847..6ecb6dd0e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -599,9 +599,9 @@ class RoomMemberHandler(BaseHandler): # that we are allowed to join when we decide whether or not we # need to do the invite/join dance. - room = yield self.store.get_room(room_id) + hosts = yield self.store.get_joined_hosts_for_room(room_id) - if room: + if self.hs.hostname in hosts: should_do_dance = False elif room_host: should_do_dance = True @@ -621,8 +621,15 @@ class RoomMemberHandler(BaseHandler): else: should_do_dance = False + have_joined = False + if should_do_dance: + handler = self.hs.get_handlers().federation_handler + have_joined = yield handler.do_invite_join( + room_host, room_id, event.user_id, event.content + ) + # We want to do the _do_update inside the room lock. - if not should_do_dance: + if not have_joined: logger.debug("Doing normal join") if do_auth: @@ -635,14 +642,6 @@ class RoomMemberHandler(BaseHandler): broadcast_msg=broadcast_msg, ) - if should_do_dance: - yield self._do_invite_join_dance( - room_id=room_id, - joinee=event.user_id, - target_host=room_host, - content=event.content, - ) - user = self.hs.parse_userid(event.user_id) self.distributor.fire( "user_joined_room", user=user, room_id=room_id @@ -748,32 +747,6 @@ class RoomMemberHandler(BaseHandler): membership=event.content["membership"] ) - @defer.inlineCallbacks - def _do_invite_join_dance(self, room_id, joinee, target_host, content): - logger.debug("Doing remote join dance") - - # do invite join dance - federation = self.hs.get_federation() - new_event = self.event_factory.create_event( - etype=InviteJoinEvent.TYPE, - target_host=target_host, - room_id=room_id, - user_id=joinee, - content=content - ) - - new_event.destinations = [target_host] - - yield self.store.store_room( - room_id, "", is_public=False - ) - - # yield self.state_handler.handle_new_event(event) - yield federation.handle_new_event(new_event) - yield federation.get_state_for_room( - target_host, room_id - ) - @defer.inlineCallbacks def _inject_membership_msg(self, room_id=None, source=None, target=None, membership=None): From 5c4c591c614322e175eeded7ea6e82e9c9f68f77 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Aug 2014 14:59:43 +0100 Subject: [PATCH 052/112] Fix federation test, since we now hit store.get_room --- tests/handlers/test_federation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index cb45169dd..f4cf54b7e 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2014 matrix.org # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,6 +40,7 @@ class FederationTestCase(unittest.TestCase): datastore=NonCallableMock(spec_set=[ "persist_event", "store_room", + "get_room", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -69,6 +69,7 @@ class FederationTestCase(unittest.TestCase): store_id = "ASD" self.datastore.persist_event.return_value = defer.succeed(store_id) + self.datastore.get_room.return_value = defer.succeed(True) yield self.handlers.federation_handler.on_receive(event, False, False) From d100ac8c825d08640634850b22d3bd013e0bcfd1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Aug 2014 15:10:36 +0100 Subject: [PATCH 053/112] Fix test. get_joined_hosts_for_room get's called multiple times --- tests/handlers/test_room.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index bfdde6135..be68f1769 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -140,9 +140,11 @@ class RoomMemberHandlerTestCase(unittest.TestCase): joined = ["red", "green"] self.state_handler.handle_new_event.return_value = defer.succeed(True) - self.datastore.get_joined_hosts_for_room.return_value = ( - defer.succeed(joined) - ) + + def get_joined(*args): + return defer.succeed(joined) + + self.datastore.get_joined_hosts_for_room.side_effect = get_joined store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) From e8244c23baf150c59e737761c15ce540a3e9e26f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Aug 2014 15:53:07 +0100 Subject: [PATCH 054/112] Give the event_id of the failed event --- synapse/storage/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 470b7b766..773290692 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -121,7 +121,10 @@ class DataStore(RoomMemberStore, RoomStore, try: yield self._simple_insert("events", vals) except: - logger.exception("Failed to persist, probably duplicate") + logger.exception( + "Failed to persist, probably duplicate: %s", + event_id + ) return if not backfilled and hasattr(event, "state_key"): From da2f5aac0eb2124654fe8e8c76f978cab31e9729 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Wed, 20 Aug 2014 15:41:14 +0200 Subject: [PATCH 055/112] Sanitize message text content only if the type of current message in the ng-repeat loop is "text" In case of image message, the body can be a JSON object (ImageInfo) and ngSanitize does not like that (ie it generates exception in the console) --- webclient/room/room.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index 99b0bf681..e89bf670e 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -40,8 +40,8 @@
- - {{ msg.content.body }} + +
From 6d3391f2f0bbbc99d3a6201bdb134b04e0b10f18 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Wed, 20 Aug 2014 16:18:50 +0200 Subject: [PATCH 056/112] Send images with their imageInfo (size, mymetype, width & height) --- .../fileUpload/file-upload-service.js | 4 +- webclient/components/matrix/matrix-service.js | 4 +- .../components/utilities/utilities-service.js | 53 +++++++++++++++++++ webclient/index.html | 1 + webclient/room/room-controller.js | 44 ++++++++++----- 5 files changed, 89 insertions(+), 17 deletions(-) create mode 100644 webclient/components/utilities/utilities-service.js diff --git a/webclient/components/fileUpload/file-upload-service.js b/webclient/components/fileUpload/file-upload-service.js index d620e6a4d..65c24f309 100644 --- a/webclient/components/fileUpload/file-upload-service.js +++ b/webclient/components/fileUpload/file-upload-service.js @@ -27,10 +27,10 @@ angular.module('mFileUpload', []) * Upload an HTML5 file to a server and returned a promise * that will provide the URL of the uploaded file. */ - this.uploadFile = function(file) { + this.uploadFile = function(file, body) { var deferred = $q.defer(); console.log("Uploading " + file.name + "... to /matrix/content"); - matrixService.uploadContent(file).then( + matrixService.uploadContent(file, body).then( function(response) { var content_url = location.origin + "/matrix/content/" + response.data.content_token; console.log(" -> Successfully uploaded! Available at " + content_url); diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index 664c5967a..cd37a0c23 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -204,11 +204,11 @@ angular.module('matrixService', []) }, // Send an image message - sendImageMessage: function(room_id, image_url, image_alt, msg_id) { + sendImageMessage: function(room_id, image_url, image_body, msg_id) { var content = { msgtype: "m.image", url: image_url, - body: image_alt + body: image_body }; return this.sendMessage(room_id, msg_id, content); diff --git a/webclient/components/utilities/utilities-service.js b/webclient/components/utilities/utilities-service.js new file mode 100644 index 000000000..fc0ee580d --- /dev/null +++ b/webclient/components/utilities/utilities-service.js @@ -0,0 +1,53 @@ +/* + Copyright 2014 matrix.org + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +'use strict'; + +/* + * This service contains multipurpose helper functions. + */ +angular.module('mUtilities', []) +.service('mUtilities', ['$q', function ($q) { + /* + * Gets the size of an image + * @param {File} imageFile the file containing the image + * @returns {promise} A promise that will be resolved by an object with 2 members: + * width & height + */ + this.getImageSize = function(imageFile) { + var deferred = $q.defer(); + + // Load the file into an html element + var img = document.createElement("img"); + + var reader = new FileReader(); + reader.onload = function(e) { + img.src = e.target.result; + + // Once ready, returns its size + deferred.resolve({ + width: img.width, + height: img.height + }); + }; + reader.onerror = function(e) { + deferred.reject(e); + }; + reader.readAsDataURL(imageFile); + + return deferred.promise; + }; +}]); \ No newline at end of file diff --git a/webclient/index.html b/webclient/index.html index a7e9cd934..27d920819 100644 --- a/webclient/index.html +++ b/webclient/index.html @@ -25,6 +25,7 @@ + diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index ca6d3d4a3..558a865f3 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -angular.module('RoomController', ['ngSanitize']) -.controller('RoomController', ['$scope', '$http', '$timeout', '$routeParams', '$location', 'matrixService', 'eventStreamService', 'eventHandlerService', 'mFileUpload', - function($scope, $http, $timeout, $routeParams, $location, matrixService, eventStreamService, eventHandlerService, mFileUpload) { +angular.module('RoomController', ['ngSanitize', 'mUtilities']) +.controller('RoomController', ['$scope', '$http', '$timeout', '$routeParams', '$location', 'matrixService', 'eventStreamService', 'eventHandlerService', 'mFileUpload', 'mUtilities', + function($scope, $http, $timeout, $routeParams, $location, matrixService, eventStreamService, eventHandlerService, mFileUpload, mUtilities) { 'use strict'; var MESSAGES_PER_PAGINATION = 30; @@ -366,10 +366,10 @@ angular.module('RoomController', ['ngSanitize']) }); }; - $scope.sendImage = function(url) { + $scope.sendImage = function(url, body) { $scope.state.sending = true; - matrixService.sendImageMessage($scope.room_id, url).then( + matrixService.sendImageMessage($scope.room_id, url, body).then( function() { console.log("Image sent"); $scope.state.sending = false; @@ -386,17 +386,35 @@ angular.module('RoomController', ['ngSanitize']) $scope.state.sending = true; - // First download the image to the Internet - console.log("Uploading image..."); - mFileUpload.uploadFile($scope.imageFileToSend).then( - function(url) { - // Then share the URL - $scope.sendImage(url); + // First, get the image sise + mUtilities.getImageSize($scope.imageFileToSend).then( + function(size) { + + // Upload the image to the Internet + console.log("Uploading image..."); + mFileUpload.uploadFile($scope.imageFileToSend).then( + function(url) { + // Build the image info data + var imageInfo = { + size: $scope.imageFileToSend.size, + mimetype: $scope.imageFileToSend.type, + w: size.width, + h: size.height + }; + + // Then share the URL and the metadata + $scope.sendImage(url, imageInfo); + }, + function(error) { + $scope.feedback = "Can't upload image"; + $scope.state.sending = false; + } + ); }, function(error) { - $scope.feedback = "Can't upload image"; + $scope.feedback = "Can't get selected image size"; $scope.state.sending = false; - } + } ); } }); From ba88c9105c0bd071c3cf044cd5d3aa4604c81e93 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Wed, 20 Aug 2014 17:04:32 +0200 Subject: [PATCH 057/112] Create a placeholder for each image of the chat thread. The height of this placeholder is the height of the image so that the scroller position will not be disrupted when the image will be actually loaded and displayed in its full height --- webclient/room/room.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index e89bf670e..4d7417a80 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -41,7 +41,10 @@
- +
+ +
From 9c0e5704963f232a14545f26a4501b672a32beb4 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 20 Aug 2014 14:58:45 +0100 Subject: [PATCH 058/112] Kill the "_homeserver_" injected messages for room membership changes --- synapse/handlers/room.py | 60 +++------------------------------------- synapse/rest/room.py | 4 +-- 2 files changed, 6 insertions(+), 58 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6ecb6dd0e..6bdba3f5e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -383,7 +383,6 @@ class RoomCreationHandler(BaseHandler): yield self.hs.get_handlers().room_member_handler.change_membership( join_event, - broadcast_msg=True, do_auth=False ) @@ -495,19 +494,15 @@ class RoomMemberHandler(BaseHandler): defer.returnValue(member) @defer.inlineCallbacks - def change_membership(self, event=None, broadcast_msg=False, do_auth=True): + def change_membership(self, event=None, do_auth=True): """ Change the membership status of a user in a room. Args: event (SynapseEvent): The membership event - broadcast_msg (bool): True to inject a membership message into this - room on success. Raises: SynapseError if there was a problem changing the membership. """ - broadcast_msg = False - prev_state = yield self.store.get_room_member( event.target_user_id, event.room_id ) @@ -528,9 +523,7 @@ class RoomMemberHandler(BaseHandler): # if this HS is not currently in the room, i.e. we have to do the # invite/join dance. if event.membership == Membership.JOIN: - yield self._do_join( - event, do_auth=do_auth, broadcast_msg=broadcast_msg - ) + yield self._do_join(event, do_auth=do_auth) else: # This is not a JOIN, so we can handle it normally. if do_auth: @@ -548,7 +541,6 @@ class RoomMemberHandler(BaseHandler): yield self._do_local_membership_update( event, membership=event.content["membership"], - broadcast_msg=broadcast_msg, ) defer.returnValue({"room_id": room_id}) @@ -583,8 +575,7 @@ class RoomMemberHandler(BaseHandler): defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks - def _do_join(self, event, room_host=None, do_auth=True, - broadcast_msg=True): + def _do_join(self, event, room_host=None, do_auth=True): joinee = self.hs.parse_userid(event.target_user_id) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -639,7 +630,6 @@ class RoomMemberHandler(BaseHandler): yield self._do_local_membership_update( event, membership=event.content["membership"], - broadcast_msg=broadcast_msg, ) user = self.hs.parse_userid(event.user_id) @@ -710,7 +700,7 @@ class RoomMemberHandler(BaseHandler): defer.returnValue([r.room_id for r in rooms]) @defer.inlineCallbacks - def _do_local_membership_update(self, event, membership, broadcast_msg): + def _do_local_membership_update(self, event, membership): # store membership store_id = yield self.store.persist_event(event) @@ -739,48 +729,6 @@ class RoomMemberHandler(BaseHandler): yield self.hs.get_federation().handle_new_event(event) self.notifier.on_new_room_event(event, store_id) - if broadcast_msg: - yield self._inject_membership_msg( - source=event.user_id, - target=event.target_user_id, - room_id=event.room_id, - membership=event.content["membership"] - ) - - @defer.inlineCallbacks - def _inject_membership_msg(self, room_id=None, source=None, target=None, - membership=None): - # TODO this should be a different type of message, not m.text - if membership == Membership.INVITE: - body = "%s invited %s to the room." % (source, target) - elif membership == Membership.JOIN: - body = "%s joined the room." % (target) - elif membership == Membership.LEAVE: - body = "%s left the room." % (target) - else: - raise RoomError(500, "Unknown membership value %s" % membership) - - membership_json = { - "msgtype": u"m.text", - "body": body, - "membership_source": source, - "membership_target": target, - "membership": membership, - } - - msg_id = "m%s" % int(self.clock.time_msec()) - - event = self.event_factory.create_event( - etype=MessageEvent.TYPE, - room_id=room_id, - user_id="_homeserver_", - msg_id=msg_id, - content=membership_json - ) - - handler = self.hs.get_handlers().message_handler - yield handler.send_message(event, suppress_auth=True) - class RoomListHandler(BaseHandler): diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 1c48e6362..f5b547b96 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -193,7 +193,7 @@ class RoomMemberRestServlet(RestServlet): ) handler = self.handlers.room_member_handler - yield handler.change_membership(event, broadcast_msg=True) + yield handler.change_membership(event) defer.returnValue((200, "")) @defer.inlineCallbacks @@ -220,7 +220,7 @@ class RoomMemberRestServlet(RestServlet): ) handler = self.handlers.room_member_handler - yield handler.change_membership(event, broadcast_msg=True) + yield handler.change_membership(event) defer.returnValue((200, "")) From e01bdf2432511b8bf51ecfd4fb46f140dbedae04 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 20 Aug 2014 15:25:17 +0100 Subject: [PATCH 059/112] Define __copy__ and __deepcopy__ as identity functions on DomainSpecificString, so that copy.deepcopy() will work on them --- synapse/types.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/types.py b/synapse/types.py index b8e191bb3..fd6a3d1d7 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -38,6 +38,14 @@ class DomainSpecificString( def __iter__(self): raise ValueError("Attempted to iterate a %s" % (type(self).__name__)) + # Because this class is a namedtuple of strings and booleans, it is deeply + # immutable. + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + @classmethod def from_string(cls, s, hs): """Parse the string given by 's' into a structure object.""" From 50718825bd8d0ecc7ca8e700d2187360857ac8df Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 20 Aug 2014 15:50:37 +0100 Subject: [PATCH 060/112] Fix exception name in _fill_out_join_content() exception --- synapse/handlers/room.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6bdba3f5e..4c297dbe3 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -640,6 +640,8 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _fill_out_join_content(self, user_id, content): # If event doesn't include a display name, add one. + # TODO(paul): This really ought to use the distributor's + # collect_presencelike_data signal instead. profile_handler = self.hs.get_handlers().profile_handler if "displayname" not in content: try: @@ -661,7 +663,7 @@ class RoomMemberHandler(BaseHandler): if avatar_url: content["avatar_url"] = avatar_url except: - logger.exception("Failed to set display_name") + logger.exception("Failed to set avatar_url") @defer.inlineCallbacks def _should_invite_join(self, room_id, prev_state, do_auth): From 583add34fe6908f642a78be9d08a15e0b47498d0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 20 Aug 2014 16:04:01 +0100 Subject: [PATCH 061/112] Use the "collect_presencelike_data" distributor signal instead of re-implementing its behaviour --- synapse/handlers/room.py | 32 ++------------------------------ tests/handlers/test_room.py | 2 ++ 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 4c297dbe3..6229ee9bf 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -581,8 +581,8 @@ class RoomMemberHandler(BaseHandler): room_id = event.room_id # If event doesn't include a display name, add one. - yield self._fill_out_join_content( - joinee, event.content + yield self.distributor.fire( + "collect_presencelike_data", joinee, event.content ) # XXX: We don't do an auth check if we are doing an invite @@ -637,34 +637,6 @@ class RoomMemberHandler(BaseHandler): "user_joined_room", user=user, room_id=room_id ) - @defer.inlineCallbacks - def _fill_out_join_content(self, user_id, content): - # If event doesn't include a display name, add one. - # TODO(paul): This really ought to use the distributor's - # collect_presencelike_data signal instead. - profile_handler = self.hs.get_handlers().profile_handler - if "displayname" not in content: - try: - display_name = yield profile_handler.get_displayname( - user_id - ) - - if display_name: - content["displayname"] = display_name - except: - logger.exception("Failed to set display_name") - - if "avatar_url" not in content: - try: - avatar_url = yield profile_handler.get_avatar_url( - user_id - ) - - if avatar_url: - content["avatar_url"] = avatar_url - except: - logger.exception("Failed to set avatar_url") - @defer.inlineCallbacks def _should_invite_join(self, room_id, prev_state, do_auth): logger.debug("_should_invite_join: room_id: %s", room_id) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index be68f1769..bf71d3be3 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -69,6 +69,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.distributor = hs.get_distributor() self.hs = hs + self.distributor.declare("collect_presencelike_data") + self.handlers.room_member_handler = RoomMemberHandler(self.hs) self.handlers.profile_handler = ProfileHandler(self.hs) self.room_member_handler = self.handlers.room_member_handler From 96da42085cc77e9f75009ea6152bc1bc64a2dcc8 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Wed, 20 Aug 2014 17:08:05 +0200 Subject: [PATCH 062/112] BF: Wait for the room_id being resolved before starting pagination --- webclient/room/room-controller.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 558a865f3..35abeeca0 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -89,7 +89,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) var paginate = function(numItems) { // console.log("paginate " + numItems); - if ($scope.state.paginating) { + if ($scope.state.paginating || !$scope.room_id) { return; } else { @@ -145,7 +145,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) console.log("Failed to paginateBackMessages: " + JSON.stringify(error)); $scope.state.paginating = false; } - ) + ); }; var updateMemberList = function(chunk) { From 2f52e8ee18126fef4a4a13d27077ca926aed1425 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Wed, 20 Aug 2014 17:17:17 +0200 Subject: [PATCH 063/112] BF: Apply image place holder only if the image message has the height information --- webclient/room/room.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index 4d7417a80..db6add4ee 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -42,7 +42,7 @@
+ ng-style="msg.content.body.h && { 'height' : (msg.content.body.h < 320) ? msg.content.body.h : 320}">
From 7371e68f55bedff5d88ed0bd504edaed5642a2f5 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Wed, 20 Aug 2014 17:46:16 +0200 Subject: [PATCH 064/112] Quick fix to support array of room aliases --- webclient/rooms/rooms-controller.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js index a237b59b4..c25e24c8b 100644 --- a/webclient/rooms/rooms-controller.js +++ b/webclient/rooms/rooms-controller.js @@ -71,9 +71,10 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', // use the existing alias from storage data[i].room_alias = alias; } - else if (data[i].room_alias) { + else if (data[i].aliases && data[i].aliases[0]) { // save the mapping - matrixService.createRoomIdToAliasMapping(data[i].room_id, data[i].room_alias); + // TODO: select the smarter alias from the array + matrixService.createRoomIdToAliasMapping(data[i].room_id, data[i].aliases[0]); } else { // last resort use the room id From ebd3c41edeebcceb990fc3d60974c5166a7d143a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 20 Aug 2014 16:07:20 +0100 Subject: [PATCH 065/112] Make event stream storage return all membership events about the user, regardless of if they were in the room or not. --- synapse/storage/stream.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index f2be27564..e994017bf 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -151,10 +151,12 @@ class StreamStore(SQLBaseStore): "WHERE m.user_id = ?" ) - invites_sql = ( + # We also want to get any membership events about that user, e.g. + # invites or leave notifications. + membership_sql = ( "SELECT m.event_id FROM room_memberships as m " "INNER JOIN current_state_events as c ON m.event_id = c.event_id " - "WHERE m.user_id = ? AND m.membership = ?" + "WHERE m.user_id = ? " ) if limit: @@ -178,13 +180,13 @@ class StreamStore(SQLBaseStore): "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "current": current_room_membership_sql, - "invites": invites_sql, + "invites": membership_sql, "limit": limit } rows = yield self._execute_and_decode( sql, - user_id, user_id, Membership.INVITE, from_id, to_id + user_id, user_id, from_id, to_id ) ret = [self._parse_event_from_row(r) for r in rows] From efe5aa6464200a69d3159c52e151a0131923f46d Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Thu, 21 Aug 2014 13:35:04 +0200 Subject: [PATCH 066/112] Added resizeImage() --- .../components/utilities/utilities-service.js | 87 ++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/webclient/components/utilities/utilities-service.js b/webclient/components/utilities/utilities-service.js index fc0ee580d..f9e52eacf 100644 --- a/webclient/components/utilities/utilities-service.js +++ b/webclient/components/utilities/utilities-service.js @@ -22,7 +22,7 @@ angular.module('mUtilities', []) .service('mUtilities', ['$q', function ($q) { /* - * Gets the size of an image + * Get the size of an image * @param {File} imageFile the file containing the image * @returns {promise} A promise that will be resolved by an object with 2 members: * width & height @@ -50,4 +50,89 @@ angular.module('mUtilities', []) return deferred.promise; }; + + /* + * Resize the image to fit in a square of the side maxSize. + * The aspect ratio is kept. The returned image data uses JPEG compression. + * Source: http://hacks.mozilla.org/2011/01/how-to-develop-a-html5-image-uploader/ + * @param {File} imageFile the file containing the image + * @param {Integer} maxSize the max side size + * @returns {promise} A promise that will be resolved by a Blob object containing + * the resized image data + */ + this.resizeImage = function(imageFile, maxSize) { + var self = this; + var deferred = $q.defer(); + + var canvas = document.createElement("canvas"); + + var img = document.createElement("img"); + var reader = new FileReader(); + reader.onload = function(e) { + + img.src = e.target.result; + + var ctx = canvas.getContext("2d"); + ctx.drawImage(img, 0, 0); + + var MAX_WIDTH = maxSize; + var MAX_HEIGHT = maxSize; + var width = img.width; + var height = img.height; + + if (width > height) { + if (width > MAX_WIDTH) { + height *= MAX_WIDTH / width; + width = MAX_WIDTH; + } + } else { + if (height > MAX_HEIGHT) { + width *= MAX_HEIGHT / height; + height = MAX_HEIGHT; + } + } + canvas.width = width; + canvas.height = height; + var ctx = canvas.getContext("2d"); + ctx.drawImage(img, 0, 0, width, height); + + var dataUrl = canvas.toDataURL("image/jpeg", 0.7); + deferred.resolve(self.dataURItoBlob(dataUrl)); + }; + reader.onerror = function(e) { + deferred.reject(e); + }; + reader.readAsDataURL(imageFile); + + return deferred.promise; + }; + + /* + * Convert a dataURI string to a blob + * Source: http://stackoverflow.com/a/17682951 + * @param {String} dataURI the dataURI can be a base64 encoded string or an URL encoded string. + * @returns {Blob} the blob + */ + this.dataURItoBlob = function(dataURI) { + // convert base64 to raw binary data held in a string + // doesn't handle URLEncoded DataURIs + var byteString; + if (dataURI.split(',')[0].indexOf('base64') >= 0) + byteString = atob(dataURI.split(',')[1]); + else + byteString = unescape(dataURI.split(',')[1]); + // separate out the mime component + var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0]; + + // write the bytes of the string to an ArrayBuffer + var ab = new ArrayBuffer(byteString.length); + var ia = new Uint8Array(ab); + for (var i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + + // write the ArrayBuffer to a blob, and you're done + return new Blob([ab],{type: mimeString}); + }; + }]); \ No newline at end of file From 9d4bc8985f72ffffedd510eaa39b2bd5dc71f9ed Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Thu, 21 Aug 2014 13:36:14 +0200 Subject: [PATCH 067/112] Made uploadContent compatible for sending Blob objects --- webclient/components/matrix/matrix-service.js | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index cd37a0c23..fa5a6091d 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -61,14 +61,22 @@ angular.module('matrixService', []) return doBaseRequest(config.homeserver, method, path, params, data, undefined); }; - var doBaseRequest = function(baseUrl, method, path, params, data, headers) { - return $http({ + var doBaseRequest = function(baseUrl, method, path, params, data, headers, $httpParams) { + + var request = { method: method, url: baseUrl + path, params: params, data: data, headers: headers - }); + }; + + // Add additional $http parameters + if ($httpParams) { + angular.extend(request, $httpParams); + } + + return $http(request); }; @@ -326,7 +334,17 @@ angular.module('matrixService', []) var params = { access_token: config.access_token }; - return doBaseRequest(config.homeserver, "POST", path, params, file, headers); + + // If the file is actually a Blob object, prevent $http from JSON-stringified it before sending + // (Equivalent to jQuery ajax processData = false) + var $httpParams; + if (file instanceof Blob) { + $httpParams = { + transformRequest: angular.identity + }; + } + + return doBaseRequest(config.homeserver, "POST", path, params, file, headers, $httpParams); }, // start listening on /events From aac52fce15a592ac0715f72864f144600e4c15a1 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Thu, 21 Aug 2014 14:30:41 +0200 Subject: [PATCH 068/112] Generate thumbnail client side and send its URL and info with the image message body --- .../fileUpload/file-upload-service.js | 141 +++++++++++++++++- .../components/utilities/utilities-service.js | 2 +- webclient/room/room-controller.js | 34 ++--- 3 files changed, 149 insertions(+), 28 deletions(-) diff --git a/webclient/components/fileUpload/file-upload-service.js b/webclient/components/fileUpload/file-upload-service.js index 65c24f309..6606f31e2 100644 --- a/webclient/components/fileUpload/file-upload-service.js +++ b/webclient/components/fileUpload/file-upload-service.js @@ -20,17 +20,18 @@ /* * Upload an HTML5 file to a server */ -angular.module('mFileUpload', []) -.service('mFileUpload', ['matrixService', '$q', function (matrixService, $q) { +angular.module('mFileUpload', ['matrixService', 'mUtilities']) +.service('mFileUpload', ['$q', 'matrixService', 'mUtilities', function ($q, matrixService, mUtilities) { /* - * Upload an HTML5 file to a server and returned a promise + * Upload an HTML5 file or blob to a server and returned a promise * that will provide the URL of the uploaded file. + * @param {File|Blob} file the file data to send */ - this.uploadFile = function(file, body) { + this.uploadFile = function(file) { var deferred = $q.defer(); console.log("Uploading " + file.name + "... to /matrix/content"); - matrixService.uploadContent(file, body).then( + matrixService.uploadContent(file).then( function(response) { var content_url = location.origin + "/matrix/content/" + response.data.content_token; console.log(" -> Successfully uploaded! Available at " + content_url); @@ -44,4 +45,134 @@ angular.module('mFileUpload', []) return deferred.promise; }; + + /* + * Upload an image file plus generate a thumbnail of it and upload it so that + * we will have all information to fulfill an image message request data. + * @param {File} imageFile the imageFile to send + * @param {Integer} thumbnailSize the max side size of the thumbnail to create + * @returns {promise} A promise that will be resolved by a image message object + * ready to be send with the Matrix API + */ + this.uploadImageAndThumbnail = function(imageFile, thumbnailSize) { + var self = this; + var deferred = $q.defer(); + + console.log("uploadImageAndThumbnail " + imageFile.name + " - thumbnailSize: " + thumbnailSize); + + // The message structure that will be returned in the promise + var imageMessage = { + msgtype: "m.image", + url: undefined, + body: { + size: undefined, + w: undefined, + h: undefined, + mimetype: undefined + }, + thumbnail_url: undefined, + thumbnail_info: { + size: undefined, + w: undefined, + h: undefined, + mimetype: undefined + } + }; + + // First, get the image size + mUtilities.getImageSize(imageFile).then( + function(size) { + + // The final operation: send imageFile + var uploadImage = function() { + self.uploadFile(imageFile).then( + function(url) { + // Update message metadata + imageMessage.url = url; + imageMessage.body = { + size: imageFile.size, + w: size.width, + h: size.height, + mimetype: imageFile.type + }; + + // If there is no thumbnail (because the original image is smaller than thumbnailSize), + // reuse the original image info for thumbnail data + if (!imageMessage.thumbnail_url) { + imageMessage.thumbnail_url = imageMessage.url; + imageMessage.thumbnail_info = imageMessage.body; + } + + // We are done + deferred.resolve(imageMessage); + }, + function(error) { + console.log(" -> Can't upload image"); + deferred.reject(error); + } + ); + }; + + // Create a thumbnail if the image size exceeds thumbnailSize + if (Math.max(size.width, size.height) > thumbnailSize) { + console.log(" Creating thumbnail..."); + mUtilities.resizeImage(imageFile, thumbnailSize).then( + function(thumbnailBlob) { + + // Get its size + mUtilities.getImageSize(thumbnailBlob).then( + function(thumbnailSize) { + console.log(" -> Thumbnail size: " + JSON.stringify(thumbnailSize)); + + // Upload it to the server + self.uploadFile(thumbnailBlob).then( + function(thumbnailUrl) { + + // Update image message data + imageMessage.thumbnail_url = thumbnailUrl; + imageMessage.thumbnail_info = { + size: thumbnailBlob.size, + w: thumbnailSize.width, + h: thumbnailSize.height, + mimetype: thumbnailBlob.type + }; + + // Then, upload the original image + uploadImage(); + }, + function(error) { + console.log(" -> Can't upload thumbnail"); + deferred.reject(error); + } + ); + }, + function(error) { + console.log(" -> Failed to get thumbnail size"); + deferred.reject(error); + } + ); + + }, + function(error) { + console.log(" -> Failed to create thumbnail: " + error); + deferred.reject(error); + } + ); + } + else { + // No need of thumbnail + console.log(" Thumbnail is not required"); + uploadImage(); + } + + }, + function(error) { + console.log(" -> Failed to get image size"); + deferred.reject(error); + } + ); + + return deferred.promise; + }; + }]); diff --git a/webclient/components/utilities/utilities-service.js b/webclient/components/utilities/utilities-service.js index f9e52eacf..9cf858ef3 100644 --- a/webclient/components/utilities/utilities-service.js +++ b/webclient/components/utilities/utilities-service.js @@ -23,7 +23,7 @@ angular.module('mUtilities', []) .service('mUtilities', ['$q', function ($q) { /* * Get the size of an image - * @param {File} imageFile the file containing the image + * @param {File|Blob} imageFile the file containing the image * @returns {promise} A promise that will be resolved by an object with 2 members: * width & height */ diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 35abeeca0..7de50dd96 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -19,6 +19,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) function($scope, $http, $timeout, $routeParams, $location, matrixService, eventStreamService, eventHandlerService, mFileUpload, mUtilities) { 'use strict'; var MESSAGES_PER_PAGINATION = 30; + var THUMBNAIL_SIZE = 320; // Room ids. Computed and resolved in onInit $scope.room_id = undefined; @@ -386,33 +387,22 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) $scope.state.sending = true; - // First, get the image sise - mUtilities.getImageSize($scope.imageFileToSend).then( - function(size) { - - // Upload the image to the Internet - console.log("Uploading image..."); - mFileUpload.uploadFile($scope.imageFileToSend).then( - function(url) { - // Build the image info data - var imageInfo = { - size: $scope.imageFileToSend.size, - mimetype: $scope.imageFileToSend.type, - w: size.width, - h: size.height - }; - - // Then share the URL and the metadata - $scope.sendImage(url, imageInfo); + // Upload this image with its thumbnail to Internet + mFileUpload.uploadImageAndThumbnail($scope.imageFileToSend, THUMBNAIL_SIZE).then( + function(imageMessage) { + // imageMessage is complete message structure, send it as is + matrixService.sendMessage($scope.room_id, undefined, imageMessage).then( + function() { + console.log("Image message sent"); + $scope.state.sending = false; }, function(error) { - $scope.feedback = "Can't upload image"; + $scope.feedback = "Failed to send image message: " + error.data.error; $scope.state.sending = false; - } - ); + }); }, function(error) { - $scope.feedback = "Can't get selected image size"; + $scope.feedback = "Can't upload image"; $scope.state.sending = false; } ); From e4f0e1af1aa075e6d86921d46e824d85855c1def Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Thu, 21 Aug 2014 14:58:26 +0200 Subject: [PATCH 069/112] If there are available, show image thumbnails in the messages list --- webclient/room/room.html | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index db6add4ee..5dcc8caa1 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -41,9 +41,13 @@
-
- +
+
+ +
+
+ +
From 1587ea26fef65157f2a35b150f01bd8035e5e785 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 14:38:22 +0100 Subject: [PATCH 070/112] Wait for getting a Join in response to an invite/join dance. --- synapse/handlers/_base.py | 1 + synapse/handlers/federation.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index c2f4685c9..3f07b5aa4 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -24,4 +24,5 @@ class BaseHandler(object): self.notifier = hs.get_notifier() self.room_lock = hs.get_room_lock_manager() self.state_handler = hs.get_state_handler() + self.distributor = hs.get_distributor() self.hs = hs diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index aa3bf273f..9cff44477 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -32,6 +32,15 @@ logger = logging.getLogger(__name__) class FederationHandler(BaseHandler): """Handles events that originated from federation.""" + def __init__(self, hs): + super(FederationHandler, self).__init__(hs) + + self.distributor.observe( + "user_joined_room", + self._on_user_joined + ) + + self.waiting_for_join_list = {} @log_function @defer.inlineCallbacks @@ -103,6 +112,13 @@ class FederationHandler(BaseHandler): if not backfilled: yield self.notifier.on_new_room_event(event, store_id) + if event.type == RoomMemberEvent.TYPE: + if event.membership == Membership.JOIN: + user = self.hs.parse_userid(event.target_user_id) + self.distributor.fire( + "user_joined_room", user=user, room_id=event.room_id + ) + @log_function @defer.inlineCallbacks @@ -152,8 +168,10 @@ class FederationHandler(BaseHandler): yield federation.handle_new_event(new_event) - store_id = yield self.store.persist_event(new_event) - self.notifier.on_new_room_event(new_event, store_id) + # TODO (erikj): Time out here. + d = defer.Deferred() + self.waiting_for_join_list.setdefault((joinee, room_id), []).append(d) + yield d try: yield self.store.store_room( @@ -166,3 +184,10 @@ class FederationHandler(BaseHandler): defer.returnValue(True) + + + @log_function + def _on_user_joined(self, user, room_id): + waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) + while waiters: + waiters.pop().callback(None) From 063e1b22e62915ec77bfd3cb9477c29600acb568 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 15:06:00 +0100 Subject: [PATCH 071/112] Stop internal keys from getting into SynapseEvents --- synapse/api/events/__init__.py | 1 + synapse/storage/_base.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 921fd0883..aa04dbece 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -51,6 +51,7 @@ class SynapseEvent(JsonEncodedObject): "depth", "destinations", "origin", + "outlier", ] required_keys = [ diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 36cc57c1b..75aab2d3b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -294,6 +294,11 @@ class SQLBaseStore(object): def _parse_event_from_row(self, row_dict): d = copy.deepcopy({k: v for k, v in row_dict.items() if v}) + + d.pop("stream_ordering", None) + d.pop("topological_ordering", None) + d.pop("processed", None) + d.update(json.loads(row_dict["unrecognized_keys"])) d["content"] = json.loads(d["content"]) del d["unrecognized_keys"] From c6950b18cca665f6afe8ac00fcfa2322d8b35544 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 15:06:22 +0100 Subject: [PATCH 072/112] Return the current state in the initial sync api. --- synapse/handlers/room.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6229ee9bf..91415afbb 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -279,6 +279,9 @@ class MessageHandler(BaseHandler): "start": token[0], "end": token[1], } + + current_state = yield self.store.get_current_state(event.room_id) + d["state"] = [c.get_dict() for c in current_state] except: logger.exception("Failed to get snapshot") From 3d1cae0e7954085bdc1dd1fca6a4ea4986e3d6f5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 15:07:08 +0100 Subject: [PATCH 073/112] In the initial sync api, return the inviter for rooms in the 'invited' state --- synapse/handlers/room.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 91415afbb..d9809bd6d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -264,6 +264,10 @@ class MessageHandler(BaseHandler): "room_id": event.room_id, "membership": event.membership, } + + if event.membership == Membership.INVITE: + d["inviter"] = event.user_id + ret.append(d) if event.membership != Membership.JOIN: From bb4490c2d78d4e0dcae01853513e0308776055a5 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Thu, 21 Aug 2014 16:09:42 +0200 Subject: [PATCH 074/112] Show image fullscreen when clicking on the thumbnail --- webclient/app.css | 25 ++++++++++++++++++++++++- webclient/room/room.html | 9 +++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/webclient/app.css b/webclient/app.css index 869db69cd..d2b951d3b 100644 --- a/webclient/app.css +++ b/webclient/app.css @@ -66,6 +66,10 @@ h1 { background-color: #faa; } +.mouse-pointer { + cursor: pointer; +} + /*** Participant list ***/ #usersTableWrapper { @@ -89,7 +93,6 @@ h1 { height: 100px; position: relative; background-color: #000; - cursor: pointer; } .userAvatar .userAvatarImage { @@ -245,6 +248,26 @@ h1 { text-align: left ! important; } +#room-fullscreen-image { + position: absolute; + top: 0px; + height: 0px; + width: 100%; + height: 100%; +} + +#room-fullscreen-image img { + max-width: 100%; + max-height: 100%; + bottom: 0; + left: 0; + margin: auto; + overflow: auto; + position: fixed; + right: 0; + top: 0; +} + /*** Profile ***/ .profile-avatar { diff --git a/webclient/room/room.html b/webclient/room/room.html index 5dcc8caa1..cb9cf1d1f 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -10,7 +10,7 @@
-
+ {{ member.displayname || member.id.substr(0, member.id.indexOf(':')) }}
- +
@@ -96,4 +97,8 @@ +
+ +
+ From 01a129cb9a3dea54faf65bea4cf10dee22b55fde Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 21 Aug 2014 15:26:19 +0100 Subject: [PATCH 075/112] cheer up erik and remove the double-horizontal-border between adjacent text plinths --- webclient/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/webclient/app.css b/webclient/app.css index 869db69cd..1717b1b35 100644 --- a/webclient/app.css +++ b/webclient/app.css @@ -192,6 +192,7 @@ h1 { border: 1px solid #d8d8d8; height: 31px; display: inline-table; + margin-top: -1px; max-width: 90%; font-size: 16px; /* word-wrap: break-word; */ From 14b99896604c8860ebb2a6ed607fae40fe04494b Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Thu, 21 Aug 2014 16:27:15 +0200 Subject: [PATCH 076/112] Fixed first pagination detection --- webclient/room/room-controller.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 35abeeca0..6d714151f 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -28,6 +28,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) user_id: matrixService.config().user_id, events_from: "END", // when to start the event stream from. earliest_token: "END", // stores how far back we've paginated. + first_pagination: true, // this is toggled off when the first pagination is done can_paginate: true, // this is toggled off when we run out of items paginating: false, // used to avoid concurrent pagination requests pulling in dup contents stream_failure: undefined, // the response when the stream fails @@ -99,7 +100,6 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) var originalTopRow = $("#messageTable>tbody>tr:first")[0]; matrixService.paginateBackMessages($scope.room_id, $scope.state.earliest_token, numItems).then( function(response) { - var firstPagination = !$scope.events.rooms[$scope.room_id]; eventHandlerService.handleEvents(response.data.chunk, false); $scope.state.earliest_token = response.data.end; if (response.data.chunk.length < MESSAGES_PER_PAGINATION) { @@ -125,8 +125,9 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) }, 0); } - if (firstPagination) { + if ($scope.state.first_pagination) { scrollToBottom(); + $scope.state.first_pagination = false; } else { // lock the scroll position From 4c228df167ca1708964c93c3c20e46d631899ce1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 15:30:57 +0100 Subject: [PATCH 077/112] Use the new 'inviter' key from im sync for room display names. --- webclient/rooms/rooms-controller.js | 9 +++++++-- webclient/rooms/rooms.html | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js index c25e24c8b..f2ff4a25b 100644 --- a/webclient/rooms/rooms-controller.js +++ b/webclient/rooms/rooms-controller.js @@ -59,7 +59,7 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', // FIXME push membership to top level key to match /im/sync event.membership = event.content.membership; // FIXME bodge a nicer name than the room ID for this invite. - event.room_alias = event.user_id + "'s room"; + event.room_display_name = event.user_id + "'s room"; $scope.rooms[event.room_id] = event; } }); @@ -70,15 +70,20 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', if (alias) { // use the existing alias from storage data[i].room_alias = alias; + data[i].room_display_name = alias; } else if (data[i].aliases && data[i].aliases[0]) { // save the mapping // TODO: select the smarter alias from the array matrixService.createRoomIdToAliasMapping(data[i].room_id, data[i].aliases[0]); + data[i].room_display_name = data[i].aliases[0]; + } + else if (data[i].membership == "invite" && "inviter" in data[i]) { + data[i].room_display_name = data[i].inviter + "'s room" } else { // last resort use the room id - data[i].room_alias = data[i].room_id; + data[i].room_display_name = data[i].room_id; } } return data; diff --git a/webclient/rooms/rooms.html b/webclient/rooms/rooms.html index 2602209bd..ba3b7d8ba 100644 --- a/webclient/rooms/rooms.html +++ b/webclient/rooms/rooms.html @@ -65,7 +65,7 @@
- {{ room.room_alias }} {{room.membership === 'invite' ? ' (invited)' : ''}} + {{ room.room_display_name }} {{room.membership === 'invite' ? ' (invited)' : ''}}

From ad869fa4b30d660bb3307e8bdab26c36c52a7221 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 21 Aug 2014 15:43:47 +0100 Subject: [PATCH 078/112] stop hammering the HS for displayname and avatar URLs --- webclient/room/room-controller.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 8dea64a80..eee805daf 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -160,8 +160,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) if ("mtime_age" in chunk.content) { chunk.mtime_age = chunk.content.mtime_age; } -/* - // FIXME: once the HS reliably returns the displaynames & avatar_urls for both + // Once the HS reliably returns the displaynames & avatar_urls for both // local and remote users, we should use this rather than the evalAsync block // below if ("displayname" in chunk.content) { @@ -170,9 +169,11 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) if ("avatar_url" in chunk.content) { chunk.avatar_url = chunk.content.avatar_url; } - */ $scope.members[chunk.target_user_id] = chunk; +/* + // Stale code for explicitly hammering the homeserver for every displayname & avatar_url + // get their display name and profile picture and set it to their // member entry in $scope.members. We HAVE to use $timeout with 0 delay // to make this function run AFTER the current digest cycle, else the @@ -196,6 +197,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) } ); }); +*/ } else { // selectively update membership else it will nuke the picture and displayname too :/ From e7ee0b9fc113b1fd29b8cb96eea7a00641e56887 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 16:40:21 +0100 Subject: [PATCH 079/112] Change IM sync api to also return the current presence list. --- synapse/handlers/room.py | 24 +++++++++++++++++++++--- synapse/storage/stream.py | 5 ++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d9809bd6d..899b653fb 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -24,6 +24,7 @@ from synapse.api.events.room import ( RoomConfigEvent ) from synapse.api.streams.event import EventStream, EventsStreamData +from synapse.handlers.presence import PresenceStreamData from synapse.util import stringutils from ._base import BaseHandler @@ -257,7 +258,19 @@ class MessageHandler(BaseHandler): membership_list=[Membership.INVITE, Membership.JOIN] ) - ret = [] + rooms_ret = [] + + now_rooms_token = yield self.store.get_room_events_max_id() + + # FIXME (erikj): Fix this. + presence_stream = PresenceStreamData(self.hs) + now_presence_token = yield presence_stream.max_token() + presence = yield presence_stream.get_rows( + user_id, 0, now_presence_token, None, None + ) + + # FIXME (erikj): We need to not generate this token, + now_token = "%s_%s" % (now_rooms_token, now_presence_token) for event in room_list: d = { @@ -268,14 +281,15 @@ class MessageHandler(BaseHandler): if event.membership == Membership.INVITE: d["inviter"] = event.user_id - ret.append(d) + rooms_ret.append(d) if event.membership != Membership.JOIN: continue try: messages, token = yield self.store.get_recent_events_for_room( event.room_id, - limit=50, + limit=10, + end_token=now_rooms_token, ) d["messages"] = { @@ -289,6 +303,10 @@ class MessageHandler(BaseHandler): except: logger.exception("Failed to get snapshot") + user = self.hs.parse_userid(user_id) + + ret = {"rooms": rooms_ret, "presence": presence[0], "end": now_token} + logger.debug("snapshot_all_rooms returning: %s", ret) defer.returnValue(ret) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index e994017bf..8bc502483 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -249,11 +249,10 @@ class StreamStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_recent_events_for_room(self, room_id, limit, with_feedback=False): + def get_recent_events_for_room(self, room_id, limit, end_token, + with_feedback=False): # TODO (erikj): Handle compressed feedback - end_token = yield self.get_room_events_max_id() - sql = ( "SELECT * FROM events " "WHERE room_id = ? AND stream_ordering <= ? " From 7dac1bfc9148e4e23d388d8281aacee2bb41d5db Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 17:17:41 +0100 Subject: [PATCH 080/112] Change webclient to always hit the im sync api before streaming so we get current presence state --- .../components/matrix/event-stream-service.js | 41 ++++++++++++++++--- webclient/rooms/rooms-controller.js | 7 +++- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/webclient/components/matrix/event-stream-service.js b/webclient/components/matrix/event-stream-service.js index a446fad5d..9a8f6eac4 100644 --- a/webclient/components/matrix/event-stream-service.js +++ b/webclient/components/matrix/event-stream-service.js @@ -48,11 +48,12 @@ angular.module('eventStreamService', []) var saveStreamSettings = function() { localStorage.setItem("streamSettings", JSON.stringify(settings)); }; - - var startEventStream = function() { + + var doEventStream = function(deferred) { settings.shouldPoll = true; settings.isActive = true; - var deferred = $q.defer(); + deferred = deferred || $q.defer(); + // run the stream from the latest token matrixService.getEventStream(settings.from, TIMEOUT_MS).then( function(response) { @@ -63,13 +64,16 @@ angular.module('eventStreamService', []) settings.from = response.data.end; - console.log("[EventStream] Got response from "+settings.from+" to "+response.data.end); + console.log( + "[EventStream] Got response from "+settings.from+ + " to "+response.data.end + ); eventHandlerService.handleEvents(response.data.chunk, true); deferred.resolve(response); if (settings.shouldPoll) { - $timeout(startEventStream, 0); + $timeout(doEventStream, 0); } else { console.log("[EventStream] Stopping poll."); @@ -83,13 +87,38 @@ angular.module('eventStreamService', []) deferred.reject(error); if (settings.shouldPoll) { - $timeout(startEventStream, ERR_TIMEOUT_MS); + $timeout(doEventStream, ERR_TIMEOUT_MS); } else { console.log("[EventStream] Stopping polling."); } } ); + + return deferred.promise; + } + + var startEventStream = function() { + settings.shouldPoll = true; + settings.isActive = true; + var deferred = $q.defer(); + + // FIXME: We are discarding all the messages. + matrixService.rooms().then( + function(response) { + var presence = response.data.presence; + for (var i = 0; i < presence.length; ++i) { + eventHandlerService.handleEvent(presence[i], false); + } + + settings.from = response.data.end + doEventStream(deferred); + }, + function(error) { + $scope.feedback = "Failure: " + error.data; + } + ); + return deferred.promise; }; diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js index f2ff4a25b..6bbb2b2ba 100644 --- a/webclient/rooms/rooms-controller.js +++ b/webclient/rooms/rooms-controller.js @@ -93,11 +93,16 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', // List all rooms joined or been invited to matrixService.rooms().then( function(response) { - var data = assignRoomAliases(response.data); + var data = assignRoomAliases(response.data.rooms); $scope.feedback = "Success"; for (var i=0; i Date: Thu, 21 Aug 2014 17:46:52 +0100 Subject: [PATCH 081/112] Add ts field to all events. --- synapse/api/events/factory.py | 7 ++++++- synapse/server.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index b61dac7ac..c2cdcddf4 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -33,16 +33,21 @@ class EventFactory(object): RoomConfigEvent ] - def __init__(self): + def __init__(self, hs): self._event_list = {} # dict of TYPE to event class for event_class in EventFactory._event_classes: self._event_list[event_class.TYPE] = event_class + self.clock = hs.get_clock() + def create_event(self, etype=None, **kwargs): kwargs["type"] = etype if "event_id" not in kwargs: kwargs["event_id"] = random_string(10) + if "ts" not in kwargs: + kwargs["ts"] = int(self.clock.time_msec()) + if etype in self._event_list: handler = self._event_list[etype] else: diff --git a/synapse/server.py b/synapse/server.py index d4c248148..c5b0a3275 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -159,7 +159,7 @@ class HomeServer(BaseHomeServer): return DataStore(self) def build_event_factory(self): - return EventFactory() + return EventFactory(self) def build_handlers(self): return Handlers(self) From 2e1ab9db08e3fe41822a65fdf38feafbd22173b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 17:55:41 +0100 Subject: [PATCH 082/112] Only start event streaming after having set up the controllers. --- demo/start.sh | 3 ++- webclient/app-controller.js | 2 +- webclient/app.js | 2 +- webclient/components/matrix/event-stream-service.js | 10 ++++++++++ webclient/room/room-controller.js | 1 + webclient/rooms/rooms-controller.js | 6 ++++-- 6 files changed, 19 insertions(+), 5 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index 470187292..fa2998a5e 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -15,7 +15,8 @@ for port in "8080" "8081" "8082"; do -f "$DIR/$port.log" \ -d "$DIR/$port.db" \ -vv \ - -D --pid-file "$DIR/$port.pid" + -D --pid-file "$DIR/$port.pid"\ + -w done echo "Starting webclient on port 8000..." diff --git a/webclient/app-controller.js b/webclient/app-controller.js index 96656e12c..c53f29aa7 100644 --- a/webclient/app-controller.js +++ b/webclient/app-controller.js @@ -53,7 +53,7 @@ angular.module('MatrixWebClientController', ['matrixService']) }; if (matrixService.isUserLoggedIn()) { - eventStreamService.resume(); + // eventStreamService.resume(); } // Logs the user out diff --git a/webclient/app.js b/webclient/app.js index f27ebedc6..944b8ec27 100644 --- a/webclient/app.js +++ b/webclient/app.js @@ -80,6 +80,6 @@ matrixWebClient.run(['$location', 'matrixService', 'eventStreamService', functio $location.path("login"); } else { - eventStreamService.resume(); + // eventStreamService.resume(); } }]); diff --git a/webclient/components/matrix/event-stream-service.js b/webclient/components/matrix/event-stream-service.js index 9a8f6eac4..a1a98b2a3 100644 --- a/webclient/components/matrix/event-stream-service.js +++ b/webclient/components/matrix/event-stream-service.js @@ -106,6 +106,16 @@ angular.module('eventStreamService', []) // FIXME: We are discarding all the messages. matrixService.rooms().then( function(response) { + var rooms = response.data.rooms; + for (var i = 0; i < rooms.length; ++i) { + var room = rooms[i]; + if ("state" in room) { + for (var j = 0; j < room.state.length; ++j) { + eventHandlerService.handleEvents(room.state[j], false); + } + } + } + var presence = response.data.presence; for (var i = 0; i < presence.length; ++i) { eventHandlerService.handleEvent(presence[i], false); diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index eee805daf..214166a43 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -328,6 +328,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) var chunk = response.data.chunk[i]; updateMemberList(chunk); } + eventStreamService.resume(); }, function(error) { $scope.feedback = "Failed get member list: " + error.data.error; diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js index 6bbb2b2ba..c2d7bcb6f 100644 --- a/webclient/rooms/rooms-controller.js +++ b/webclient/rooms/rooms-controller.js @@ -17,8 +17,8 @@ limitations under the License. 'use strict'; angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', 'eventHandlerService']) -.controller('RoomsController', ['$scope', '$location', 'matrixService', 'mFileUpload', 'eventHandlerService', - function($scope, $location, matrixService, mFileUpload, eventHandlerService) { +.controller('RoomsController', ['$scope', '$location', 'matrixService', 'mFileUpload', 'eventHandlerService', 'eventStreamService', + function($scope, $location, matrixService, mFileUpload, eventHandlerService, eventStreamService) { $scope.rooms = {}; $scope.public_rooms = []; @@ -113,6 +113,8 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', $scope.public_rooms = assignRoomAliases(response.data.chunk); } ); + + eventStreamService.resume(); }; $scope.createNewRoom = function(room_id, isPrivate) { From 0045a2647ad3e0e088dacbee8497dbbb7d118269 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 17:59:07 +0100 Subject: [PATCH 083/112] Add a var. --- webclient/app-filter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webclient/app-filter.js b/webclient/app-filter.js index 64c3bb04d..f007b4c89 100644 --- a/webclient/app-filter.js +++ b/webclient/app-filter.js @@ -58,7 +58,7 @@ angular.module('matrixWebClient') angular.forEach(displayNames, function(value, key) { if (value.length > 1) { // console.log(key + ": " + value); - for (i=0; i < value.length; i++) { + for (var i=0; i < value.length; i++) { var v = value[i]; members[v].displayname += " (" + v + ")"; // console.log(v + " " + members[v]); From 3277a650529d4ecaf816987e6cbcb87fdf3371da Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 21 Aug 2014 19:02:00 +0100 Subject: [PATCH 084/112] actually display room metadata based on m.room.membe events --- webclient/app.css | 4 ++++ webclient/components/matrix/event-handler-service.js | 11 +++++++++++ webclient/room/room.html | 10 ++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/webclient/app.css b/webclient/app.css index 83b0c9c65..a63b5db4d 100644 --- a/webclient/app.css +++ b/webclient/app.css @@ -245,6 +245,10 @@ h1 { background-color: #fff ! important; } +.mine .membership { + background-color: #fff ! important; +} + .mine .text .bubble { text-align: left ! important; } diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index b8529895f..6a01b3fb5 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -69,6 +69,17 @@ angular.module('eventHandlerService', []) var handleRoomMember = function(event, isLiveEvent) { initRoom(event.room_id); + + // add membership changes as if they were a room message if something interesting changed + if (event.content.prev !== event.content.membership) { + if (isLiveEvent) { + $rootScope.events.rooms[event.room_id].messages.push(event); + } + else { + $rootScope.events.rooms[event.room_id].messages.unshift(event); + } + } + $rootScope.events.rooms[event.room_id].members[event.user_id] = event; $rootScope.$broadcast(MEMBER_EVENT, event, isLiveEvent); }; diff --git a/webclient/room/room.html b/webclient/room/room.html index cb9cf1d1f..4a07dfdaa 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -26,19 +26,25 @@
+ -
{{ members[msg.user_id].displayname || msg.user_id }}
-
{{ msg.content.hsob_ts | date:'MMM d HH:mm:ss' }}
+
{{ (msg.content.hsob_ts || msg.ts) | date:'MMM d HH:mm:ss' }}
 +
+ + {{ members[msg.user_id].displayname || msg.user_id }} + {{ {"join": "joined", "leave": "left", "invite": "invited"}[msg.content.membership] }} + {{ msg.content.target_id || '' }} +
From 1b0d4272853ee2187014536de253e47bd318e198 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 21 Aug 2014 23:35:45 +0100 Subject: [PATCH 085/112] host a webclient by default --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ca102236c..6b39da4a7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -235,8 +235,8 @@ def setup(): parser.add_argument('--pid-file', dest="pid", help="When running as a " "daemon, the file to store the pid in", default="hs.pid") - parser.add_argument("-w", "--webclient", dest="webclient", - action="store_true", help="Host the web client.") + parser.add_argument("-W", "--webclient", dest="webclient", default=True, + action="store_false", help="Don't host a web client.") args = parser.parse_args() verbosity = int(args.verbose) if args.verbose else None From 019f3a66f605222576d4a061df1ecfbaebebf0c0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Aug 2014 01:32:17 +0100 Subject: [PATCH 086/112] add fixme pointing out name disambiguation is a bit flakey --- webclient/app-filter.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webclient/app-filter.js b/webclient/app-filter.js index f007b4c89..b8f4ed25b 100644 --- a/webclient/app-filter.js +++ b/webclient/app-filter.js @@ -54,12 +54,15 @@ angular.module('matrixWebClient') }); // FIXME: we shouldn't disambiguate displayNames on every orderMembersList - // invocation but keep track of duplicates incrementally somewhere + // invocation but keep track of duplicates incrementally somewhere angular.forEach(displayNames, function(value, key) { if (value.length > 1) { // console.log(key + ": " + value); for (var i=0; i < value.length; i++) { var v = value[i]; + // FIXME: this permenantly rewrites the displayname for a given + // room member. which means we can't reset their name if it is + // no longer ambiguous! members[v].displayname += " (" + v + ")"; // console.log(v + " " + members[v]); }; From ab27b49deddbd6f74bad126b9a275b015a7fb6cd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Aug 2014 01:33:05 +0100 Subject: [PATCH 087/112] rename autoComplete directive as tabComplete to avoid confusion with the autocomplete html attribute --- webclient/room/room-directive.js | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/webclient/room/room-directive.js b/webclient/room/room-directive.js index 94655336d..1a99a37ab 100644 --- a/webclient/room/room-directive.js +++ b/webclient/room/room-directive.js @@ -17,30 +17,30 @@ 'use strict'; angular.module('RoomController') -.directive('autoComplete', ['$timeout', function ($timeout) { +.directive('tabComplete', ['$timeout', function ($timeout) { return function (scope, element, attrs) { element.bind("keydown keypress", function (event) { // console.log("event: " + event.which); if (event.which === 9) { - if (!scope.autoCompleting) { // cache our starting text + if (!scope.tabCompleting) { // cache our starting text // console.log("caching " + element[0].value); - scope.autoCompleteOriginal = element[0].value; - scope.autoCompleting = true; + scope.tabCompleteOriginal = element[0].value; + scope.tabCompleting = true; } if (event.shiftKey) { - scope.autoCompleteIndex--; - if (scope.autoCompleteIndex < 0) { - scope.autoCompleteIndex = 0; + scope.tabCompleteIndex--; + if (scope.tabCompleteIndex < 0) { + scope.tabCompleteIndex = 0; } } else { - scope.autoCompleteIndex++; + scope.tabCompleteIndex++; } var searchIndex = 0; - var targetIndex = scope.autoCompleteIndex; - var text = scope.autoCompleteOriginal; + var targetIndex = scope.tabCompleteIndex; + var text = scope.tabCompleteOriginal; // console.log("targetIndex: " + targetIndex + ", text=" + text); @@ -90,17 +90,17 @@ angular.module('RoomController') element[0].className = ""; }, 150); element[0].value = text; - scope.autoCompleteIndex = 0; + scope.tabCompleteIndex = 0; } } else { - scope.autoCompleteIndex = 0; + scope.tabCompleteIndex = 0; } event.preventDefault(); } - else if (event.which !== 16 && scope.autoCompleting) { - scope.autoCompleting = false; - scope.autoCompleteIndex = 0; + else if (event.which !== 16 && scope.tabCompleting) { + scope.tabCompleting = false; + scope.tabCompleteIndex = 0; } }); }; From fd47f55e943dc6950a1a84414e0ed8a08fbc504c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Aug 2014 01:33:34 +0100 Subject: [PATCH 088/112] sacrifice a goat or two to make wordwrap actually work properly --- webclient/app.css | 45 +++++++++++++++++----------------------- webclient/room/room.html | 4 ++-- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/webclient/app.css b/webclient/app.css index a63b5db4d..dfc919e4c 100644 --- a/webclient/app.css +++ b/webclient/app.css @@ -145,6 +145,7 @@ h1 { max-width: 1280px; width: 100%; border-collapse: collapse; + table-layout: fixed; } #messageTable td { @@ -190,25 +191,13 @@ h1 { object-fit: cover; } -.text { - background-color: #eee; - border: 1px solid #d8d8d8; - height: 31px; - display: inline-table; - margin-top: -1px; - max-width: 90%; - font-size: 16px; - /* word-wrap: break-word; */ - word-break: break-all; -} - .emote { - background-color: #fff ! important; + background-color: transparent ! important; border: 0px ! important; } .membership { - background-color: #fff ! important; + background-color: transparent ! important; border: 0px ! important; } @@ -221,6 +210,13 @@ h1 { } .bubble { + background-color: #eee; + border: 1px solid #d8d8d8; + display: inline-block; + margin-bottom: -1px; + max-width: 90%; + font-size: 16px; + word-wrap: break-word; padding-top: 7px; padding-bottom: 5px; padding-left: 1em; @@ -229,27 +225,24 @@ h1 { } .differentUser td { - padding-top: 5px ! important; - margin-top: 5px ! important; + padding-bottom: 5px ! important; } .mine { text-align: right; } -.mine .text { - background-color: #f8f8ff ! important; -} - -.mine .emote { - background-color: #fff ! important; -} - -.mine .membership { - background-color: #fff ! important; +.text.emote .bubble, +.text.membership .bubble, +.mine .text.emote .bubble, +.mine .text.membership .bubble + { + background-color: transparent ! important; + border: 0px ! important; } .mine .text .bubble { + background-color: #f8f8ff ! important; text-align: left ! important; } diff --git a/webclient/room/room.html b/webclient/room/room.html index 4a07dfdaa..e7560a5dc 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -29,7 +29,7 @@ + ng-class="(events.rooms[room_id].messages[$index + 1].user_id !== msg.user_id ? 'differentUser' : '') + (msg.user_id === state.user_id ? ' mine' : '')" scroll-item>
{{ members[msg.user_id].displayname || msg.user_id }}
{{ (msg.content.hsob_ts || msg.ts) | date:'MMM d HH:mm:ss' }}
@@ -77,7 +77,7 @@ {{ state.user_id }} 		- + From 868fa1a1e349d365a41b9594a6fad64fbac36777 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Aug 2014 01:41:38 +0100 Subject: [PATCH 089/112] fix weird fontsizes on iOS --- webclient/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/webclient/app.css b/webclient/app.css index dfc919e4c..da1a840f4 100644 --- a/webclient/app.css +++ b/webclient/app.css @@ -222,6 +222,7 @@ h1 { padding-left: 1em; padding-right: 1em; vertical-align: middle; + -webkit-text-size-adjust:100% } .differentUser td { From 3248aed03b03e0eba3a4b43776ef2f7685b27701 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Aug 2014 01:54:37 +0100 Subject: [PATCH 090/112] fix mainInput retaining focus between sending consecutive messages by disabling commit 955662d6 --- webclient/room/room-controller.js | 4 ++-- webclient/room/room.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 214166a43..451c6242f 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -33,6 +33,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) can_paginate: true, // this is toggled off when we run out of items paginating: false, // used to avoid concurrent pagination requests pulling in dup contents stream_failure: undefined, // the response when the stream fails + // FIXME: sending has been disabled, as surely messages should be sent in the background rather than locking the UI synchronously --Matthew sending: false // true when a message is being sent. It helps to disable the UI when a process is running }; $scope.members = {}; @@ -239,7 +240,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) } $scope.state.sending = true; - + // Send the text message var promise; // FIXME: handle other commands too @@ -263,7 +264,6 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) }; $scope.onInit = function() { - // $timeout(function() { document.getElementById('textInput').focus() }, 0); console.log("onInit"); // Does the room ID provided in the URL? diff --git a/webclient/room/room.html b/webclient/room/room.html index e7560a5dc..95da06771 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -77,10 +77,10 @@ {{ state.user_id }} - + - + From 8d5ceccfc797c60723e84d0b0d3ad6045f694a61 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Aug 2014 02:04:13 +0100 Subject: [PATCH 091/112] -w is no more --- demo/start.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index fa2998a5e..470187292 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -15,8 +15,7 @@ for port in "8080" "8081" "8082"; do -f "$DIR/$port.log" \ -d "$DIR/$port.db" \ -vv \ - -D --pid-file "$DIR/$port.pid"\ - -w + -D --pid-file "$DIR/$port.pid" done echo "Starting webclient on port 8000..." From 8f7fbc1bb0d9ace628cdf4fa824e96b5d4d30c13 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Aug 2014 02:11:33 +0100 Subject: [PATCH 092/112] improve leftBlock css --- webclient/app.css | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/webclient/app.css b/webclient/app.css index da1a840f4..207f35f5f 100644 --- a/webclient/app.css +++ b/webclient/app.css @@ -153,7 +153,8 @@ h1 { } .leftBlock { - width: 10em; + width: 14em; + word-wrap: break-word; vertical-align: top; background-color: #fff; color: #888; @@ -209,6 +210,10 @@ h1 { height: auto; } +.text { + vertical-align: top; +} + .bubble { background-color: #eee; border: 1px solid #d8d8d8; From be2f948da512a2f62c49635f24033892e39d359e Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 22 Aug 2014 02:23:59 +0100 Subject: [PATCH 093/112] homeserver runs webclient by default now --- README.rst | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 378b460d0..50440f57f 100644 --- a/README.rst +++ b/README.rst @@ -24,11 +24,8 @@ To get up and running: - To run your own **private** homeserver on localhost:8080, install synapse with ``python setup.py develop --user`` and then run one with - ``python synapse/app/homeserver.py`` - - - To run your own webclient, add ``-w``: - ``python synapse/app/homeserver.py -w`` and hit http://localhost:8080/matrix/client - in your web browser (a recent Chrome, Safari or Firefox for now, + ``python synapse/app/homeserver.py`` - you will find a webclient running + at http://localhost:8080 (use a recent Chrome, Safari or Firefox for now, please...) - To make the homeserver **public** and let it exchange messages with @@ -201,9 +198,7 @@ http://localhost:8080. Simply run:: Running The Demo Web Client =========================== -You can run the web client when you run the homeserver by adding ``-w`` to the -command to run ``homeserver.py``. The web client can be accessed via -http://localhost:8080/matrix/client +The homeserver runs a web client by default at http://localhost:8080. If this is the first time you have used the client from that browser (it uses HTML5 local storage to remember its config), you will need to log in to your From c8d0c4762da432aafa4372928aa70ef55646134b Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Fri, 22 Aug 2014 10:15:15 +0200 Subject: [PATCH 094/112] Safari needs the img.onload event before actually working on the img --- .../fileUpload/file-upload-service.js | 1 + .../components/utilities/utilities-service.js | 65 +++++++++++-------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/webclient/components/fileUpload/file-upload-service.js b/webclient/components/fileUpload/file-upload-service.js index 6606f31e2..398124fcc 100644 --- a/webclient/components/fileUpload/file-upload-service.js +++ b/webclient/components/fileUpload/file-upload-service.js @@ -82,6 +82,7 @@ angular.module('mFileUpload', ['matrixService', 'mUtilities']) // First, get the image size mUtilities.getImageSize(imageFile).then( function(size) { + console.log("image size: " + JSON.stringify(size)); // The final operation: send imageFile var uploadImage = function() { diff --git a/webclient/components/utilities/utilities-service.js b/webclient/components/utilities/utilities-service.js index 9cf858ef3..5e9f70722 100644 --- a/webclient/components/utilities/utilities-service.js +++ b/webclient/components/utilities/utilities-service.js @@ -38,10 +38,15 @@ angular.module('mUtilities', []) img.src = e.target.result; // Once ready, returns its size - deferred.resolve({ - width: img.width, - height: img.height - }); + img.onload = function() { + deferred.resolve({ + width: img.width, + height: img.height + }); + }; + img.onerror = function(e) { + deferred.reject(e); + }; }; reader.onerror = function(e) { deferred.reject(e); @@ -71,33 +76,39 @@ angular.module('mUtilities', []) reader.onload = function(e) { img.src = e.target.result; + + // Once ready, returns its size + img.onload = function() { + var ctx = canvas.getContext("2d"); + ctx.drawImage(img, 0, 0); - var ctx = canvas.getContext("2d"); - ctx.drawImage(img, 0, 0); + var MAX_WIDTH = maxSize; + var MAX_HEIGHT = maxSize; + var width = img.width; + var height = img.height; - var MAX_WIDTH = maxSize; - var MAX_HEIGHT = maxSize; - var width = img.width; - var height = img.height; - - if (width > height) { - if (width > MAX_WIDTH) { - height *= MAX_WIDTH / width; - width = MAX_WIDTH; + if (width > height) { + if (width > MAX_WIDTH) { + height *= MAX_WIDTH / width; + width = MAX_WIDTH; + } + } else { + if (height > MAX_HEIGHT) { + width *= MAX_HEIGHT / height; + height = MAX_HEIGHT; + } } - } else { - if (height > MAX_HEIGHT) { - width *= MAX_HEIGHT / height; - height = MAX_HEIGHT; - } - } - canvas.width = width; - canvas.height = height; - var ctx = canvas.getContext("2d"); - ctx.drawImage(img, 0, 0, width, height); + canvas.width = width; + canvas.height = height; + var ctx = canvas.getContext("2d"); + ctx.drawImage(img, 0, 0, width, height); - var dataUrl = canvas.toDataURL("image/jpeg", 0.7); - deferred.resolve(self.dataURItoBlob(dataUrl)); + var dataUrl = canvas.toDataURL("image/jpeg", 0.7); + deferred.resolve(self.dataURItoBlob(dataUrl)); + }; + img.onerror = function(e) { + deferred.reject(e); + }; }; reader.onerror = function(e) { deferred.reject(e); From 53f4fbd99a223a4321377ea670d1d19b669b6f4a Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Fri, 22 Aug 2014 10:48:00 +0200 Subject: [PATCH 095/112] resizeImage: generate an image in the format of the original image. (Tested with tranparent PNG, transparent GIF, BMP, JPEG) --- webclient/components/utilities/utilities-service.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webclient/components/utilities/utilities-service.js b/webclient/components/utilities/utilities-service.js index 5e9f70722..3df2f0445 100644 --- a/webclient/components/utilities/utilities-service.js +++ b/webclient/components/utilities/utilities-service.js @@ -103,7 +103,9 @@ angular.module('mUtilities', []) var ctx = canvas.getContext("2d"); ctx.drawImage(img, 0, 0, width, height); - var dataUrl = canvas.toDataURL("image/jpeg", 0.7); + // Extract image data in the same format as the original one. + // The 0.7 compression value will work with formats that supports it like JPEG. + var dataUrl = canvas.toDataURL(imageFile.type, 0.7); deferred.resolve(self.dataURItoBlob(dataUrl)); }; img.onerror = function(e) { From acf51276042cf438cbb02bb5ef31c42206d7685d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 10:25:27 +0100 Subject: [PATCH 096/112] Make the content repo work with in daemon mode. Return the full url on upload. Update the webclient to use new content repo api. --- synapse/app/homeserver.py | 5 ++-- synapse/http/server.py | 26 ++++++++++++++----- .../fileUpload/file-upload-service.js | 2 +- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6b39da4a7..495149466 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -56,7 +56,7 @@ class SynapseHomeServer(HomeServer): return File("webclient") # TODO configurable? def build_resource_for_content_repo(self): - return ContentRepoResource("uploads", self.auth) + return ContentRepoResource(self, self.upload_dir, self.auth) def build_db_pool(self): """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we @@ -257,7 +257,8 @@ def setup(): hs = SynapseHomeServer( args.host, - db_name=db_name + upload_dir=os.path.abspath("uploads"), + db_name=db_name, ) # This object doesn't need to be saved because it's set as the handler for diff --git a/synapse/http/server.py b/synapse/http/server.py index c28d9a33f..d1f99460c 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -212,8 +212,9 @@ class ContentRepoResource(resource.Resource): """ isLeaf = True - def __init__(self, directory, auth): + def __init__(self, hs, directory, auth): resource.Resource.__init__(self) + self.hs = hs self.directory = directory self.auth = auth @@ -250,7 +251,8 @@ class ContentRepoResource(resource.Resource): file_ext = re.sub("[^a-z]", "", file_ext) suffix += "." + file_ext - file_path = os.path.join(self.directory, prefix + main_part + suffix) + file_name = prefix + main_part + suffix + file_path = os.path.join(self.directory, file_name) logger.info("User %s is uploading a file to path %s", auth_user.to_string(), file_path) @@ -259,8 +261,8 @@ class ContentRepoResource(resource.Resource): attempts = 0 while os.path.exists(file_path): main_part = random_string(24) - file_path = os.path.join(self.directory, - prefix + main_part + suffix) + file_name = prefix + main_part + suffix + file_path = os.path.join(self.directory, file_name) attempts += 1 if attempts > 25: # really? Really? raise SynapseError(500, "Unable to create file.") @@ -272,11 +274,14 @@ class ContentRepoResource(resource.Resource): # servers. # TODO: A little crude here, we could do this better. - filename = request.path.split(self.directory + "/")[1] + filename = request.path.split('/')[-1] # be paranoid filename = re.sub("[^0-9A-z.-_]", "", filename) file_path = self.directory + "/" + filename + + logger.debug("Searching for %s", file_path) + if os.path.isfile(file_path): # filename has the content type base64_contentype = filename.split(".")[1] @@ -304,6 +309,10 @@ class ContentRepoResource(resource.Resource): self._async_render(request) return server.NOT_DONE_YET + def render_OPTIONS(self, request): + respond_with_json_bytes(request, 200, {}, send_cors=True) + return server.NOT_DONE_YET + @defer.inlineCallbacks def _async_render(self, request): try: @@ -313,8 +322,13 @@ class ContentRepoResource(resource.Resource): with open(fname, "wb") as f: f.write(request.content.read()) + + # FIXME (erikj): These should use constants. + file_name = os.path.basename(fname) + url = "http://%s/matrix/content/%s" % (self.hs.hostname, file_name) + respond_with_json_bytes(request, 200, - json.dumps({"content_token": fname}), + json.dumps({"content_token": url}), send_cors=True) except CodeMessageException as e: diff --git a/webclient/components/fileUpload/file-upload-service.js b/webclient/components/fileUpload/file-upload-service.js index 398124fcc..5f01478fd 100644 --- a/webclient/components/fileUpload/file-upload-service.js +++ b/webclient/components/fileUpload/file-upload-service.js @@ -33,7 +33,7 @@ angular.module('mFileUpload', ['matrixService', 'mUtilities']) console.log("Uploading " + file.name + "... to /matrix/content"); matrixService.uploadContent(file).then( function(response) { - var content_url = location.origin + "/matrix/content/" + response.data.content_token; + var content_url = response.data.content_token; console.log(" -> Successfully uploaded! Available at " + content_url); deferred.resolve(content_url); }, From 3c349b408b302b21bf2ad0d9086fc3b6fb46dc7a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Aug 2014 11:34:27 +0200 Subject: [PATCH 097/112] Update web client to use new IS API. --- webclient/components/matrix/matrix-service.js | 18 ++++-- webclient/login/login-controller.js | 2 + webclient/rooms/rooms-controller.js | 62 +++++++++++++------ 3 files changed, 58 insertions(+), 24 deletions(-) diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index fa5a6091d..d5738e01c 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -79,7 +79,6 @@ angular.module('matrixService', []) return $http(request); }; - return { /****** Home server API ******/ prefix: prefixPath, @@ -310,17 +309,25 @@ angular.module('matrixService', []) }, // hit the Identity Server for a 3PID request. - linkEmail: function(email) { + linkEmail: function(email, clientSecret, sendAttempt) { var path = "/matrix/identity/api/v1/validate/email/requestToken" - var data = "clientSecret=abc123&email=" + encodeURIComponent(email); + var data = "clientSecret="+clientSecret+"&email=" + encodeURIComponent(email)+"&sendAttempt="+sendAttempt; var headers = {}; headers["Content-Type"] = "application/x-www-form-urlencoded"; return doBaseRequest(config.identityServer, "POST", path, {}, data, headers); }, - authEmail: function(userId, tokenId, code) { + authEmail: function(clientSecret, tokenId, code) { var path = "/matrix/identity/api/v1/validate/email/submitToken"; - var data = "token="+code+"&mxId="+encodeURIComponent(userId)+"&tokenId="+tokenId; + var data = "token="+code+"&sid="+tokenId+"&clientSecret="+clientSecret; + var headers = {}; + headers["Content-Type"] = "application/x-www-form-urlencoded"; + return doBaseRequest(config.identityServer, "POST", path, {}, data, headers); + }, + + bindEmail: function(userId, tokenId, clientSecret) { + var path = "/matrix/identity/api/v1/3pid/bind"; + var data = "mxid="+encodeURIComponent(userId)+"&sid="+tokenId+"&clientSecret="+clientSecret; var headers = {}; headers["Content-Type"] = "application/x-www-form-urlencoded"; return doBaseRequest(config.identityServer, "POST", path, {}, data, headers); @@ -393,6 +400,7 @@ angular.module('matrixService', []) // Set a new config (Use saveConfig to actually store it permanently) setConfig: function(newConfig) { config = newConfig; + console.log("new IS: "+config.identityServer); }, // Commits config into permanent storage diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js index 67d0b7b90..35886c558 100644 --- a/webclient/login/login-controller.js +++ b/webclient/login/login-controller.js @@ -70,6 +70,7 @@ angular.module('LoginController', ['matrixService']) $scope.login = function() { matrixService.setConfig({ homeserver: $scope.account.homeserver, + identityServer: $scope.account.identityServer, user_id: $scope.account.user_id }); // try to login @@ -79,6 +80,7 @@ angular.module('LoginController', ['matrixService']) $scope.feedback = "Login successful."; matrixService.setConfig({ homeserver: $scope.account.homeserver, + identityServer: $scope.account.identityServer, user_id: response.data.user_id, access_token: response.data.access_token }); diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js index c25e24c8b..65d345d7a 100644 --- a/webclient/rooms/rooms-controller.js +++ b/webclient/rooms/rooms-controller.js @@ -48,6 +48,8 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', linkNewEmail: "", // the email entry box emailBeingAuthed: undefined, // to populate verification text authTokenId: undefined, // the token id from the IS + clientSecret: undefined, // our client secret + sendAttempt: 1, emailCode: "", // the code entry box linkedEmailList: matrixService.config().emailList // linked email list }; @@ -207,11 +209,27 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', ); }; + var generateClientSecret = function() { + var ret = ""; + var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + for (var i = 0; i < 32; i++) { + ret += chars.charAt(Math.floor(Math.random() * chars.length)); + } + + return ret; + }; + + $scope.linkEmail = function(email) { - matrixService.linkEmail(email).then( + if (email != $scope.linkedEmails.emailBeingAuthed) { + $scope.linkedEmails.clientSecret = generateClientSecret(); + $scope.linkedEmails.sendAttempt = 1; + } + matrixService.linkEmail(email, $scope.linkedEmails.clientSecret, $scope.linkedEmails.sendAttempt).then( function(response) { if (response.data.success === true) { - $scope.linkedEmails.authTokenId = response.data.tokenId; + $scope.linkedEmails.authTokenId = response.data.sid; $scope.emailFeedback = "You have been sent an email."; $scope.linkedEmails.emailBeingAuthed = email; } @@ -231,28 +249,34 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', $scope.emailFeedback = "You have not requested a code with this email."; return; } - matrixService.authEmail(matrixService.config().user_id, tokenId, code).then( + matrixService.authEmail(matrixService.config().user_id, tokenId, code, $scope.linkedEmails.clientSecret).then( function(response) { if ("success" in response.data && response.data.success === false) { $scope.emailFeedback = "Failed to authenticate email."; return; } - var config = matrixService.config(); - var emailList = {}; - if ("emailList" in config) { - emailList = config.emailList; - } - emailList[response.address] = response; - // save the new email list - config.emailList = emailList; - matrixService.setConfig(config); - matrixService.saveConfig(); - // invalidate the email being authed and update UI. - $scope.linkedEmails.emailBeingAuthed = undefined; - $scope.emailFeedback = ""; - $scope.linkedEmails.linkedEmailList = emailList; - $scope.linkedEmails.linkNewEmail = ""; - $scope.linkedEmails.emailCode = ""; + matrixService.bindEmail(matrixService.config().user_id, tokenId, $scope.linkedEmails.clientSecret).then( + function(response) { + var config = matrixService.config(); + var emailList = {}; + if ("emailList" in config) { + emailList = config.emailList; + } + emailList[$scope.linkedEmails.emailBeingAuthed] = response; + // save the new email list + config.emailList = emailList; + matrixService.setConfig(config); + matrixService.saveConfig(); + // invalidate the email being authed and update UI. + $scope.linkedEmails.emailBeingAuthed = undefined; + $scope.emailFeedback = ""; + $scope.linkedEmails.linkedEmailList = emailList; + $scope.linkedEmails.linkNewEmail = ""; + $scope.linkedEmails.emailCode = ""; + }, function(reason) { + $scope.emailFeedback = "Failed to link email: " + reason; + } + ); }, function(reason) { $scope.emailFeedback = "Failed to auth email: " + reason; From dde50d4245136cdbd11ac3b4af42102945cd14f9 Mon Sep 17 00:00:00 2001 From: Emmanuel ROHEE Date: Fri, 22 Aug 2014 11:43:54 +0200 Subject: [PATCH 098/112] Use $location.url instead of $location.path to get clean page URL without hash arguments of the previous page. This happpens with room URL like http://127.0.0.1:8080/matrix/client/#/room/#public:localhost. The second hash part is transferred to the next page when using $location.path. --- webclient/app-controller.js | 2 +- webclient/login/login-controller.js | 4 ++-- webclient/room/room-controller.js | 6 +++--- webclient/rooms/rooms-controller.js | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/webclient/app-controller.js b/webclient/app-controller.js index c53f29aa7..92ad01e4f 100644 --- a/webclient/app-controller.js +++ b/webclient/app-controller.js @@ -66,7 +66,7 @@ angular.module('MatrixWebClientController', ['matrixService']) matrixService.saveConfig(); // And go to the login page - $location.path("login"); + $location.url("login"); }; // Listen to the event indicating that the access token is no longer valid. diff --git a/webclient/login/login-controller.js b/webclient/login/login-controller.js index 67d0b7b90..2f1f224a9 100644 --- a/webclient/login/login-controller.js +++ b/webclient/login/login-controller.js @@ -53,7 +53,7 @@ angular.module('LoginController', ['matrixService']) matrixService.saveConfig(); eventStreamService.resume(); // Go to the user's rooms list page - $location.path("rooms"); + $location.url("rooms"); }, function(error) { if (error.data) { @@ -84,7 +84,7 @@ angular.module('LoginController', ['matrixService']) }); matrixService.saveConfig(); eventStreamService.resume(); - $location.path("rooms"); + $location.url("rooms"); } else { $scope.feedback = "Failed to login: " + JSON.stringify(response.data); diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 451c6242f..26d1836fc 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -293,7 +293,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) else { // In case of issue, go to the default page console.log("Error: cannot extract room alias"); - $location.path("/"); + $location.url("/"); return; } } @@ -310,7 +310,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) function () { // In case of issue, go to the default page console.log("Error: cannot resolve room alias"); - $location.path("/"); + $location.url("/"); }); } }; @@ -364,7 +364,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) matrixService.leave($scope.room_id).then( function(response) { console.log("Left room "); - $location.path("rooms"); + $location.url("rooms"); }, function(error) { $scope.feedback = "Failed to leave room: " + error.data.error; diff --git a/webclient/rooms/rooms-controller.js b/webclient/rooms/rooms-controller.js index c2d7bcb6f..557fbe237 100644 --- a/webclient/rooms/rooms-controller.js +++ b/webclient/rooms/rooms-controller.js @@ -141,17 +141,17 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', // Go to a room $scope.goToRoom = function(room_id) { // Simply open the room page on this room id - //$location.path("room/" + room_id); + //$location.url("room/" + room_id); matrixService.join(room_id).then( function(response) { if (response.data.hasOwnProperty("room_id")) { if (response.data.room_id != room_id) { - $location.path("room/" + response.data.room_id); + $location.url("room/" + response.data.room_id); return; } } - $location.path("room/" + room_id); + $location.url("room/" + room_id); }, function(error) { $scope.feedback = "Can't join room: " + error.data; @@ -163,7 +163,7 @@ angular.module('RoomsController', ['matrixService', 'mFileInput', 'mFileUpload', matrixService.joinAlias(room_alias).then( function(response) { // Go to this room - $location.path("room/" + room_alias); + $location.url("room/" + room_alias); }, function(error) { $scope.feedback = "Can't join room: " + error.data; From 74c90f78159e8067b25bfa8a009d2e68419947c8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 10:50:10 +0100 Subject: [PATCH 099/112] Reinitialize room when creating a RoomController so that we start off with a clean slate, as it expects/ --- webclient/components/matrix/event-handler-service.js | 12 +++++++++++- webclient/room/room-controller.js | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 6a01b3fb5..aa8867425 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -44,6 +44,12 @@ angular.module('eventHandlerService', []) $rootScope.events.rooms[room_id].members = {}; } } + + var reInitRoom = function(room_id) { + $rootScope.events.rooms[room_id] = {}; + $rootScope.events.rooms[room_id].messages = []; + $rootScope.events.rooms[room_id].members = {}; + } var handleMessage = function(event, isLiveEvent) { if ("membership_target" in event.content) { @@ -118,6 +124,10 @@ angular.module('eventHandlerService', []) for (var i=0; i Date: Fri, 22 Aug 2014 10:50:38 +0100 Subject: [PATCH 100/112] Keep track of people's presence and query that when we update the members list. --- webclient/components/matrix/event-handler-service.js | 3 +++ webclient/room/room-controller.js | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index aa8867425..b5eb73d92 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -35,6 +35,8 @@ angular.module('eventHandlerService', []) $rootScope.events = { rooms: {}, // will contain roomId: { messages:[], members:{userid1: event} } }; + + $rootScope.presence = {}; var initRoom = function(room_id) { if (!(room_id in $rootScope.events.rooms)) { @@ -91,6 +93,7 @@ angular.module('eventHandlerService', []) }; var handlePresence = function(event, isLiveEvent) { + $rootScope.presence[event.content.user_id] = event; $rootScope.$broadcast(PRESENCE_EVENT, event, isLiveEvent); }; diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index a0485e84e..e204a27e0 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -15,8 +15,8 @@ limitations under the License. */ angular.module('RoomController', ['ngSanitize', 'mUtilities']) -.controller('RoomController', ['$scope', '$http', '$timeout', '$routeParams', '$location', 'matrixService', 'eventStreamService', 'eventHandlerService', 'mFileUpload', 'mUtilities', - function($scope, $http, $timeout, $routeParams, $location, matrixService, eventStreamService, eventHandlerService, mFileUpload, mUtilities) { +.controller('RoomController', ['$scope', '$http', '$timeout', '$routeParams', '$location', 'matrixService', 'eventStreamService', 'eventHandlerService', 'mFileUpload', 'mUtilities', '$rootScope', + function($scope, $http, $timeout, $routeParams, $location, matrixService, eventStreamService, eventHandlerService, mFileUpload, mUtilities, $rootScope) { 'use strict'; var MESSAGES_PER_PAGINATION = 30; var THUMBNAIL_SIZE = 320; @@ -199,6 +199,10 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) ); }); */ + + if (chunk.target_user_id in $rootScope.presence) { + updatePresence($rootScope.presence[chunk.target_user_id]); + } } else { // selectively update membership else it will nuke the picture and displayname too :/ @@ -265,7 +269,7 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) $scope.onInit = function() { console.log("onInit"); - + // Does the room ID provided in the URL? var room_id_or_alias; if ($routeParams.room_id_or_alias) { From 47a4bff139e3d2b444b196845c4926d7c68a230b Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 22 Aug 2014 11:55:37 +0200 Subject: [PATCH 101/112] Updater command line client to new IS API --- cmdclient/console.py | 62 ++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/cmdclient/console.py b/cmdclient/console.py index 6c6e2085b..64557a4c4 100755 --- a/cmdclient/console.py +++ b/cmdclient/console.py @@ -233,56 +233,68 @@ class SynapseCmd(cmd.Cmd): defer.returnValue(False) defer.returnValue(True) - def do_3pidrequest(self, line): + def do_emailrequest(self, line): """Requests the association of a third party identifier - The medium of the identifer (currently only 'email') -
The address of the identifer (ie. the email address) +
The email address) + A string of characters generated when requesting an email that you'll supply in subsequent calls to identify yourself + The number of times the user has requested an email. Leave this the same between requests to retry the request at the transport level. Increment it to request that the email be sent again. """ - args = self._parse(line, ['medium', 'address']) + args = self._parse(line, ['address', 'clientSecret', 'sendAttempt']) - if not args['medium'] == 'email': - print "Only email is supported currently" - return + postArgs = {'email': args['address'], 'clientSecret': args['clientSecret'], 'sendAttempt': args['sendAttempt']} - postArgs = {'email': args['address'], 'clientSecret': '____'} - - reactor.callFromThread(self._do_3pidrequest, postArgs) + reactor.callFromThread(self._do_emailrequest, postArgs) @defer.inlineCallbacks - def _do_3pidrequest(self, args): + def _do_emailrequest(self, args): url = self._identityServerUrl()+"/matrix/identity/api/v1/validate/email/requestToken" json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False, headers={'Content-Type': ['application/x-www-form-urlencoded']}) print json_res - if 'tokenId' in json_res: - print "Token ID %s sent" % (json_res['tokenId']) + if 'sid' in json_res: + print "Token sent. Your session ID is %s" % (json_res['sid']) - def do_3pidvalidate(self, line): + def do_emailvalidate(self, line): """Validate and associate a third party ID - The medium of the identifer (currently only 'email') - The identifier iof the token given in 3pidrequest + The session ID (sid) given to you in the response to requestToken The token sent to your third party identifier address + The same clientSecret you supplied in requestToken """ - args = self._parse(line, ['medium', 'tokenId', 'token']) + args = self._parse(line, ['sid', 'token', 'clientSecret']) - if not args['medium'] == 'email': - print "Only email is supported currently" - return + postArgs = { 'sid' : args['sid'], 'token' : args['token'], 'clientSecret': args['clientSecret'] } - postArgs = { 'tokenId' : args['tokenId'], 'token' : args['token'] } - postArgs['mxId'] = self.config["user"] - - reactor.callFromThread(self._do_3pidvalidate, postArgs) + reactor.callFromThread(self._do_emailvalidate, postArgs) @defer.inlineCallbacks - def _do_3pidvalidate(self, args): + def _do_emailvalidate(self, args): url = self._identityServerUrl()+"/matrix/identity/api/v1/validate/email/submitToken" json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False, headers={'Content-Type': ['application/x-www-form-urlencoded']}) print json_res + def do_3pidbind(self, line): + """Validate and associate a third party ID + The session ID (sid) given to you in the response to requestToken + The same clientSecret you supplied in requestToken + """ + args = self._parse(line, ['sid', 'clientSecret']) + + postArgs = { 'sid' : args['sid'], 'clientSecret': args['clientSecret'] } + postArgs['mxid'] = self.config["user"] + + reactor.callFromThread(self._do_3pidbind, postArgs) + + @defer.inlineCallbacks + def _do_3pidbind(self, args): + url = self._identityServerUrl()+"/matrix/identity/api/v1/3pid/bind" + + json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False, + headers={'Content-Type': ['application/x-www-form-urlencoded']}) + print json_res + def do_join(self, line): """Joins a room: "join " """ try: From f3cea238b9c51861965d31cd9352153338d6705b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 10:56:09 +0100 Subject: [PATCH 102/112] Check if the membership message was for the room we were in before updating the membership list --- webclient/room/room-controller.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index e204a27e0..58ba432ce 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -152,6 +152,8 @@ angular.module('RoomController', ['ngSanitize', 'mUtilities']) }; var updateMemberList = function(chunk) { + if (chunk.room_id != $scope.room_id) return; + var isNewMember = !(chunk.target_user_id in $scope.members); if (isNewMember) { // FIXME: why are we copying these fields around inside chunk? From c7d7bc02543eb0d2b794d8d70b23e27616384222 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 12:06:27 +0100 Subject: [PATCH 103/112] Allow people to specify database location in database-save.sh --- database-save.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database-save.sh b/database-save.sh index c80f676f7..040c8a494 100755 --- a/database-save.sh +++ b/database-save.sh @@ -8,7 +8,7 @@ # # $ sqlite3 homeserver.db < table-save.sql -sqlite3 homeserver.db <<'EOF' >table-save.sql +sqlite3 "$1" <<'EOF' >table-save.sql .dump users .dump access_tokens .dump presence From c2e983b8db466a8f456c9a22d4438dec5060490d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 12:06:50 +0100 Subject: [PATCH 104/112] Bump versions to 0.0.1 --- VERSION | 1 + setup.py | 2 +- synapse/__init__.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 VERSION diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..8acdd82b7 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/setup.py b/setup.py index fca3c7770..f01eec436 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def read(fname): setup( name="SynapseHomeServer", - version="0.1", + version="0.0.1", packages=find_packages(exclude=["tests"]), description="Reference Synapse Home Server", install_requires=[ diff --git a/synapse/__init__.py b/synapse/__init__.py index 1e7b2ab27..47fc1b2ea 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -15,3 +15,5 @@ """ This is a reference implementation of a synapse home server. """ + +__version__ = "0.0.1" From 5494815c701f806d012a60d5117519e4d2ef1b39 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 12:18:05 +0100 Subject: [PATCH 105/112] Add database-prepare-for-0.0.1.sh that should be run before starting a v0.0.1 homeserver. --- database-prepare-for-0.0.1.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 database-prepare-for-0.0.1.sh diff --git a/database-prepare-for-0.0.1.sh b/database-prepare-for-0.0.1.sh new file mode 100755 index 000000000..17c0c5f34 --- /dev/null +++ b/database-prepare-for-0.0.1.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# This is will prepare a synapse database for running with v0.0.1 of synapse. +# It will store all the user information, but will *delete* all messages and +# room data. + +cp "$1" "$1.bak" + +DUMP=$(sqlite3 "$1" << 'EOF' +.dump users +.dump access_tokens +.dump presence +.dump profiles +EOF +) + +rm "$1" + +sqlite3 "$1" <<< "$DUMP" From 1317afcb9a457a9b60dde95dc5d9aea9b9d80789 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 12:22:38 +0100 Subject: [PATCH 106/112] Add a database-prepare-for-0.0.1.sh --- database-prepare-for-0.0.1.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/database-prepare-for-0.0.1.sh b/database-prepare-for-0.0.1.sh index 17c0c5f34..43d759a5c 100755 --- a/database-prepare-for-0.0.1.sh +++ b/database-prepare-for-0.0.1.sh @@ -4,6 +4,8 @@ # It will store all the user information, but will *delete* all messages and # room data. +set -e + cp "$1" "$1.bak" DUMP=$(sqlite3 "$1" << 'EOF' From 808f663ed179dcb16a315810f7f0f8a7eec77e01 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 13:06:07 +0100 Subject: [PATCH 107/112] Don't return state event outlier's when paginating. --- synapse/storage/__init__.py | 7 ++++++- synapse/storage/schema/im.sql | 1 + synapse/storage/stream.py | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 773290692..d06033b98 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -105,6 +105,11 @@ class DataStore(RoomMemberStore, RoomStore, "processed": True, } + if hasattr(event, "outlier"): + vals["outlier"] = event.outlier + else: + vals["outlier"] = False + if backfilled: if not self.min_token_deferred.called: yield self.min_token_deferred @@ -123,7 +128,7 @@ class DataStore(RoomMemberStore, RoomStore, except: logger.exception( "Failed to persist, probably duplicate: %s", - event_id + event.event_id ) return diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index ea04261ff..39a1ed703 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS events( content TEXT NOT NULL, unrecognized_keys TEXT, processed BOOL NOT NULL, + outlier BOOL NOT NULL, CONSTRAINT ev_uniq UNIQUE (event_id) ); diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8bc502483..87ae961cc 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -177,6 +177,7 @@ class StreamStore(SQLBaseStore): "((room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering < ? " + "AND e.outlier = 0 " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "current": current_room_membership_sql, @@ -224,7 +225,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events " - "WHERE room_id = ? AND %(bounds)s " + "WHERE outlier = 0 AND room_id = ? AND %(bounds)s " "ORDER BY topological_ordering %(order)s, stream_ordering %(order)s %(limit)s " ) % {"bounds": bounds, "order": order, "limit": limit_str} From 87b315ce21deca1db67917acd8b40a20ccdd8c2c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 13:52:38 +0100 Subject: [PATCH 108/112] Add CHANGES and UPGRADE files. --- CHANGES | 20 ++++++++++++++++++++ UPGRADE | 23 +++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 CHANGES create mode 100644 UPGRADE diff --git a/CHANGES b/CHANGES new file mode 100644 index 000000000..055f8bc01 --- /dev/null +++ b/CHANGES @@ -0,0 +1,20 @@ +Changes in synapse 0.0.1 +======================= +Homeserver: + * Completely change the database schema to support generic event types. + * Improve presence reliability. + * Improve reliability of joining remote rooms. + * Fix bug where room join events were duplicated. + * Improve initial sync API to return more information to the client. + * Stop generating fake messages for room membership events. + +Webclient: + * Add tab completion of names. + * Add ability to upload and send images. + * Add profile pages. + * Improve CSS layout of room. + * Disambiguate identical display names. + * Don't get remote users display names and avatars individually. + * Use the new initial sync API to reduce number of round trips to the homeserver. + * Change url scheme to use room aliases instead of room ids where known. + * Increase longpoll timeout. diff --git a/UPGRADE b/UPGRADE new file mode 100644 index 000000000..95f933597 --- /dev/null +++ b/UPGRADE @@ -0,0 +1,23 @@ +Upgrading to v0.0.1 +================== +This release completely changes the database schema and so requires upgrading +it before starting the new version of the homeserver. + +The script "database-prepare-for-0.0.1.sh" should be used to upgrade the +database. This will save all user information, such as logins and profiles, +but will otherwise purge the database. This includes messages, which +rooms the home server was a member of and room alias mappings. + +Before running the command the homeserver should be first completely +shutdown. To run it, simply specify the location of the database, e.g.: + + ./database-prepare-for-0.0.1.sh "homeserver.db" + +Once this has successfully completed it will be safe to restart the +homeserver. You may notice that the homeserver takes a few seconds longer to +restart than usual as it reinitializes the database. + +On startup of the new version, users can either rejoin remote rooms using room +aliases or by being reinvited. Alternatively, if any other homeserver sends a +message to a room that the homeserver was previously in the local HS will +automatically rejoin the room. From 7d3a841a83d7e6b84e2972a4a5d4e8e41cfa738d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 15:09:03 +0100 Subject: [PATCH 109/112] Add a missing '=' --- UPGRADE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/UPGRADE b/UPGRADE index 95f933597..2e75d77bc 100644 --- a/UPGRADE +++ b/UPGRADE @@ -1,5 +1,6 @@ Upgrading to v0.0.1 -================== +=================== + This release completely changes the database schema and so requires upgrading it before starting the new version of the homeserver. From a0e114fe648f2c74b56bdb82687b3c86c86d34d4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 15:20:33 +0100 Subject: [PATCH 110/112] Rename files to .rst for consistency. --- CHANGES => CHANGES.rst | 0 UPGRADE => UPGRADE.rst | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename CHANGES => CHANGES.rst (100%) rename UPGRADE => UPGRADE.rst (100%) diff --git a/CHANGES b/CHANGES.rst similarity index 100% rename from CHANGES rename to CHANGES.rst diff --git a/UPGRADE b/UPGRADE.rst similarity index 100% rename from UPGRADE rename to UPGRADE.rst From f81692dab4695afb00e1197b69b87974d22ecefc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 15:20:53 +0100 Subject: [PATCH 111/112] Update the README.rst to refer people to UPGRADE.rst --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 50440f57f..0e85c91af 100644 --- a/README.rst +++ b/README.rst @@ -33,6 +33,12 @@ To get up and running: up port 8080 and run ``python synapse/app/homeserver.py --host machine.my.domain.name``. Then come join ``#matrix:matrix.org`` and say hi! :) + +Upgrading an existing homeserver +================================ + +Before upgrading an existing homeserver to a new version, please refer to +UPGRADE.rst for any additional instructions. About Matrix ============ From 9521e6758f6ca870f377f4b757b50c6a6826f328 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 15:23:02 +0100 Subject: [PATCH 112/112] Move the 'Upgrade' section to just below the 'Installation' section --- README.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 0e85c91af..069f37ec0 100644 --- a/README.rst +++ b/README.rst @@ -34,12 +34,7 @@ To get up and running: machine.my.domain.name``. Then come join ``#matrix:matrix.org`` and say hi! :) -Upgrading an existing homeserver -================================ - -Before upgrading an existing homeserver to a new version, please refer to -UPGRADE.rst for any additional instructions. - + About Matrix ============ @@ -149,6 +144,13 @@ This should end with a 'PASSED' result:: PASSED (successes=143) +Upgrading an existing homeserver +================================ + +Before upgrading an existing homeserver to a new version, please refer to +UPGRADE.rst for any additional instructions. + + Setting up Federation =====================