From 3dac27a8a9846a892284971f71e05c2440225484 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Oct 2014 14:54:26 +0100 Subject: [PATCH 001/265] Storage for pdu signatures --- synapse/storage/schema/signatures.sql | 36 +++++++++++ synapse/storage/signatures.py | 90 +++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 synapse/storage/schema/signatures.sql create mode 100644 synapse/storage/signatures.py diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql new file mode 100644 index 000000000..ba3bbb547 --- /dev/null +++ b/synapse/storage/schema/signatures.sql @@ -0,0 +1,36 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS pdu_hashes ( + pdu_id TEXT, + origin TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) +); + +CREATE INDEX IF NOT EXISTS pdu_hashes_id ON pdu_hashes (pdu_id, origin); + +CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( + pdu_id TEXT, + origin TEXT, + key_id TEXT, + signature BLOB, + CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) +); + +CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( + pdu_id, origin, +); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py new file mode 100644 index 000000000..bb860f09f --- /dev/null +++ b/synapse/storage/signatures.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _base import SQLBaseStore + +from twisted.internet import defer + + +class SignatureStore(SQLBaseStore): + """Persistence for PDU signatures and hashes""" + + def _get_pdu_hashes_txn(self, txn, pdu_id, origin): + """Get all the hashes for a given PDU. + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + Returns: + A dict of algorithm -> hash. + """ + query = ( + "SELECT algorithm, hash" + " FROM pdu_hashes" + " WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + return dict(txn.fetchall()) + + def _store_pdu_hash_txn(self, txn, pdu_id, origin, algorithm, hash_bytes): + """Store a hash for a PDU + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + algorithm (str): Hashing algorithm. + hash_bytes (bytes): Hash function output bytes. + """ + self._simple_insert_txn(self, txn, "pdu_hashes", { + "pdu_id": pdu_id, + "origin": origin, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) + + def _get_pdu_origin_signatures_txn(self, txn, pdu_id, origin): + """Get all the signatures for a given PDU. + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + Returns: + A dict of key_id -> signature_bytes. + """ + query = ( + "SELECT key_id, signature" + " FROM pdu_origin_signatures" + " WHERE WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + return dict(txn.fetchall()) + + def _store_pdu_origin_signature_txn(self, txn, pdu_id, origin, key_id, + signature_bytes): + """Store a signature from the origin server for a PDU. + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + key_id (str): Id for the signing key. + signature (bytes): The signature. + """ + self._simple_insert_txn(self, txn, "pdu_origin_signatures", { + "pdu_id": pdu_id, + "origin": origin, + "key_id": key_id, + "signature": buffer(signature_bytes), + }) + From 5fefc12d1e2da56895d5652e3d7516ac59ab8824 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Oct 2014 16:59:51 +0100 Subject: [PATCH 002/265] Begin implementing state groups. --- synapse/state.py | 87 +++++++++++++++++++++++++++++++- synapse/storage/__init__.py | 6 ++- synapse/storage/schema/state.sql | 33 ++++++++++++ 3 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 synapse/storage/schema/state.sql diff --git a/synapse/state.py b/synapse/state.py index 9db84c9b5..8f09b7d50 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -35,7 +35,7 @@ KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) class StateHandler(object): - """ Repsonsible for doing state conflict resolution. + """ Responsible for doing state conflict resolution. """ def __init__(self, hs): @@ -50,7 +50,7 @@ class StateHandler(object): to update the state and b) works out what the prev_state should be. Returns: - Deferred: Resolved with a boolean indicating if we succesfully + Deferred: Resolved with a boolean indicating if we successfully updated the state. Raised: @@ -83,6 +83,8 @@ class StateHandler(object): current_state.pdu_id, current_state.origin ) + yield self.update_state_groups(event) + # TODO check current_state to see if the min power level is less # than the power level of the user # power_level = self._get_power_level_for_event(event) @@ -128,6 +130,87 @@ class StateHandler(object): defer.returnValue(is_new) + @defer.inlineCallbacks + def update_state_groups(self, event): + state_groups = yield self.store.get_state_groups( + event.prev_events + ) + + if len(state_groups) == 1 and not hasattr(event, "state_key"): + event.state_group = state_groups[0].group + event.current_state = state_groups[0].state + return + + state = {} + state_sets = {} + for group in state_groups: + for s in group.state: + state.setdefault((s.type, s.state_key), []).add(s) + + state_sets.setdefault( + (s.type, s.state_key), + set() + ).add(s.event_id) + + unconflicted_state = { + k: v.pop() for k, v in state_sets.items() + if len(v) == 1 + } + + conflicted_state = { + k: state[k] + for k, v in state_sets.items() + if len(v) > 1 + } + + new_state = {} + new_state.update(unconflicted_state) + for key, events in conflicted_state.items(): + new_state[key] = yield self.resolve(events) + + if hasattr(event, "state_key"): + new_state[(event.type, event.state_key)] = event + + event.state_group = None + event.current_state = new_state.values() + + @defer.inlineCallbacks + def resolve(self, events): + curr_events = events + + new_powers_deferreds = [] + for e in curr_events: + new_powers_deferreds.append( + self.store.get_power_level(e.context, e.user_id) + ) + + new_powers = yield defer.gatherResults( + new_powers_deferreds, + consumeErrors=True + ) + + max_power = max([int(p) for p in new_powers]) + + curr_events = [ + z[0] for z in zip(curr_events, new_powers) + if int(z[1]) == max_power + ] + + if not curr_events: + raise RuntimeError("Max didn't get a max?") + elif len(curr_events) == 1: + defer.returnValue(curr_events[0]) + + # TODO: For now, just choose the one with the largest event_id. + defer.returnValue( + sorted( + curr_events, + key=lambda e: hashlib.sha1( + e.event_id + e.user_id + e.room_id + e.type + ).hexdigest() + )[0] + ) + def _get_power_level_for_event(self, event): # return self._persistence.get_power_level_for_user(event.room_id, # event.sender) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6dadeb8cc..10456688e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -40,6 +40,7 @@ from .stream import StreamStore from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore +from .state import StateStore import json import logging @@ -59,6 +60,7 @@ SCHEMAS = [ "room_aliases", "keys", "redactions", + "state", ] @@ -76,7 +78,7 @@ class _RollbackButIsFineException(Exception): class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, PduStore, StatePduStore, TransactionStore, - DirectoryStore, KeyStore): + DirectoryStore, KeyStore, StateStore): def __init__(self, hs): super(DataStore, self).__init__(hs) @@ -222,6 +224,8 @@ class DataStore(RoomMemberStore, RoomStore, ) raise _RollbackButIsFineException("_persist_event") + self._store_state_groups_txn(txn, event) + is_state = hasattr(event, "state_key") and event.state_key is not None if is_new_state and is_state: vals = { diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql new file mode 100644 index 000000000..b5c345fae --- /dev/null +++ b/synapse/storage/schema/state.sql @@ -0,0 +1,33 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS state_groups( + id INTEGER PRIMARY KEY AUTOINCREMENT, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS state_groups_state( + state_group INTEGER NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS event_to_state_groups( + event_id TEXT NOT NULL, + state_group INTEGER NOT NULL +); \ No newline at end of file From 80472ac1988da937f8318651fe5fc764ea869d35 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Oct 2014 10:04:55 +0100 Subject: [PATCH 003/265] Add missing package storate.state --- synapse/storage/state.py | 97 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 synapse/storage/state.py diff --git a/synapse/storage/state.py b/synapse/storage/state.py new file mode 100644 index 000000000..b910646d7 --- /dev/null +++ b/synapse/storage/state.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore +from twisted.internet import defer + +from collections import namedtuple + + +StateGroup = namedtuple("StateGroup", ("group", "state")) + + +class StateStore(SQLBaseStore): + + @defer.inlineCallbacks + def get_state_groups(self, event_ids): + groups = set() + for event_id in event_ids: + group = yield self._simple_select_one_onecol( + table="event_to_state_groups", + keyvalues={"event_id": event_id}, + retcol="state_group", + allow_none=True, + ) + if group: + groups.add(group) + + res = [] + for group in groups: + state_ids = yield self._simple_select_onecol( + table="state_groups_state", + keyvalues={"state_group": group}, + retcol="event_id", + ) + state = [] + for state_id in state_ids: + s = yield self.get_event( + state_id, + allow_none=True, + ) + if s: + state.append(s) + + res.append(StateGroup(group, state)) + + defer.returnValue(res) + + def store_state_groups(self, event): + return self.runInteraction( + self._store_state_groups_txn, event + ) + + def _store_state_groups_txn(self, txn, event): + state_group = event.state_group + if not state_group: + state_group = self._simple_insert_txn( + txn, + table="state_groups", + values={ + "room_id": event.room_id, + "event_id": event.event_id, + } + ) + + for state in event.state_events: + self._simple_insert_txn( + txn, + table="state_groups_state", + values={ + "state_group": state_group, + "room_id": state.room_id, + "type": state.type, + "state_key": state.state_key, + "event_id": state.event_id, + } + ) + + self._simple_insert_txn( + txn, + table="event_to_state_groups", + values={ + "state_group": state_group, + "event_id": event.event_id, + } + ) From e7bc1291a079224315cea5c756061ad711241be1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Oct 2014 16:06:59 +0100 Subject: [PATCH 004/265] Begin making auth use event.old_state_events --- synapse/api/auth.py | 113 +++++++++++++++++++------------ synapse/handlers/_base.py | 10 ++- synapse/handlers/directory.py | 5 +- synapse/handlers/federation.py | 11 ++- synapse/handlers/message.py | 13 +--- synapse/handlers/profile.py | 5 +- synapse/handlers/room.py | 19 +++--- synapse/state.py | 18 ++--- synapse/storage/schema/state.sql | 2 +- synapse/storage/state.py | 2 +- 10 files changed, 115 insertions(+), 83 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e1b1823cd..d951cb265 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,6 +21,7 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, + RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, ) from synapse.util.logutils import log_function @@ -55,11 +56,7 @@ class Auth(object): defer.returnValue(allowed) return - self._check_joined_room( - member=snapshot.membership_state, - user_id=snapshot.user_id, - room_id=snapshot.room_id, - ) + self.check_event_sender_in_room(event) if is_state: # TODO (erikj): This really only should be called for *new* @@ -98,6 +95,16 @@ class Auth(object): pass defer.returnValue(None) + def check_event_sender_in_room(self, event): + key = (RoomMemberEvent.TYPE, event.user_id, ) + member_event = event.state_events.get(key) + + return self._check_joined_room( + member_event, + event.user_id, + event.room_id + ) + def _check_joined_room(self, member, user_id, room_id): if not member or member.membership != Membership.JOIN: raise AuthError(403, "User %s not in room %s (%s)" % ( @@ -114,29 +121,39 @@ class Auth(object): raise AuthError(403, "Room does not exist") # get info about the caller - try: - caller = yield self.store.get_room_member( - user_id=event.user_id, - room_id=event.room_id) - except: - caller = None + key = (RoomMemberEvent.TYPE, event.user_id, ) + caller = event.old_state_events.get(key) + caller_in_room = caller and caller.membership == "join" # get info about the target - try: - target = yield self.store.get_room_member( - user_id=target_user_id, - room_id=event.room_id) - except: - target = None + key = (RoomMemberEvent.TYPE, target_user_id, ) + target = event.old_state_events.get(key) + target_in_room = target and target.membership == "join" membership = event.content["membership"] - join_rule = yield self.store.get_room_join_rule(event.room_id) - if not join_rule: + key = (RoomJoinRulesEvent.TYPE, "", ) + join_rule_event = event.old_state_events.get(key) + if join_rule_event: + join_rule = join_rule_event.content.get( + "join_rule", JoinRules.INVITE + ) + else: join_rule = JoinRules.INVITE + user_level = self._get_power_level_from_event_state( + event, + event.user_id, + ) + + ban_level, kick_level, redact_level = ( + yield self._get_ops_level_from_event_state( + event + ) + ) + if Membership.INVITE == membership: # TODO (erikj): We should probably handle this more intelligently # PRIVATE join rules. @@ -171,29 +188,16 @@ class Auth(object): if not caller_in_room: # trying to leave a room you aren't joined raise AuthError(403, "You are not in room %s." % event.room_id) elif target_user_id != event.user_id: - user_level = yield self.store.get_power_level( - event.room_id, - event.user_id, - ) - _, kick_level, _ = yield self.store.get_ops_levels(event.room_id) - if kick_level: kick_level = int(kick_level) else: - kick_level = 50 + kick_level = 50 # FIXME (erikj): What should we do here? if user_level < kick_level: raise AuthError( 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: - user_level = yield self.store.get_power_level( - event.room_id, - event.user_id, - ) - - ban_level, _, _ = yield self.store.get_ops_levels(event.room_id) - if ban_level: ban_level = int(ban_level) else: @@ -206,6 +210,29 @@ class Auth(object): defer.returnValue(True) + def _get_power_level_from_event_state(self, event, user_id): + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) + level = None + if power_level_event: + level = power_level_event.content[user_id] + if not level: + level = power_level_event.content["default"] + + return level + + def _get_ops_level_from_event_state(self, event): + key = (RoomOpsPowerLevelsEvent.TYPE, "", ) + ops_event = event.old_state_events.get(key) + + if ops_event: + return ( + ops_event.content.get("ban_level"), + ops_event.content.get("kick_level"), + ops_event.content.get("redact_level"), + ) + return None, None, None, + @defer.inlineCallbacks def get_user_by_req(self, request): """ Get a registered user's ID. @@ -282,8 +309,8 @@ class Auth(object): else: send_level = 0 - user_level = yield self.store.get_power_level( - event.room_id, + user_level = self._get_power_level_from_event_state( + event, event.user_id, ) @@ -308,8 +335,8 @@ class Auth(object): add_level = int(add_level) - user_level = yield self.store.get_power_level( - event.room_id, + user_level = self._get_power_level_from_event_state( + event, event.user_id, ) @@ -333,8 +360,8 @@ class Auth(object): if current_state: current_state = current_state[0] - user_level = yield self.store.get_power_level( - event.room_id, + user_level = self._get_power_level_from_event_state( + event, event.user_id, ) @@ -363,10 +390,10 @@ class Auth(object): event.user_id, ) - if user_level: - user_level = int(user_level) - else: - user_level = 0 + user_level = self._get_power_level_from_event_state( + event, + event.user_id, + ) _, _, redact_level = yield self.store.get_ops_levels(event.room_id) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index de4d23bbb..cd6c35f19 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -44,9 +44,17 @@ class BaseHandler(object): @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], - extra_users=[]): + extra_users=[], suppress_auth=False): snapshot.fill_out_prev_events(event) + yield self.state_handler.annotate_state_groups(event) + + if not suppress_auth: + yield self.auth.check(event, snapshot, raises=True) + + if hasattr(event, "state_key"): + yield self.state_handler.handle_new_event(event, snapshot) + yield self.store.persist_event(event) destinations = set(extra_destinations) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index a56830d52..6e897e915 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -152,5 +152,6 @@ class DirectoryHandler(BaseHandler): user_id=user_id, ) - yield self.state_handler.handle_new_event(event, snapshot) - yield self._on_new_room_event(event, snapshot, extra_users=[user_id]) + yield self._on_new_room_event( + event, snapshot, extra_users=[user_id], suppress_auth=True + ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f52591d2a..44bf7def2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -95,6 +95,8 @@ class FederationHandler(BaseHandler): logger.debug("Got event: %s", event.event_id) + yield self.state_handler.annotate_state_groups(event) + with (yield self.lock_manager.lock(pdu.context)): if event.is_state and not backfilled: is_new_state = yield self.state_handler.handle_new_state( @@ -195,7 +197,12 @@ class FederationHandler(BaseHandler): for pdu in pdus: event = self.pdu_codec.event_from_pdu(pdu) + + # FIXME (erikj): Not sure this actually works :/ + yield self.state_handler.annotate_state_groups(event) + events.append(event) + yield self.store.persist_event(event, backfilled=True) defer.returnValue(events) @@ -235,6 +242,7 @@ class FederationHandler(BaseHandler): new_event.destinations = [target_host] snapshot.fill_out_prev_events(new_event) + yield self.state_handler.annotate_state_groups(new_event) yield self.handle_new_event(new_event, snapshot) # TODO (erikj): Time out here. @@ -254,12 +262,11 @@ class FederationHandler(BaseHandler): is_public=False ) except: + # FIXME pass - defer.returnValue(True) - @log_function def _on_user_joined(self, user, room_id): waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 317ef2c80..1c2cbce15 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -87,10 +87,9 @@ class MessageHandler(BaseHandler): snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) - if not suppress_auth: - yield self.auth.check(event, snapshot, raises=True) - - yield self._on_new_room_event(event, snapshot) + yield self._on_new_room_event( + event, snapshot, suppress_auth=suppress_auth + ) self.hs.get_handlers().presence_handler.bump_presence_active_time( user @@ -149,13 +148,9 @@ class MessageHandler(BaseHandler): state_key=event.state_key, ) - yield self.auth.check(event, snapshot, raises=True) - if stamp_event: event.content["hsob_ts"] = int(self.clock.time_msec()) - yield self.state_handler.handle_new_event(event, snapshot) - yield self._on_new_room_event(event, snapshot) @defer.inlineCallbacks @@ -227,8 +222,6 @@ class MessageHandler(BaseHandler): snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) - yield self.auth.check(event, snapshot, raises=True) - # store message in db yield self._on_new_room_event(event, snapshot) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index dab9b03f0..4cd0a0609 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -218,5 +218,6 @@ class ProfileHandler(BaseHandler): user_id=j.state_key, ) - yield self.state_handler.handle_new_event(new_event, snapshot) - yield self._on_new_room_event(new_event, snapshot) + yield self._on_new_room_event( + new_event, snapshot, suppress_auth=True + ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c0f9a7c80..cb5bd17d2 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -129,8 +129,9 @@ class RoomCreationHandler(BaseHandler): logger.debug("Event: %s", event) - yield self.state_handler.handle_new_event(event, snapshot) - yield self._on_new_room_event(event, snapshot, extra_users=[user]) + yield self._on_new_room_event( + event, snapshot, extra_users=[user], suppress_auth=True + ) for event in creation_events: yield handle_event(event) @@ -396,8 +397,6 @@ class RoomMemberHandler(BaseHandler): yield self._do_join(event, snapshot, do_auth=do_auth) else: # This is not a JOIN, so we can handle it normally. - if do_auth: - yield self.auth.check(event, snapshot, raises=True) # If we're banning someone, set a req power level if event.membership == Membership.BAN: @@ -419,6 +418,7 @@ class RoomMemberHandler(BaseHandler): event, membership=event.content["membership"], snapshot=snapshot, + do_auth=do_auth, ) defer.returnValue({"room_id": room_id}) @@ -507,14 +507,11 @@ class RoomMemberHandler(BaseHandler): if not have_joined: logger.debug("Doing normal join") - if do_auth: - yield self.auth.check(event, snapshot, raises=True) - - yield self.state_handler.handle_new_event(event, snapshot) yield self._do_local_membership_update( event, membership=event.content["membership"], snapshot=snapshot, + do_auth=do_auth, ) user = self.hs.parse_userid(event.user_id) @@ -558,7 +555,8 @@ class RoomMemberHandler(BaseHandler): defer.returnValue([r.room_id for r in rooms]) - def _do_local_membership_update(self, event, membership, snapshot): + def _do_local_membership_update(self, event, membership, snapshot, + do_auth): destinations = [] # If we're inviting someone, then we should also send it to that @@ -575,9 +573,10 @@ class RoomMemberHandler(BaseHandler): return self._on_new_room_event( event, snapshot, extra_destinations=destinations, - extra_users=[target_user] + extra_users=[target_user], suppress_auth=(not do_auth), ) + class RoomListHandler(BaseHandler): @defer.inlineCallbacks diff --git a/synapse/state.py b/synapse/state.py index 8f09b7d50..9be6b716e 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -71,6 +71,7 @@ class StateHandler(object): # (w.r.t. to power levels) snapshot.fill_out_prev_events(event) + yield self.annotate_state_groups(event) event.prev_events = [ e for e in event.prev_events if e != event.event_id @@ -83,8 +84,6 @@ class StateHandler(object): current_state.pdu_id, current_state.origin ) - yield self.update_state_groups(event) - # TODO check current_state to see if the min power level is less # than the power level of the user # power_level = self._get_power_level_for_event(event) @@ -131,21 +130,16 @@ class StateHandler(object): defer.returnValue(is_new) @defer.inlineCallbacks - def update_state_groups(self, event): + def annotate_state_groups(self, event): state_groups = yield self.store.get_state_groups( event.prev_events ) - if len(state_groups) == 1 and not hasattr(event, "state_key"): - event.state_group = state_groups[0].group - event.current_state = state_groups[0].state - return - state = {} state_sets = {} for group in state_groups: for s in group.state: - state.setdefault((s.type, s.state_key), []).add(s) + state.setdefault((s.type, s.state_key), []).append(s) state_sets.setdefault( (s.type, s.state_key), @@ -153,7 +147,7 @@ class StateHandler(object): ).add(s.event_id) unconflicted_state = { - k: v.pop() for k, v in state_sets.items() + k: state[k].pop() for k, v in state_sets.items() if len(v) == 1 } @@ -168,11 +162,13 @@ class StateHandler(object): for key, events in conflicted_state.items(): new_state[key] = yield self.resolve(events) + event.old_state_events = new_state + if hasattr(event, "state_key"): new_state[(event.type, event.state_key)] = event event.state_group = None - event.current_state = new_state.values() + event.state_events = new_state @defer.inlineCallbacks def resolve(self, events): diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql index b5c345fae..b44c56b51 100644 --- a/synapse/storage/schema/state.sql +++ b/synapse/storage/schema/state.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, room_id TEXT NOT NULL, event_id TEXT NOT NULL ); diff --git a/synapse/storage/state.py b/synapse/storage/state.py index b910646d7..9496c935a 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -74,7 +74,7 @@ class StateStore(SQLBaseStore): } ) - for state in event.state_events: + for state in event.state_events.values(): self._simple_insert_txn( txn, table="state_groups_state", From 1c445f88f64beabf0bd9bec3950a4a4c0d529e8a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 15 Oct 2014 17:09:04 +0100 Subject: [PATCH 005/265] persist hashes and origin signatures for PDUs --- synapse/api/events/utils.py | 23 ++++++--- synapse/crypto/event_signing.py | 70 +++++++++++++++++++++++++++ synapse/federation/units.py | 17 ++++++- synapse/storage/__init__.py | 21 +++++++- synapse/storage/pdu.py | 11 ++++- synapse/storage/schema/signatures.sql | 4 +- tests/federation/test_federation.py | 4 +- 7 files changed, 135 insertions(+), 15 deletions(-) create mode 100644 synapse/crypto/event_signing.py diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index c3a32be8c..7fdf45a26 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -27,7 +27,14 @@ def prune_event(event): the user has specified, but we do want to keep necessary information like type, state_key etc. """ + return _prune_event_or_pdu(event.type, event) +def prune_pdu(pdu): + """Removes keys that contain unrestricted and non-essential data from a PDU + """ + return _prune_event_or_pdu(pdu.pdu_type, pdu) + +def _prune_event_or_pdu(event_type, event): # Remove all extraneous fields. event.unrecognized_keys = {} @@ -38,25 +45,25 @@ def prune_event(event): if field in event.content: new_content[field] = event.content[field] - if event.type == RoomMemberEvent.TYPE: + if event_type == RoomMemberEvent.TYPE: add_fields("membership") - elif event.type == RoomCreateEvent.TYPE: + elif event_type == RoomCreateEvent.TYPE: add_fields("creator") - elif event.type == RoomJoinRulesEvent.TYPE: + elif event_type == RoomJoinRulesEvent.TYPE: add_fields("join_rule") - elif event.type == RoomPowerLevelsEvent.TYPE: + elif event_type == RoomPowerLevelsEvent.TYPE: # TODO: Actually check these are valid user_ids etc. add_fields("default") for k, v in event.content.items(): if k.startswith("@") and isinstance(v, (int, long)): new_content[k] = v - elif event.type == RoomAddStateLevelEvent.TYPE: + elif event_type == RoomAddStateLevelEvent.TYPE: add_fields("level") - elif event.type == RoomSendEventLevelEvent.TYPE: + elif event_type == RoomSendEventLevelEvent.TYPE: add_fields("level") - elif event.type == RoomOpsPowerLevelsEvent.TYPE: + elif event_type == RoomOpsPowerLevelsEvent.TYPE: add_fields("kick_level", "ban_level", "redact_level") - elif event.type == RoomAliasesEvent.TYPE: + elif event_type == RoomAliasesEvent.TYPE: add_fields("aliases") event.content = new_content diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py new file mode 100644 index 000000000..6557727e0 --- /dev/null +++ b/synapse/crypto/event_signing.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from synapse.api.events.utils import prune_pdu +from syutil.jsonutil import encode_canonical_json +from syutil.base64util import encode_base64, decode_base64 +from syutil.crypto.jsonsign import sign_json, verify_signed_json + +import hashlib + + +def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256): + hashed = _compute_hash(pdu, hash_algortithm) + hashes[hashed.name] = encode_base64(hashed.digest()) + pdu.hashes = hashes + return pdu + + +def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256): + """Check whether the hash for this PDU matches the contents""" + computed_hash = _compute_hash(pdu, hash_algortithm) + if computed_hash.name not in pdu.hashes: + raise Exception("Algorithm %s not in hashes %s" % ( + computed_hash.name, list(pdu.hashes) + )) + message_hash_base64 = hashes[computed_hash.name] + try: + message_hash_bytes = decode_base64(message_hash_base64) + except: + raise Exception("Invalid base64: %s" % (message_hash_base64,)) + return message_hash_bytes == computed_hash.digest() + + +def _compute_hash(pdu, hash_algorithm): + pdu_json = pdu.get_dict() + pdu_json.pop("meta", None) + pdu_json.pop("signatures", None) + hashes = pdu_json.pop("hashes", {}) + pdu_json_bytes = encode_canonical_json(pdu_json) + return hash_algorithm(pdu_json_bytes) + + +def sign_event_pdu(pdu, signature_name, signing_key): + tmp_pdu = Pdu(**pdu.get_dict()) + tmp_pdu = prune_pdu(tmp_pdu) + pdu_json = tmp_pdu.get_dict() + pdu_jdon = sign_json(pdu_json, signature_name, signing_key) + pdu.signatures = pdu_json["signatures"] + return pdu + + +def verify_signed_event_pdu(pdu, signature_name, verify_key): + tmp_pdu = Pdu(**pdu.get_dict()) + tmp_pdu = prune_pdu(tmp_pdu) + pdu_json = tmp_pdu.get_dict() + verify_signed_json(pdu_json, signature_name, verify_key) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d97aeb698..3518efb21 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -18,6 +18,7 @@ server protocol. """ from synapse.util.jsonobject import JsonEncodedObject +from syutil.base64util import encode_base64 import logging import json @@ -63,6 +64,8 @@ class Pdu(JsonEncodedObject): "depth", "content", "outlier", + "hashes", + "signatures", "is_state", # Below this are keys valid only for State Pdus. "state_key", "power_level", @@ -91,7 +94,7 @@ class Pdu(JsonEncodedObject): # just leaving it as a dict. (OR DO WE?!) def __init__(self, destinations=[], is_state=False, prev_pdus=[], - outlier=False, **kwargs): + outlier=False, hashes={}, signatures={}, **kwargs): if is_state: for required_key in ["state_key"]: if required_key not in kwargs: @@ -102,6 +105,8 @@ class Pdu(JsonEncodedObject): is_state=is_state, prev_pdus=prev_pdus, outlier=outlier, + hashes=hashes, + signatures=signatures, **kwargs ) @@ -126,6 +131,16 @@ class Pdu(JsonEncodedObject): if "unrecognized_keys" in d and d["unrecognized_keys"]: args.update(json.loads(d["unrecognized_keys"])) + hashes = { + alg: encode_base64(hsh) + for alg, hsh in pdu_tuple.hashes.items() + } + + signatures = { + kid: encode_base64(sig) + for kid, sig in pdu_tuple.signatures.items() + } + return Pdu( prev_pdus=pdu_tuple.prev_pdu_list, **args diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6dadeb8cc..bfeab7d1e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -40,6 +40,8 @@ from .stream import StreamStore from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore +from .signatures import SignatureStore + import json import logging @@ -59,6 +61,7 @@ SCHEMAS = [ "room_aliases", "keys", "redactions", + "signatures", ] @@ -76,7 +79,7 @@ class _RollbackButIsFineException(Exception): class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, PduStore, StatePduStore, TransactionStore, - DirectoryStore, KeyStore): + DirectoryStore, KeyStore, SignatureStore): def __init__(self, hs): super(DataStore, self).__init__(hs) @@ -144,6 +147,8 @@ class DataStore(RoomMemberStore, RoomStore, def _persist_event_pdu_txn(self, txn, pdu): cols = dict(pdu.__dict__) unrec_keys = dict(pdu.unrecognized_keys) + del cols["hashes"] + del cols["signatures"] del cols["content"] del cols["prev_pdus"] cols["content_json"] = json.dumps(pdu.content) @@ -157,6 +162,20 @@ class DataStore(RoomMemberStore, RoomStore, logger.debug("Persisting: %s", repr(cols)) + for hash_alg, hash_base64 in pdu.hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_pdu_hash_txn( + txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, + ) + + signatures = pdu.sigatures.get(pdu.orgin, {}) + + for key_id, signature_base64 in signatures: + signature_bytes = decode_base64(signature_base64) + self.store_pdu_origin_signatures_txn( + txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, + ) + if pdu.is_state: self._persist_state_txn(txn, pdu.prev_pdus, cols) else: diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index d70467dcd..9d624429b 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -64,6 +64,11 @@ class PduStore(SQLBaseStore): for r in PduEdgesTable.decode_results(txn.fetchall()) ] + hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + signatures = self._get_pdu_origin_signatures_txn( + txn, pdu_id, origin + ) + query = ( "SELECT %(fields)s FROM %(pdus)s as p " "LEFT JOIN %(state)s as s " @@ -80,7 +85,9 @@ class PduStore(SQLBaseStore): row = txn.fetchone() if row: - results.append(PduTuple(PduEntry(*row), edges)) + results.append(PduTuple( + PduEntry(*row), edges, hashes, signatures + )) return results @@ -908,7 +915,7 @@ This does not include a prev_pdus key. PduTuple = namedtuple( "PduTuple", - ("pdu_entry", "prev_pdu_list") + ("pdu_entry", "prev_pdu_list", "hashes", "signatures") ) """ This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent the `prev_pdus` key of a PDU. diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql index ba3bbb547..86ee0f237 100644 --- a/synapse/storage/schema/signatures.sql +++ b/synapse/storage/schema/signatures.sql @@ -28,9 +28,9 @@ CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( origin TEXT, key_id TEXT, signature BLOB, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) + CONSTRAINT uniqueness UNIQUE (pdu_id, origin, key_id) ); CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( - pdu_id, origin, + pdu_id, origin ); diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index d86ce83b2..03b2167cf 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -41,7 +41,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus) + return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}) class FederationTestCase(unittest.TestCase): @@ -183,6 +183,8 @@ class FederationTestCase(unittest.TestCase): "is_state": False, "content": {"testing": "content here"}, "depth": 1, + "hashes": {}, + "signatures": {}, }, ] }, From 66104da10c4191aa1e048f2379190574755109e6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 16 Oct 2014 00:09:48 +0100 Subject: [PATCH 006/265] Sign outgoing PDUs. --- synapse/crypto/event_signing.py | 4 ++-- synapse/federation/pdu_codec.py | 6 +++++- synapse/storage/__init__.py | 7 ++++--- synapse/storage/signatures.py | 6 +++--- tests/federation/test_pdu_codec.py | 13 +++++++++--- tests/rest/test_events.py | 7 +++++-- tests/rest/test_profile.py | 8 ++++++-- tests/rest/test_rooms.py | 32 +++++++++++++++++++++++------- tests/utils.py | 3 ++- 9 files changed, 62 insertions(+), 24 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 6557727e0..a115967c0 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -15,6 +15,7 @@ # limitations under the License. +from synapse.federation.units import Pdu from synapse.api.events.utils import prune_pdu from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 @@ -25,8 +26,7 @@ import hashlib def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256): hashed = _compute_hash(pdu, hash_algortithm) - hashes[hashed.name] = encode_base64(hashed.digest()) - pdu.hashes = hashes + pdu.hashes[hashed.name] = encode_base64(hashed.digest()) return pdu diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index cef61108d..bcac5f9ae 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -14,6 +14,7 @@ # limitations under the License. from .units import Pdu +from synapse.crypto.event_signing import hash_event_pdu, sign_event_pdu import copy @@ -33,6 +34,7 @@ def encode_event_id(pdu_id, origin): class PduCodec(object): def __init__(self, hs): + self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.event_factory = hs.get_event_factory() self.clock = hs.get_clock() @@ -99,4 +101,6 @@ class PduCodec(object): if "ts" not in kwargs: kwargs["ts"] = int(self.clock.time_msec()) - return Pdu(**kwargs) + pdu = Pdu(**kwargs) + pdu = hash_event_pdu(pdu) + return sign_event_pdu(pdu, self.server_name, self.signing_key) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index bfeab7d1e..b2a3f0b56 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -42,6 +42,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .signatures import SignatureStore +from syutil.base64util import decode_base64 import json import logging @@ -168,11 +169,11 @@ class DataStore(RoomMemberStore, RoomStore, txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, ) - signatures = pdu.sigatures.get(pdu.orgin, {}) + signatures = pdu.signatures.get(pdu.origin, {}) - for key_id, signature_base64 in signatures: + for key_id, signature_base64 in signatures.items(): signature_bytes = decode_base64(signature_base64) - self.store_pdu_origin_signatures_txn( + self._store_pdu_origin_signature_txn( txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, ) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index bb860f09f..1f0a68050 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -47,7 +47,7 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(self, txn, "pdu_hashes", { + self._simple_insert_txn(txn, "pdu_hashes", { "pdu_id": pdu_id, "origin": origin, "algorithm": algorithm, @@ -66,7 +66,7 @@ class SignatureStore(SQLBaseStore): query = ( "SELECT key_id, signature" " FROM pdu_origin_signatures" - " WHERE WHERE pdu_id = ? and origin = ?" + " WHERE pdu_id = ? and origin = ?" ) txn.execute(query, (pdu_id, origin)) return dict(txn.fetchall()) @@ -81,7 +81,7 @@ class SignatureStore(SQLBaseStore): key_id (str): Id for the signing key. signature (bytes): The signature. """ - self._simple_insert_txn(self, txn, "pdu_origin_signatures", { + self._simple_insert_txn(txn, "pdu_origin_signatures", { "pdu_id": pdu_id, "origin": origin, "key_id": key_id, diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 344e1baf6..80851a425 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -23,14 +23,21 @@ from synapse.federation.units import Pdu from synapse.server import HomeServer -from mock import Mock +from mock import Mock, NonCallableMock + +from ..utils import MockKey class PduCodecTestCase(unittest.TestCase): def setUp(self): - self.hs = HomeServer("blargle.net") - self.event_factory = self.hs.get_event_factory() + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + self.hs = HomeServer( + "blargle.net", + config=self.mock_config, + ) + self.event_factory = self.hs.get_event_factory() self.codec = PduCodec(self.hs) def test_decode_event_id(self): diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 79b371c04..362c7bc01 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -28,7 +28,7 @@ from synapse.server import HomeServer # python imports import json -from ..utils import MockHttpResource, MemoryDataStore +from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -122,6 +122,9 @@ class EventStreamPermissionsTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "test", db_pool=None, @@ -139,7 +142,7 @@ class EventStreamPermissionsTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py index b0f48e7fd..3a0d1e700 100644 --- a/tests/rest/test_profile.py +++ b/tests/rest/test_profile.py @@ -18,9 +18,9 @@ from tests import unittest from twisted.internet import defer -from mock import Mock +from mock import Mock, NonCallableMock -from ..utils import MockHttpResource +from ..utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer @@ -41,6 +41,9 @@ class ProfileTestCase(unittest.TestCase): "set_avatar_url", ]) + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=None, http_client=None, @@ -48,6 +51,7 @@ class ProfileTestCase(unittest.TestCase): federation=Mock(), replication_layer=Mock(), datastore=None, + config=self.mock_config, ) def _get_user_by_req(request=None): diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 1ce9b8a83..717019305 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -27,7 +27,7 @@ from synapse.server import HomeServer import json import urllib -from ..utils import MockHttpResource, MemoryDataStore +from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -50,6 +50,9 @@ class RoomPermissionsTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -61,7 +64,7 @@ class RoomPermissionsTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -408,6 +411,9 @@ class RoomsMemberListTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -419,7 +425,7 @@ class RoomsMemberListTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -497,6 +503,9 @@ class RoomsCreateTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -508,7 +517,7 @@ class RoomsCreateTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -598,6 +607,9 @@ class RoomTopicTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -609,7 +621,7 @@ class RoomTopicTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -712,6 +724,9 @@ class RoomMemberStateTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -723,7 +738,7 @@ class RoomMemberStateTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -853,6 +868,9 @@ class RoomMessagesTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -864,7 +882,7 @@ class RoomMessagesTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/utils.py b/tests/utils.py index 60fd6085a..d8be73dba 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -118,13 +118,14 @@ class MockHttpResource(HttpServer): class MockKey(object): alg = "mock_alg" version = "mock_version" + signature = b"\x9a\x87$" @property def verify_key(self): return self def sign(self, message): - return b"\x9a\x87$" + return self def verify(self, message, sig): assert sig == b"\x9a\x87$" From 1116f5330ec80533954026f67018e0db190cbae0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Oct 2014 16:56:51 +0100 Subject: [PATCH 007/265] Start implementing the invite/join dance. Continue moving auth to use event.state_events --- synapse/api/auth.py | 16 +++--- synapse/federation/replication.py | 22 ++++++-- synapse/federation/transport.py | 34 ++++++++++++- synapse/handlers/federation.py | 83 ++++++++++++++++++++++++++++--- 4 files changed, 133 insertions(+), 22 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d951cb265..12ddef1b0 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,7 +21,7 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, - RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, + RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, InviteJoinEvent, ) from synapse.util.logutils import log_function @@ -56,7 +56,8 @@ class Auth(object): defer.returnValue(allowed) return - self.check_event_sender_in_room(event) + if not event.type == InviteJoinEvent.TYPE: + self.check_event_sender_in_room(event) if is_state: # TODO (erikj): This really only should be called for *new* @@ -115,11 +116,6 @@ class Auth(object): def is_membership_change_allowed(self, event): target_user_id = event.state_key - # does this room even exist - room = yield self.store.get_room(event.room_id) - if not room: - raise AuthError(403, "Room does not exist") - # get info about the caller key = (RoomMemberEvent.TYPE, event.user_id, ) caller = event.old_state_events.get(key) @@ -170,7 +166,7 @@ class Auth(object): # joined: It's a NOOP if event.user_id != target_user_id: raise AuthError(403, "Cannot force another user to join.") - elif join_rule == JoinRules.PUBLIC or room.is_public: + elif join_rule == JoinRules.PUBLIC: pass elif join_rule == JoinRules.INVITE: if ( @@ -215,9 +211,9 @@ class Auth(object): power_level_event = event.old_state_events.get(key) level = None if power_level_event: - level = power_level_event.content[user_id] + level = power_level_event.content.get(user_id) if not level: - level = power_level_event.content["default"] + level = power_level_event.content.get("default", 0) return level diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 2346d5504..08c29dece 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -393,9 +393,25 @@ class ReplicationLayer(object): response = yield self.query_handlers[query_type](args) defer.returnValue((200, response)) else: - defer.returnValue((404, "No handler for Query type '%s'" - % (query_type) - )) + defer.returnValue( + (404, "No handler for Query type '%s'" % (query_type, )) + ) + + def on_make_join_request(self, context, user_id): + return self.handler.on_make_join_request(context, user_id) + + @defer.inlineCallbacks + def on_send_join_request(self, origin, content): + pdu = Pdu(**content) + state = yield self.handler.on_send_join_request(origin, pdu) + defer.returnValue((200, self._transaction_from_pdus(state).get_dict())) + + def make_join(self, destination, context, user_id): + return self.transport_layer.make_join( + destination=destination, + context=context, + user_id=user_id, + ) @defer.inlineCallbacks @log_function diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 755eee8cf..4f552272e 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -197,6 +197,19 @@ class TransportLayer(object): defer.returnValue(response) + @defer.inlineCallbacks + @log_function + def make_join(self, destination, context, user_id, retry_on_dns_fail=True): + path = PREFIX + "/make_join/%s/%s" % (context, user_id,) + + response = yield self.client.get_json( + destination=destination, + path=path, + retry_on_dns_fail=retry_on_dns_fail, + ) + + defer.returnValue(response) + @defer.inlineCallbacks def _authenticate_request(self, request): json_request = { @@ -353,6 +366,12 @@ class TransportLayer(object): ) ) + self.server.register_path( + "GET", + re.compile("^" + PREFIX + "/make_join/([^/]*)/([^/]*)$"), + self._on_make_join_request + ) + @defer.inlineCallbacks @log_function def _on_send_request(self, origin, content, query, transaction_id): @@ -438,7 +457,20 @@ class TransportLayer(object): versions = [v.split(",", 1) for v in v_list] return self.request_handler.on_backfill_request( - context, versions, limit) + context, versions, limit + ) + + @log_function + def _on_make_join_request(self, origin, content, query, context, user_id): + return self.request_handler.on_make_join_request( + context, user_id, + ) + + @log_function + def _on_send_join_request(self, origin, content, query): + return self.request_handler.on_send_join_request( + origin, content, + ) class TransportReceivedHandler(object): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 44bf7def2..a4f6c739c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -89,7 +89,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def on_receive_pdu(self, pdu, backfilled): """ Called by the ReplicationLayer when we have a new pdu. We need to - do auth checks and put it throught the StateHandler. + do auth checks and put it through the StateHandler. """ event = self.pdu_codec.event_from_pdu(pdu) @@ -97,13 +97,17 @@ class FederationHandler(BaseHandler): yield self.state_handler.annotate_state_groups(event) - with (yield self.lock_manager.lock(pdu.context)): - if event.is_state and not backfilled: - is_new_state = yield self.state_handler.handle_new_state( - pdu - ) - else: - is_new_state = False + logger.debug("Event: %s", event) + + if not backfilled: + yield self.auth.check(event, None, raises=True) + + if event.is_state and not backfilled: + is_new_state = yield self.state_handler.handle_new_state( + pdu + ) + else: + is_new_state = False # TODO: Implement something in federation that allows us to # respond to PDU. @@ -267,6 +271,69 @@ class FederationHandler(BaseHandler): defer.returnValue(True) + @defer.inlineCallbacks + def on_make_join_request(self, context, user_id): + event = self.event_factory.create_event( + etype=RoomMemberEvent.TYPE, + content={"membership": Membership.JOIN}, + room_id=context, + user_id=user_id, + state_key=user_id, + ) + + snapshot = yield self.store.snapshot_room( + event.room_id, event.user_id, + ) + snapshot.fill_out_prev_events(event) + + pdu = self.pdu_codec.pdu_from_event(event) + + defer.returnValue(pdu) + + @defer.inlineCallbacks + def on_send_join_request(self, origin, pdu): + event = self.pdu_codec.event_from_pdu(pdu) + + yield self.state_handler.annotate_state_groups(event) + yield self.auth.check(event, None, raises=True) + + is_new_state = yield self.state_handler.handle_new_state( + pdu + ) + + # FIXME (erikj): All this is duplicated above :( + + yield self.store.persist_event( + event, + backfilled=False, + is_new_state=is_new_state + ) + + extra_users = [] + if event.type == RoomMemberEvent.TYPE: + target_user_id = event.state_key + target_user = self.hs.parse_userid(target_user_id) + extra_users.append(target_user) + + yield self.notifier.on_new_room_event( + event, extra_users=extra_users + ) + + if event.type == RoomMemberEvent.TYPE: + if event.membership == Membership.JOIN: + user = self.hs.parse_userid(event.state_key) + self.distributor.fire( + "user_joined_room", user=user, room_id=event.room_id + ) + + pdu.destinations = yield self.store.get_joined_hosts_for_room( + event.room_id + ) + + yield self.replication_layer.send_pdu(pdu) + + defer.returnValue(event.state_events.values()) + @log_function def _on_user_joined(self, user, room_id): waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) From bb04447c44036ebf3ae5dde7a4cc7a7909d50ef6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 16 Oct 2014 23:25:12 +0100 Subject: [PATCH 008/265] Include hashes of previous pdus when referencing them --- synapse/api/events/__init__.py | 2 +- synapse/federation/pdu_codec.py | 13 ++++------- synapse/federation/replication.py | 2 +- synapse/federation/units.py | 10 ++++++++- synapse/state.py | 4 ---- synapse/storage/__init__.py | 20 +++++++++++------ synapse/storage/pdu.py | 22 +++++++++++++------ synapse/storage/schema/signatures.sql | 16 ++++++++++++++ synapse/storage/signatures.py | 31 +++++++++++++++++++++++++++ tests/federation/test_federation.py | 2 +- tests/federation/test_pdu_codec.py | 4 ++-- 11 files changed, 95 insertions(+), 31 deletions(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index f66fea290..a5a55742e 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -65,13 +65,13 @@ class SynapseEvent(JsonEncodedObject): internal_keys = [ "is_state", - "prev_events", "depth", "destinations", "origin", "outlier", "power_level", "redacted", + "prev_pdus", ] required_keys = [ diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index bcac5f9ae..11fd7264b 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -45,9 +45,7 @@ class PduCodec(object): kwargs["event_id"] = encode_event_id(pdu.pdu_id, pdu.origin) kwargs["room_id"] = pdu.context kwargs["etype"] = pdu.pdu_type - kwargs["prev_events"] = [ - encode_event_id(p[0], p[1]) for p in pdu.prev_pdus - ] + kwargs["prev_pdus"] = pdu.prev_pdus if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): kwargs["prev_state"] = encode_event_id( @@ -78,11 +76,8 @@ class PduCodec(object): d["context"] = event.room_id d["pdu_type"] = event.type - if hasattr(event, "prev_events"): - d["prev_pdus"] = [ - decode_event_id(e, self.server_name) - for e in event.prev_events - ] + if hasattr(event, "prev_pdus"): + d["prev_pdus"] = event.prev_pdus if hasattr(event, "prev_state"): d["prev_state_id"], d["prev_state_origin"] = ( @@ -95,7 +90,7 @@ class PduCodec(object): kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() - if k not in ["event_id", "room_id", "type", "prev_events"] + if k not in ["event_id", "room_id", "type"] }) if "ts" not in kwargs: diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9363ac730..788a49b8e 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -443,7 +443,7 @@ class ReplicationLayer(object): min_depth = yield self.store.get_min_depth_for_context(pdu.context) if min_depth and pdu.depth > min_depth: - for pdu_id, origin in pdu.prev_pdus: + for pdu_id, origin, hashes in pdu.prev_pdus: exists = yield self._get_persisted_pdu(pdu_id, origin) if not exists: diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 3518efb21..6a4300783 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -141,8 +141,16 @@ class Pdu(JsonEncodedObject): for kid, sig in pdu_tuple.signatures.items() } + prev_pdus = [] + for prev_pdu in pdu_tuple.prev_pdu_list: + prev_hashes = pdu_tuple.edge_hashes.get(prev_pdu, {}) + prev_hashes = { + alg: encode_base64(hsh) for alg, hsh in prev_hashes.items() + } + prev_pdus.append((prev_pdu[0], prev_pdu[1], prev_hashes)) + return Pdu( - prev_pdus=pdu_tuple.prev_pdu_list, + prev_pdus=prev_pdus, **args ) else: diff --git a/synapse/state.py b/synapse/state.py index 9db84c9b5..bc6b928ec 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -72,10 +72,6 @@ class StateHandler(object): snapshot.fill_out_prev_events(event) - event.prev_events = [ - e for e in event.prev_events if e != event.event_id - ] - current_state = snapshot.prev_state_pdu if current_state: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b2a3f0b56..af05b4793 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -177,6 +177,14 @@ class DataStore(RoomMemberStore, RoomStore, txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, ) + for prev_pdu_id, prev_origin, prev_hashes in pdu.prev_pdus: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_pdu_hash_txn( + txn, pdu.pdu_id, pdu.origin, prev_pdu_id, prev_origin, alg, + hash_bytes + ) + if pdu.is_state: self._persist_state_txn(txn, pdu.prev_pdus, cols) else: @@ -352,6 +360,7 @@ class DataStore(RoomMemberStore, RoomStore, prev_pdus = self._get_latest_pdus_in_context( txn, room_id ) + if state_type is not None and state_key is not None: prev_state_pdu = self._get_current_state_pdu( txn, room_id, state_type, state_key @@ -401,17 +410,16 @@ class Snapshot(object): self.prev_state_pdu = prev_state_pdu def fill_out_prev_events(self, event): - if hasattr(event, "prev_events"): + if hasattr(event, "prev_pdus"): return - es = [ - "%s@%s" % (p_id, origin) for p_id, origin, _ in self.prev_pdus + event.prev_pdus = [ + (p_id, origin, hashes) + for p_id, origin, hashes, _ in self.prev_pdus ] - event.prev_events = [e for e in es if e != event.event_id] - if self.prev_pdus: - event.depth = max([int(v) for _, _, v in self.prev_pdus]) + 1 + event.depth = max([int(v) for _, _, _, v in self.prev_pdus]) + 1 else: event.depth = 0 diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 9d624429b..a423b42db 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -20,10 +20,13 @@ from ._base import SQLBaseStore, Table, JoinHelper from synapse.federation.units import Pdu from synapse.util.logutils import log_function +from syutil.base64util import encode_base64 + from collections import namedtuple import logging + logger = logging.getLogger(__name__) @@ -64,6 +67,8 @@ class PduStore(SQLBaseStore): for r in PduEdgesTable.decode_results(txn.fetchall()) ] + edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin) + hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) signatures = self._get_pdu_origin_signatures_txn( txn, pdu_id, origin @@ -86,7 +91,7 @@ class PduStore(SQLBaseStore): row = txn.fetchone() if row: results.append(PduTuple( - PduEntry(*row), edges, hashes, signatures + PduEntry(*row), edges, hashes, signatures, edge_hashes )) return results @@ -310,9 +315,14 @@ class PduStore(SQLBaseStore): (context, ) ) - results = txn.fetchall() + results = [] + for pdu_id, origin, depth in txn.fetchall(): + hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + sha256_bytes = hashes["sha256"] + prev_hashes = {"sha256": encode_base64(sha256_bytes)} + results.append((pdu_id, origin, prev_hashes, depth)) - return [(row[0], row[1], row[2]) for row in results] + return results @defer.inlineCallbacks def get_oldest_pdus_in_context(self, context): @@ -431,7 +441,7 @@ class PduStore(SQLBaseStore): "DELETE FROM %s WHERE pdu_id = ? AND origin = ?" % PduForwardExtremitiesTable.table_name ) - txn.executemany(query, prev_pdus) + txn.executemany(query, list(p[:2] for p in prev_pdus)) # We only insert as a forward extremety the new pdu if there are no # other pdus that reference it as a prev pdu @@ -454,7 +464,7 @@ class PduStore(SQLBaseStore): # deleted in a second if they're incorrect anyway. txn.executemany( PduBackwardExtremitiesTable.insert_statement(), - [(i, o, context) for i, o in prev_pdus] + [(i, o, context) for i, o, _ in prev_pdus] ) # Also delete from the backwards extremities table all ones that @@ -915,7 +925,7 @@ This does not include a prev_pdus key. PduTuple = namedtuple( "PduTuple", - ("pdu_entry", "prev_pdu_list", "hashes", "signatures") + ("pdu_entry", "prev_pdu_list", "hashes", "signatures", "edge_hashes") ) """ This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent the `prev_pdus` key of a PDU. diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql index 86ee0f237..a72c4dc35 100644 --- a/synapse/storage/schema/signatures.sql +++ b/synapse/storage/schema/signatures.sql @@ -34,3 +34,19 @@ CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( pdu_id, origin ); + +CREATE TABLE IF NOT EXISTS pdu_edge_hashes( + pdu_id TEXT, + origin TEXT, + prev_pdu_id TEXT, + prev_origin TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE ( + pdu_id, origin, prev_pdu_id, prev_origin, algorithm + ) +); + +CREATE INDEX IF NOT EXISTS pdu_edge_hashes_id ON pdu_edge_hashes( + pdu_id, origin +); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 1f0a68050..114710248 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -88,3 +88,34 @@ class SignatureStore(SQLBaseStore): "signature": buffer(signature_bytes), }) + def _get_prev_pdu_hashes_txn(self, txn, pdu_id, origin): + """Get all the hashes for previous PDUs of a PDU + Args: + txn (cursor): + pdu_id (str): Id of the PDU. + origin (str): Origin of the PDU. + Returns: + dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. + """ + query = ( + "SELECT prev_pdu_id, prev_origin, algorithm, hash" + " FROM pdu_edge_hashes" + " WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + results = {} + for prev_pdu_id, prev_origin, algorithm, hash_bytes in txn.fetchall(): + hashes = results.setdefault((prev_pdu_id, prev_origin), {}) + hashes[algorithm] = hash_bytes + return results + + def _store_prev_pdu_hash_txn(self, txn, pdu_id, origin, prev_pdu_id, + prev_origin, algorithm, hash_bytes): + self._simple_insert_txn(txn, "pdu_edge_hashes", { + "pdu_id": pdu_id, + "origin": origin, + "prev_pdu_id": prev_pdu_id, + "prev_origin": prev_origin, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 03b2167cf..eed50e633 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -41,7 +41,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}) + return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}, {}) class FederationTestCase(unittest.TestCase): diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 80851a425..0ad8cf664 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -88,7 +88,7 @@ class PduCodecTestCase(unittest.TestCase): self.assertEquals(pdu.context, event.room_id) self.assertEquals(pdu.is_state, event.is_state) self.assertEquals(pdu.depth, event.depth) - self.assertEquals(["alice@bob.com"], event.prev_events) + self.assertEquals(pdu.prev_pdus, event.prev_pdus) self.assertEquals(pdu.content, event.content) def test_pdu_from_event(self): @@ -144,7 +144,7 @@ class PduCodecTestCase(unittest.TestCase): self.assertEquals(pdu.context, event.room_id) self.assertEquals(pdu.is_state, event.is_state) self.assertEquals(pdu.depth, event.depth) - self.assertEquals(["alice@bob.com"], event.prev_events) + self.assertEquals(pdu.prev_pdus, event.prev_pdus) self.assertEquals(pdu.content, event.content) self.assertEquals(pdu.state_key, event.state_key) From c8f996e29ffd7055bc6521ea610fc12ff50502e5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 11:40:35 +0100 Subject: [PATCH 009/265] Hash the same content covered by the signature when referencing previous PDUs rather than reusing the PDU content hashes --- synapse/crypto/event_signing.py | 19 +++++++++--- synapse/federation/pdu_codec.py | 6 ++-- synapse/storage/__init__.py | 9 +++++- synapse/storage/pdu.py | 4 +-- synapse/storage/schema/signatures.sql | 18 +++++++++-- synapse/storage/signatures.py | 44 ++++++++++++++++++++++++--- 6 files changed, 84 insertions(+), 16 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index a115967c0..32d60bd30 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -24,15 +24,15 @@ from syutil.crypto.jsonsign import sign_json, verify_signed_json import hashlib -def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256): - hashed = _compute_hash(pdu, hash_algortithm) +def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): + hashed = _compute_content_hash(pdu, hash_algorithm) pdu.hashes[hashed.name] = encode_base64(hashed.digest()) return pdu -def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256): +def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" - computed_hash = _compute_hash(pdu, hash_algortithm) + computed_hash = _compute_content_hash(pdu, hash_algortithm) if computed_hash.name not in pdu.hashes: raise Exception("Algorithm %s not in hashes %s" % ( computed_hash.name, list(pdu.hashes) @@ -45,7 +45,7 @@ def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256): return message_hash_bytes == computed_hash.digest() -def _compute_hash(pdu, hash_algorithm): +def _compute_content_hash(pdu, hash_algorithm): pdu_json = pdu.get_dict() pdu_json.pop("meta", None) pdu_json.pop("signatures", None) @@ -54,6 +54,15 @@ def _compute_hash(pdu, hash_algorithm): return hash_algorithm(pdu_json_bytes) +def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256): + tmp_pdu = Pdu(**pdu.get_dict()) + tmp_pdu = prune_pdu(tmp_pdu) + pdu_json = tmp_pdu.get_dict() + pdu_json_bytes = encode_canonical_json(pdu_json) + hashed = hash_algorithm(pdu_json_bytes) + return (hashed.name, hashed.digest()) + + def sign_event_pdu(pdu, signature_name, signing_key): tmp_pdu = Pdu(**pdu.get_dict()) tmp_pdu = prune_pdu(tmp_pdu) diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 11fd7264b..7e574f451 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -14,7 +14,9 @@ # limitations under the License. from .units import Pdu -from synapse.crypto.event_signing import hash_event_pdu, sign_event_pdu +from synapse.crypto.event_signing import ( + add_event_pdu_content_hash, sign_event_pdu +) import copy @@ -97,5 +99,5 @@ class PduCodec(object): kwargs["ts"] = int(self.clock.time_msec()) pdu = Pdu(**kwargs) - pdu = hash_event_pdu(pdu) + pdu = add_event_pdu_content_hash(pdu) return sign_event_pdu(pdu, self.server_name, self.signing_key) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index af05b4793..1738260cc 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,6 +44,8 @@ from .signatures import SignatureStore from syutil.base64util import decode_base64 +from synapse.crypto.event_signing import compute_pdu_event_reference_hash + import json import logging import os @@ -165,7 +167,7 @@ class DataStore(RoomMemberStore, RoomStore, for hash_alg, hash_base64 in pdu.hashes.items(): hash_bytes = decode_base64(hash_base64) - self._store_pdu_hash_txn( + self._store_pdu_content_hash_txn( txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, ) @@ -185,6 +187,11 @@ class DataStore(RoomMemberStore, RoomStore, hash_bytes ) + (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) + self._store_pdu_reference_hash_txn( + txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes + ) + if pdu.is_state: self._persist_state_txn(txn, pdu.prev_pdus, cols) else: diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index a423b42db..3a90c382f 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -69,7 +69,7 @@ class PduStore(SQLBaseStore): edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin) - hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + hashes = self._get_pdu_content_hashes_txn(txn, pdu_id, origin) signatures = self._get_pdu_origin_signatures_txn( txn, pdu_id, origin ) @@ -317,7 +317,7 @@ class PduStore(SQLBaseStore): results = [] for pdu_id, origin, depth in txn.fetchall(): - hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin) sha256_bytes = hashes["sha256"] prev_hashes = {"sha256": encode_base64(sha256_bytes)} results.append((pdu_id, origin, prev_hashes, depth)) diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql index a72c4dc35..1c45a51be 100644 --- a/synapse/storage/schema/signatures.sql +++ b/synapse/storage/schema/signatures.sql @@ -13,7 +13,7 @@ * limitations under the License. */ -CREATE TABLE IF NOT EXISTS pdu_hashes ( +CREATE TABLE IF NOT EXISTS pdu_content_hashes ( pdu_id TEXT, origin TEXT, algorithm TEXT, @@ -21,7 +21,21 @@ CREATE TABLE IF NOT EXISTS pdu_hashes ( CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) ); -CREATE INDEX IF NOT EXISTS pdu_hashes_id ON pdu_hashes (pdu_id, origin); +CREATE INDEX IF NOT EXISTS pdu_content_hashes_id ON pdu_content_hashes ( + pdu_id, origin +); + +CREATE TABLE IF NOT EXISTS pdu_reference_hashes ( + pdu_id TEXT, + origin TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) +); + +CREATE INDEX IF NOT EXISTS pdu_reference_hashes_id ON pdu_reference_hashes ( + pdu_id, origin +); CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( pdu_id TEXT, diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 114710248..85eec7ffb 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -21,7 +21,7 @@ from twisted.internet import defer class SignatureStore(SQLBaseStore): """Persistence for PDU signatures and hashes""" - def _get_pdu_hashes_txn(self, txn, pdu_id, origin): + def _get_pdu_content_hashes_txn(self, txn, pdu_id, origin): """Get all the hashes for a given PDU. Args: txn (cursor): @@ -32,13 +32,14 @@ class SignatureStore(SQLBaseStore): """ query = ( "SELECT algorithm, hash" - " FROM pdu_hashes" + " FROM pdu_content_hashes" " WHERE pdu_id = ? and origin = ?" ) txn.execute(query, (pdu_id, origin)) return dict(txn.fetchall()) - def _store_pdu_hash_txn(self, txn, pdu_id, origin, algorithm, hash_bytes): + def _store_pdu_content_hash_txn(self, txn, pdu_id, origin, algorithm, + hash_bytes): """Store a hash for a PDU Args: txn (cursor): @@ -47,13 +48,48 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(txn, "pdu_hashes", { + self._simple_insert_txn(txn, "pdu_content_hashes", { "pdu_id": pdu_id, "origin": origin, "algorithm": algorithm, "hash": buffer(hash_bytes), }) + def _get_pdu_reference_hashes_txn(self, txn, pdu_id, origin): + """Get all the hashes for a given PDU. + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + Returns: + A dict of algorithm -> hash. + """ + query = ( + "SELECT algorithm, hash" + " FROM pdu_reference_hashes" + " WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + return dict(txn.fetchall()) + + def _store_pdu_reference_hash_txn(self, txn, pdu_id, origin, algorithm, + hash_bytes): + """Store a hash for a PDU + Args: + txn (cursor): + pdu_id (str): Id for the PDU. + origin (str): origin of the PDU. + algorithm (str): Hashing algorithm. + hash_bytes (bytes): Hash function output bytes. + """ + self._simple_insert_txn(txn, "pdu_reference_hashes", { + "pdu_id": pdu_id, + "origin": origin, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) + + def _get_pdu_origin_signatures_txn(self, txn, pdu_id, origin): """Get all the signatures for a given PDU. Args: From f71627567b4aa58c5aba7e79c6d972b8ac26b449 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 15:04:17 +0100 Subject: [PATCH 010/265] Finish implementing the new join dance. --- synapse/api/auth.py | 9 ++ synapse/api/events/factory.py | 14 ++- synapse/federation/replication.py | 66 ++++++++--- synapse/federation/transport.py | 68 +++++++++-- synapse/handlers/federation.py | 187 ++++++++++++++---------------- synapse/state.py | 10 +- 6 files changed, 225 insertions(+), 129 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 12ddef1b0..d1eca791a 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -48,6 +48,15 @@ class Auth(object): """ try: if hasattr(event, "room_id"): + if not event.old_state_events: + # Oh, we don't know what the state of the room was, so we + # are trusting that this is allowed (at least for now) + defer.returnValue(True) + + if hasattr(event, "outlier") and event.outlier: + # TODO (erikj): Auth for outliers is done differently. + defer.returnValue(True) + is_state = hasattr(event, "state_key") if event.type == RoomMemberEvent.TYPE: diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 0d94850ce..c6d1151ca 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -51,12 +51,20 @@ class EventFactory(object): self.clock = hs.get_clock() self.hs = hs + self.event_id_count = 0 + + def create_event_id(self): + i = str(self.event_id_count) + self.event_id_count += 1 + + local_part = str(int(self.clock.time())) + i + random_string(5) + + return "%s@%s" % (local_part, self.hs.hostname) + def create_event(self, etype=None, **kwargs): kwargs["type"] = etype if "event_id" not in kwargs: - kwargs["event_id"] = "%s@%s" % ( - random_string(10), self.hs.hostname - ) + kwargs["event_id"] = self.create_event_id() if "ts" not in kwargs: kwargs["ts"] = int(self.clock.time_msec()) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 08c29dece..d48219385 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -244,13 +244,14 @@ class ReplicationLayer(object): pdu = None if pdu_list: pdu = pdu_list[0] - yield self._handle_new_pdu(pdu) + yield self._handle_new_pdu(destination, pdu) defer.returnValue(pdu) @defer.inlineCallbacks @log_function - def get_state_for_context(self, destination, context): + def get_state_for_context(self, destination, context, pdu_id=None, + pdu_origin=None): """Requests all of the `current` state PDUs for a given context from a remote home server. @@ -263,13 +264,14 @@ class ReplicationLayer(object): """ transaction_data = yield self.transport_layer.get_context_state( - destination, context) + destination, context, pdu_id=pdu_id, pdu_origin=pdu_origin, + ) transaction = Transaction(**transaction_data) pdus = [Pdu(outlier=True, **p) for p in transaction.pdus] for pdu in pdus: - yield self._handle_new_pdu(pdu) + yield self._handle_new_pdu(destination, pdu) defer.returnValue(pdus) @@ -315,7 +317,7 @@ class ReplicationLayer(object): dl = [] for pdu in pdu_list: - dl.append(self._handle_new_pdu(pdu)) + dl.append(self._handle_new_pdu(transaction.origin, pdu)) if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: @@ -347,14 +349,19 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_context_state_request(self, context): - results = yield self.store.get_current_state_for_context( - context - ) + def on_context_state_request(self, context, pdu_id, pdu_origin): + if pdu_id and pdu_origin: + pdus = yield self.handler.get_state_for_pdu( + pdu_id, pdu_origin + ) + else: + results = yield self.store.get_current_state_for_context( + context + ) + pdus = [Pdu.from_pdu_tuple(p) for p in results] - logger.debug("Context returning %d results", len(results)) + logger.debug("Context returning %d results", len(pdus)) - pdus = [Pdu.from_pdu_tuple(p) for p in results] defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @defer.inlineCallbacks @@ -396,9 +403,10 @@ class ReplicationLayer(object): defer.returnValue( (404, "No handler for Query type '%s'" % (query_type, )) ) - + @defer.inlineCallbacks def on_make_join_request(self, context, user_id): - return self.handler.on_make_join_request(context, user_id) + pdu = yield self.handler.on_make_join_request(context, user_id) + defer.returnValue(pdu.get_dict()) @defer.inlineCallbacks def on_send_join_request(self, origin, content): @@ -406,13 +414,27 @@ class ReplicationLayer(object): state = yield self.handler.on_send_join_request(origin, pdu) defer.returnValue((200, self._transaction_from_pdus(state).get_dict())) + @defer.inlineCallbacks def make_join(self, destination, context, user_id): - return self.transport_layer.make_join( + pdu_dict = yield self.transport_layer.make_join( destination=destination, context=context, user_id=user_id, ) + logger.debug("Got response to make_join: %s", pdu_dict) + + defer.returnValue(Pdu(**pdu_dict)) + + def send_join(self, destination, pdu): + return self.transport_layer.send_join( + destination, + pdu.context, + pdu.pdu_id, + pdu.origin, + pdu.get_dict(), + ) + @defer.inlineCallbacks @log_function def _get_persisted_pdu(self, pdu_id, pdu_origin): @@ -443,7 +465,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def _handle_new_pdu(self, pdu, backfilled=False): + def _handle_new_pdu(self, origin, pdu, backfilled=False): # We reprocess pdus when we have seen them only as outliers existing = yield self._get_persisted_pdu(pdu.pdu_id, pdu.origin) @@ -452,6 +474,8 @@ class ReplicationLayer(object): defer.returnValue({}) return + state = None + # Get missing pdus if necessary. is_new = yield self.pdu_actions.is_new(pdu) if is_new and not pdu.outlier: @@ -475,12 +499,22 @@ class ReplicationLayer(object): except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") + else: + # We need to get the state at this event, since we have reached + # a backward extremity edge. + state = yield self.get_state_for_context( + origin, pdu.context, pdu.pdu_id, pdu.origin, + ) # Persist the Pdu, but don't mark it as processed yet. yield self.store.persist_event(pdu=pdu) if not backfilled: - ret = yield self.handler.on_receive_pdu(pdu, backfilled=backfilled) + ret = yield self.handler.on_receive_pdu( + pdu, + backfilled=backfilled, + state=state, + ) else: ret = None diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 4f552272e..a0d34fd24 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -72,7 +72,8 @@ class TransportLayer(object): self.received_handler = None @log_function - def get_context_state(self, destination, context): + def get_context_state(self, destination, context, pdu_id=None, + pdu_origin=None): """ Requests all state for a given context (i.e. room) from the given server. @@ -89,7 +90,14 @@ class TransportLayer(object): subpath = "/state/%s/" % context - return self._do_request_for_transaction(destination, subpath) + args = {} + if pdu_id and pdu_origin: + args["pdu_id"] = pdu_id + args["pdu_origin"] = pdu_origin + + return self._do_request_for_transaction( + destination, subpath, args=args + ) @log_function def get_pdu(self, destination, pdu_origin, pdu_id): @@ -135,8 +143,10 @@ class TransportLayer(object): subpath = "/backfill/%s/" % context - args = {"v": ["%s,%s" % (i, o) for i, o in pdu_tuples]} - args["limit"] = limit + args = { + "v": ["%s,%s" % (i, o) for i, o in pdu_tuples], + "limit": limit, + } return self._do_request_for_transaction( dest, @@ -210,6 +220,23 @@ class TransportLayer(object): defer.returnValue(response) + @defer.inlineCallbacks + @log_function + def send_join(self, destination, context, pdu_id, origin, content): + path = PREFIX + "/send_join/%s/%s/%s" % ( + context, + origin, + pdu_id, + ) + + response = yield self.client.put_json( + destination=destination, + path=path, + data=content, + ) + + defer.returnValue(response) + @defer.inlineCallbacks def _authenticate_request(self, request): json_request = { @@ -330,7 +357,11 @@ class TransportLayer(object): re.compile("^" + PREFIX + "/state/([^/]*)/$"), self._with_authentication( lambda origin, content, query, context: - handler.on_context_state_request(context) + handler.on_context_state_request( + context, + query.get("pdu_id", [None])[0], + query.get("pdu_origin", [None])[0] + ) ) ) @@ -369,7 +400,23 @@ class TransportLayer(object): self.server.register_path( "GET", re.compile("^" + PREFIX + "/make_join/([^/]*)/([^/]*)$"), - self._on_make_join_request + self._with_authentication( + lambda origin, content, query, context, user_id: + self._on_make_join_request( + origin, content, query, context, user_id + ) + ) + ) + + self.server.register_path( + "PUT", + re.compile("^" + PREFIX + "/send_join/([^/]*)/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, pdu_origin, pdu_id: + self._on_send_join_request( + origin, content, query, + ) + ) ) @defer.inlineCallbacks @@ -460,18 +507,23 @@ class TransportLayer(object): context, versions, limit ) + @defer.inlineCallbacks @log_function def _on_make_join_request(self, origin, content, query, context, user_id): - return self.request_handler.on_make_join_request( + content = yield self.request_handler.on_make_join_request( context, user_id, ) + defer.returnValue((200, content)) + @defer.inlineCallbacks @log_function def _on_send_join_request(self, origin, content, query): - return self.request_handler.on_send_join_request( + content = yield self.request_handler.on_send_join_request( origin, content, ) + defer.returnValue((200, content)) + class TransportReceivedHandler(object): """ Callbacks used when we receive a transaction diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index a4f6c739c..0ae0541bd 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -20,7 +20,7 @@ from ._base import BaseHandler from synapse.api.events.room import InviteJoinEvent, RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function -from synapse.federation.pdu_codec import PduCodec +from synapse.federation.pdu_codec import PduCodec, encode_event_id from synapse.api.errors import SynapseError from twisted.internet import defer, reactor @@ -87,7 +87,7 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def on_receive_pdu(self, pdu, backfilled): + def on_receive_pdu(self, pdu, backfilled, state=None): """ Called by the ReplicationLayer when we have a new pdu. We need to do auth checks and put it through the StateHandler. """ @@ -95,7 +95,10 @@ class FederationHandler(BaseHandler): logger.debug("Got event: %s", event.event_id) - yield self.state_handler.annotate_state_groups(event) + if state: + state = [self.pdu_codec.event_from_pdu(p) for p in state] + state = {(e.type, e.state_key): e for e in state} + yield self.state_handler.annotate_state_groups(event, state=state) logger.debug("Event: %s", event) @@ -108,83 +111,55 @@ class FederationHandler(BaseHandler): ) else: is_new_state = False + # TODO: Implement something in federation that allows us to # respond to PDU. - target_is_mine = False - if hasattr(event, "target_host"): - target_is_mine = event.target_host == self.hs.hostname - - if event.type == InviteJoinEvent.TYPE: - if not target_is_mine: - logger.debug("Ignoring invite/join event %s", event) - return - - # If we receive an invite/join event then we need to join the - # sender to the given room. - # TODO: We should probably auth this or some such - content = event.content - content.update({"membership": Membership.JOIN}) - new_event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - state_key=event.user_id, - room_id=event.room_id, - user_id=event.user_id, - membership=Membership.JOIN, - content=content + with (yield self.room_lock.lock(event.room_id)): + yield self.store.persist_event( + event, + backfilled, + is_new_state=is_new_state ) - yield self.hs.get_handlers().room_member_handler.change_membership( - new_event, - do_auth=False, + room = yield self.store.get_room(event.room_id) + + if not room: + # Huh, let's try and get the current state + try: + yield self.replication_layer.get_state_for_context( + event.origin, event.room_id, pdu.pdu_id, pdu.origin, + ) + + hosts = yield self.store.get_joined_hosts_for_room( + event.room_id + ) + if self.hs.hostname in hosts: + try: + yield self.store.store_room( + room_id=event.room_id, + room_creator_user_id="", + is_public=False, + ) + except: + pass + except: + logger.exception( + "Failed to get current state for room %s", + event.room_id + ) + + if not backfilled: + extra_users = [] + if event.type == RoomMemberEvent.TYPE: + target_user_id = event.state_key + target_user = self.hs.parse_userid(target_user_id) + extra_users.append(target_user) + + yield self.notifier.on_new_room_event( + event, extra_users=extra_users ) - else: - with (yield self.room_lock.lock(event.room_id)): - yield self.store.persist_event( - event, - backfilled, - is_new_state=is_new_state - ) - - room = yield self.store.get_room(event.room_id) - - if not room: - # Huh, let's try and get the current state - try: - yield self.replication_layer.get_state_for_context( - event.origin, event.room_id - ) - - hosts = yield self.store.get_joined_hosts_for_room( - event.room_id - ) - if self.hs.hostname in hosts: - try: - yield self.store.store_room( - room_id=event.room_id, - room_creator_user_id="", - is_public=False, - ) - except: - pass - except: - logger.exception( - "Failed to get current state for room %s", - event.room_id - ) - - if not backfilled: - extra_users = [] - if event.type == RoomMemberEvent.TYPE: - target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) - extra_users.append(target_user) - - yield self.notifier.on_new_room_event( - event, extra_users=extra_users - ) - if event.type == RoomMemberEvent.TYPE: if event.membership == Membership.JOIN: user = self.hs.parse_userid(event.state_key) @@ -214,40 +189,35 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks def do_invite_join(self, target_host, room_id, joinee, content, snapshot): - hosts = yield self.store.get_joined_hosts_for_room(room_id) if self.hs.hostname in hosts: # We are already in the room. logger.debug("We're already in the room apparently") defer.returnValue(False) - # First get current state to see if we are already joined. - try: - yield self.replication_layer.get_state_for_context( - target_host, room_id - ) - - hosts = yield self.store.get_joined_hosts_for_room(room_id) - if self.hs.hostname in hosts: - # Oh, we were actually in the room already. - logger.debug("We're already in the room apparently") - defer.returnValue(False) - except Exception: - logger.exception("Failed to get current state") - - new_event = self.event_factory.create_event( - etype=InviteJoinEvent.TYPE, - target_host=target_host, - room_id=room_id, - user_id=joinee, - content=content + pdu = yield self.replication_layer.make_join( + target_host, + room_id, + joinee ) - new_event.destinations = [target_host] + logger.debug("Got response to make_join: %s", pdu) - snapshot.fill_out_prev_events(new_event) - yield self.state_handler.annotate_state_groups(new_event) - yield self.handle_new_event(new_event, snapshot) + event = self.pdu_codec.event_from_pdu(pdu) + + # We should assert some things. + assert(event.type == RoomMemberEvent.TYPE) + assert(event.user_id == joinee) + assert(event.state_key == joinee) + assert(event.room_id == room_id) + + event.event_id = self.event_factory.create_event_id() + event.content = content + + state = yield self.replication_layer.send_join( + target_host, + self.pdu_codec.pdu_from_event(event) + ) # TODO (erikj): Time out here. d = defer.Deferred() @@ -326,14 +296,31 @@ class FederationHandler(BaseHandler): "user_joined_room", user=user, room_id=event.room_id ) - pdu.destinations = yield self.store.get_joined_hosts_for_room( + new_pdu = self.pdu_codec.pdu_from_event(event); + new_pdu.destinations = yield self.store.get_joined_hosts_for_room( event.room_id ) - yield self.replication_layer.send_pdu(pdu) + yield self.replication_layer.send_pdu(new_pdu) defer.returnValue(event.state_events.values()) + @defer.inlineCallbacks + def get_state_for_pdu(self, pdu_id, pdu_origin): + state_groups = yield self.store.get_state_groups( + [encode_event_id(pdu_id, pdu_origin)] + ) + + if state_groups: + defer.returnValue( + [ + self.pdu_codec.pdu_from_event(s) + for s in state_groups[0].state + ] + ) + else: + defer.returnValue([]) + @log_function def _on_user_joined(self, user, room_id): waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) diff --git a/synapse/state.py b/synapse/state.py index 9be6b716e..8c4eeb892 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -130,7 +130,13 @@ class StateHandler(object): defer.returnValue(is_new) @defer.inlineCallbacks - def annotate_state_groups(self, event): + def annotate_state_groups(self, event, state=None): + if state: + event.state_group = None + event.old_state_events = None + event.state_events = state + return + state_groups = yield self.store.get_state_groups( event.prev_events ) @@ -177,7 +183,7 @@ class StateHandler(object): new_powers_deferreds = [] for e in curr_events: new_powers_deferreds.append( - self.store.get_power_level(e.context, e.user_id) + self.store.get_power_level(e.room_id, e.user_id) ) new_powers = yield defer.gatherResults( From 4d1a7624f444deee4352645fbf73165e11f66dd0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 15:27:11 +0100 Subject: [PATCH 011/265] move 'age' into 'meta' subdict so that it is clearer that it is not part of the signed data --- synapse/federation/replication.py | 20 ++++++++++++++------ synapse/federation/units.py | 6 +++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 788a49b8e..c4993aa5e 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -295,6 +295,10 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) for p in transaction.pdus: + if "meta" in p: + meta = p["meta"] + if "age" in meta: + p["age"] = meta["age"] if "age" in p: p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) del p["age"] @@ -414,14 +418,16 @@ class ReplicationLayer(object): transmission. """ pdus = [p.get_dict() for p in pdu_list] + time_now = self._clock.time_msec() for p in pdus: - if "age_ts" in pdus: - p["age"] = int(self.clock.time_msec()) - p["age_ts"] - + if "age_ts" in p: + age = time_now - p["age_ts"] + p.setdefault("meta", {})["age"] = int(age) + del p["age_ts"] return Transaction( origin=self.server_name, pdus=pdus, - ts=int(self._clock.time_msec()), + ts=int(time_now), destination=None, ) @@ -589,7 +595,7 @@ class _TransactionQueue(object): logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( - ts=self._clock.time_msec(), + ts=int(self._clock.time_msec()), transaction_id=str(self._next_txn_id), origin=self.server_name, destination=destination, @@ -614,7 +620,9 @@ class _TransactionQueue(object): if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: - p["age"] = now - int(p["age_ts"]) + meta = p.setdefault("meta", {}) + meta["age"] = now - int(p["age_ts"]) + del p["age_ts"] return data code, response = yield self.transport_layer.send_transaction( diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 6a4300783..c4a10a412 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -68,11 +68,11 @@ class Pdu(JsonEncodedObject): "signatures", "is_state", # Below this are keys valid only for State Pdus. "state_key", - "power_level", "prev_state_id", "prev_state_origin", "required_power_level", "user_id", + "meta" ] internal_keys = [ @@ -124,6 +124,10 @@ class Pdu(JsonEncodedObject): if pdu_tuple: d = copy.copy(pdu_tuple.pdu_entry._asdict()) + for k in d.keys(): + if d[k] is None: + del d[k] + d["content"] = json.loads(d["content_json"]) del d["content_json"] From c5cec1cc77029c21f0117c318c522ab320de3923 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 16:50:04 +0100 Subject: [PATCH 012/265] Rename 'meta' to 'unsigned' --- docs/server-server/signing.rst | 16 ++++++++-------- synapse/crypto/event_signing.py | 4 +++- synapse/federation/replication.py | 14 +++++++------- synapse/federation/units.py | 1 - 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/server-server/signing.rst b/docs/server-server/signing.rst index dae10f121..60c701ca9 100644 --- a/docs/server-server/signing.rst +++ b/docs/server-server/signing.rst @@ -1,13 +1,13 @@ Signing JSON ============ -JSON is signed by encoding the JSON object without ``signatures`` or ``meta`` +JSON is signed by encoding the JSON object without ``signatures`` or ``unsigned`` keys using a canonical encoding. The JSON bytes are then signed using the signature algorithm and the signature encoded using base64 with the padding stripped. The resulting base64 signature is added to an object under the *signing key identifier* which is added to the ``signatures`` object under the name of the server signing it which is added back to the original JSON object -along with the ``meta`` object. +along with the ``unsigned`` object. The *signing key identifier* is the concatenation of the *signing algorithm* and a *key version*. The *signing algorithm* identifies the algorithm used to @@ -15,8 +15,8 @@ sign the JSON. The currently support value for *signing algorithm* is ``ed25519`` as implemented by NACL (http://nacl.cr.yp.to/). The *key version* is used to distinguish between different signing keys used by the same entity. -The ``meta`` object and the ``signatures`` object are not covered by the -signature. Therefore intermediate servers can add metadata such as time stamps +The ``unsigned`` object and the ``signatures`` object are not covered by the +signature. Therefore intermediate servers can add unsigneddata such as time stamps and additional signatures. @@ -27,7 +27,7 @@ and additional signatures. "signing_keys": { "ed25519:1": "XSl0kuyvrXNj6A+7/tkrB9sxSbRi08Of5uRhxOqZtEQ" }, - "meta": { + "unsigned": { "retrieved_ts_ms": 922834800000 }, "signatures": { @@ -41,7 +41,7 @@ and additional signatures. def sign_json(json_object, signing_key, signing_name): signatures = json_object.pop("signatures", {}) - meta = json_object.pop("meta", None) + unsigned = json_object.pop("unsigned", None) signed = signing_key.sign(encode_canonical_json(json_object)) signature_base64 = encode_base64(signed.signature) @@ -50,8 +50,8 @@ and additional signatures. signatures.setdefault(sigature_name, {})[key_id] = signature_base64 json_object["signatures"] = signatures - if meta is not None: - json_object["meta"] = meta + if unsigned is not None: + json_object["unsigned"] = unsigned return json_object diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 32d60bd30..a236f7d70 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -47,7 +47,9 @@ def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): def _compute_content_hash(pdu, hash_algorithm): pdu_json = pdu.get_dict() - pdu_json.pop("meta", None) + #TODO: Make "age_ts" key internal + pdu_json.pop("age_ts") + pdu_json.pop("unsigned", None) pdu_json.pop("signatures", None) hashes = pdu_json.pop("hashes", {}) pdu_json_bytes = encode_canonical_json(pdu_json) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index c4993aa5e..f2a5d4d5e 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -295,10 +295,10 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) for p in transaction.pdus: - if "meta" in p: - meta = p["meta"] - if "age" in meta: - p["age"] = meta["age"] + if "unsigned" in p: + unsigned = p["unsigned"] + if "age" in unsigned: + p["age"] = unsigned["age"] if "age" in p: p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) del p["age"] @@ -422,7 +422,7 @@ class ReplicationLayer(object): for p in pdus: if "age_ts" in p: age = time_now - p["age_ts"] - p.setdefault("meta", {})["age"] = int(age) + p.setdefault("unsigned", {})["age"] = int(age) del p["age_ts"] return Transaction( origin=self.server_name, @@ -620,8 +620,8 @@ class _TransactionQueue(object): if "pdus" in data: for p in data["pdus"]: if "age_ts" in p: - meta = p.setdefault("meta", {}) - meta["age"] = now - int(p["age_ts"]) + unsigned = p.setdefault("unsigned", {}) + unsigned["age"] = now - int(p["age_ts"]) del p["age_ts"] return data diff --git a/synapse/federation/units.py b/synapse/federation/units.py index c4a10a412..c629e5793 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -72,7 +72,6 @@ class Pdu(JsonEncodedObject): "prev_state_origin", "required_power_level", "user_id", - "meta" ] internal_keys = [ From 5ffe5ab43fa090111a0141b04ce6342172f60724 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 18:56:42 +0100 Subject: [PATCH 013/265] Use state groups to get current state. Make join dance actually work. --- synapse/api/auth.py | 5 ++ synapse/federation/replication.py | 17 +++++- synapse/federation/transport.py | 57 ++++++++++++++++-- synapse/handlers/federation.py | 74 ++++++++++++++++------- synapse/handlers/message.py | 6 +- synapse/rest/base.py | 5 ++ synapse/rest/events.py | 32 ++++++---- synapse/state.py | 98 +++++++++++++++++++++---------- synapse/storage/pdu.py | 6 ++ synapse/storage/state.py | 3 + 10 files changed, 231 insertions(+), 72 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d1eca791a..50ce7eb4c 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -22,6 +22,7 @@ from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, InviteJoinEvent, + RoomCreateEvent, ) from synapse.util.logutils import log_function @@ -59,6 +60,10 @@ class Auth(object): is_state = hasattr(event, "state_key") + if event.type == RoomCreateEvent.TYPE: + # FIXME + defer.returnValue(True) + if event.type == RoomMemberEvent.TYPE: yield self._can_replace_state(event) allowed = yield self.is_membership_change_allowed(event) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index d48219385..8c7d510ef 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -403,11 +403,18 @@ class ReplicationLayer(object): defer.returnValue( (404, "No handler for Query type '%s'" % (query_type, )) ) + @defer.inlineCallbacks def on_make_join_request(self, context, user_id): pdu = yield self.handler.on_make_join_request(context, user_id) defer.returnValue(pdu.get_dict()) + @defer.inlineCallbacks + def on_invite_request(self, origin, content): + pdu = Pdu(**content) + ret_pdu = yield self.handler.on_send_join_request(origin, pdu) + defer.returnValue((200, ret_pdu.get_dict())) + @defer.inlineCallbacks def on_send_join_request(self, origin, content): pdu = Pdu(**content) @@ -426,8 +433,9 @@ class ReplicationLayer(object): defer.returnValue(Pdu(**pdu_dict)) + @defer.inlineCallbacks def send_join(self, destination, pdu): - return self.transport_layer.send_join( + _, content = yield self.transport_layer.send_join( destination, pdu.context, pdu.pdu_id, @@ -435,6 +443,13 @@ class ReplicationLayer(object): pdu.get_dict(), ) + logger.debug("Got content: %s", content) + pdus = [Pdu(outlier=True, **p) for p in content.get("pdus", [])] + for pdu in pdus: + yield self._handle_new_pdu(destination, pdu) + + defer.returnValue(pdus) + @defer.inlineCallbacks @log_function def _get_persisted_pdu(self, pdu_id, pdu_origin): diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index a0d34fd24..de64702e2 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -229,13 +229,36 @@ class TransportLayer(object): pdu_id, ) - response = yield self.client.put_json( + code, content = yield self.client.put_json( destination=destination, path=path, data=content, ) - defer.returnValue(response) + if not 200 <= code < 300: + raise RuntimeError("Got %d from send_join", code) + + defer.returnValue(json.loads(content)) + + @defer.inlineCallbacks + @log_function + def send_invite(self, destination, context, pdu_id, origin, content): + path = PREFIX + "/invite/%s/%s/%s" % ( + context, + origin, + pdu_id, + ) + + code, content = yield self.client.put_json( + destination=destination, + path=path, + data=content, + ) + + if not 200 <= code < 300: + raise RuntimeError("Got %d from send_invite", code) + + defer.returnValue(json.loads(content)) @defer.inlineCallbacks def _authenticate_request(self, request): @@ -297,9 +320,13 @@ class TransportLayer(object): @defer.inlineCallbacks def new_handler(request, *args, **kwargs): (origin, content) = yield self._authenticate_request(request) - response = yield handler( - origin, content, request.args, *args, **kwargs - ) + try: + response = yield handler( + origin, content, request.args, *args, **kwargs + ) + except: + logger.exception("Callback failed") + raise defer.returnValue(response) return new_handler @@ -419,6 +446,17 @@ class TransportLayer(object): ) ) + self.server.register_path( + "PUT", + re.compile("^" + PREFIX + "/invite/([^/]*)/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, pdu_origin, pdu_id: + self._on_invite_request( + origin, content, query, + ) + ) + ) + @defer.inlineCallbacks @log_function def _on_send_request(self, origin, content, query, transaction_id): @@ -524,6 +562,15 @@ class TransportLayer(object): defer.returnValue((200, content)) + @defer.inlineCallbacks + @log_function + def _on_invite_request(self, origin, content, query): + content = yield self.request_handler.on_invite_request( + origin, content, + ) + + defer.returnValue((200, content)) + class TransportReceivedHandler(object): """ Callbacks used when we receive a transaction diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0ae0541bd..70790aaa7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -62,6 +62,9 @@ class FederationHandler(BaseHandler): self.pdu_codec = PduCodec(hs) + # When joining a room we need to queue any events for that room up + self.room_queues = {} + @log_function @defer.inlineCallbacks def handle_new_event(self, event, snapshot): @@ -95,22 +98,25 @@ class FederationHandler(BaseHandler): logger.debug("Got event: %s", event.event_id) + if event.room_id in self.room_queues: + self.room_queues[event.room_id].append(pdu) + return + if state: state = [self.pdu_codec.event_from_pdu(p) for p in state] state = {(e.type, e.state_key): e for e in state} - yield self.state_handler.annotate_state_groups(event, state=state) + + is_new_state = yield self.state_handler.annotate_state_groups( + event, + state=state + ) logger.debug("Event: %s", event) if not backfilled: yield self.auth.check(event, None, raises=True) - if event.is_state and not backfilled: - is_new_state = yield self.state_handler.handle_new_state( - pdu - ) - else: - is_new_state = False + is_new_state = is_new_state and not backfilled # TODO: Implement something in federation that allows us to # respond to PDU. @@ -211,6 +217,8 @@ class FederationHandler(BaseHandler): assert(event.state_key == joinee) assert(event.room_id == room_id) + self.room_queues[room_id] = [] + event.event_id = self.event_factory.create_event_id() event.content = content @@ -219,15 +227,14 @@ class FederationHandler(BaseHandler): self.pdu_codec.pdu_from_event(event) ) - # TODO (erikj): Time out here. - d = defer.Deferred() - self.waiting_for_join_list.setdefault((joinee, room_id), []).append(d) - reactor.callLater(10, d.cancel) + state = [self.pdu_codec.event_from_pdu(p) for p in state] - try: - yield d - except defer.CancelledError: - raise SynapseError(500, "Unable to join remote room") + logger.debug("do_invite_join state: %s", state) + + is_new_state = yield self.state_handler.annotate_state_groups( + event, + state=state + ) try: yield self.store.store_room( @@ -239,6 +246,32 @@ class FederationHandler(BaseHandler): # FIXME pass + for e in state: + # FIXME: Auth these. + is_new_state = yield self.state_handler.annotate_state_groups( + e, + state=state + ) + + yield self.store.persist_event( + e, + backfilled=False, + is_new_state=False + ) + + yield self.store.persist_event( + event, + backfilled=False, + is_new_state=is_new_state + ) + + room_queue = self.room_queues[room_id] + del self.room_queues[room_id] + + for p in room_queue: + p.outlier = True + yield self.on_receive_pdu(p, backfilled=False) + defer.returnValue(True) @defer.inlineCallbacks @@ -264,13 +297,9 @@ class FederationHandler(BaseHandler): def on_send_join_request(self, origin, pdu): event = self.pdu_codec.event_from_pdu(pdu) - yield self.state_handler.annotate_state_groups(event) + is_new_state= yield self.state_handler.annotate_state_groups(event) yield self.auth.check(event, None, raises=True) - is_new_state = yield self.state_handler.handle_new_state( - pdu - ) - # FIXME (erikj): All this is duplicated above :( yield self.store.persist_event( @@ -303,7 +332,10 @@ class FederationHandler(BaseHandler): yield self.replication_layer.send_pdu(new_pdu) - defer.returnValue(event.state_events.values()) + defer.returnValue([ + self.pdu_codec.pdu_from_event(e) + for e in event.state_events.values() + ]) @defer.inlineCallbacks def get_state_for_pdu(self, pdu_id, pdu_origin): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 1c2cbce15..4aaf97a83 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -199,7 +199,7 @@ class MessageHandler(BaseHandler): raise RoomError( 403, "Member does not meet private room rules.") - data = yield self.store.get_current_state( + data = yield self.state_handler.get_current_state( room_id, event_type, state_key ) defer.returnValue(data) @@ -238,7 +238,7 @@ class MessageHandler(BaseHandler): yield self.auth.check_joined_room(room_id, user_id) # TODO: This is duplicating logic from snapshot_all_rooms - current_state = yield self.store.get_current_state(room_id) + current_state = yield self.state_handler.get_current_state(room_id) defer.returnValue([self.hs.serialize_event(c) for c in current_state]) @defer.inlineCallbacks @@ -315,7 +315,7 @@ class MessageHandler(BaseHandler): "end": end_token.to_string(), } - current_state = yield self.store.get_current_state( + current_state = yield self.state_handler.get_current_state( event.room_id ) d["state"] = [self.hs.serialize_event(c) for c in current_state] diff --git a/synapse/rest/base.py b/synapse/rest/base.py index 2e8e3fa7d..dc784c152 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -18,6 +18,11 @@ from synapse.api.urls import CLIENT_PREFIX from synapse.rest.transactions import HttpTransactionStore import re +import logging + + +logger = logging.getLogger(__name__) + def client_path_pattern(path_regex): """Creates a regex compiled client path with the correct client path diff --git a/synapse/rest/events.py b/synapse/rest/events.py index 097195d7c..92ff5e5ca 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -20,6 +20,12 @@ from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig from synapse.rest.base import RestServlet, client_path_pattern +import logging + + +logger = logging.getLogger(__name__) + + class EventStreamRestServlet(RestServlet): PATTERN = client_path_pattern("/events$") @@ -29,18 +35,22 @@ class EventStreamRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): auth_user = yield self.auth.get_user_by_req(request) + try: + handler = self.handlers.event_stream_handler + pagin_config = PaginationConfig.from_request(request) + timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS + if "timeout" in request.args: + try: + timeout = int(request.args["timeout"][0]) + except ValueError: + raise SynapseError(400, "timeout must be in milliseconds.") - handler = self.handlers.event_stream_handler - pagin_config = PaginationConfig.from_request(request) - timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS - if "timeout" in request.args: - try: - timeout = int(request.args["timeout"][0]) - except ValueError: - raise SynapseError(400, "timeout must be in milliseconds.") - - chunk = yield handler.get_stream(auth_user.to_string(), pagin_config, - timeout=timeout) + chunk = yield handler.get_stream( + auth_user.to_string(), pagin_config, timeout=timeout + ) + except: + logger.exception("Event stream failed") + raise defer.returnValue((200, chunk)) diff --git a/synapse/state.py b/synapse/state.py index 8c4eeb892..24685c6fb 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.federation.pdu_codec import encode_event_id, decode_event_id from synapse.util.logutils import log_function +from synapse.federation.pdu_codec import encode_event_id from collections import namedtuple @@ -130,43 +131,23 @@ class StateHandler(object): defer.returnValue(is_new) @defer.inlineCallbacks + @log_function def annotate_state_groups(self, event, state=None): if state: event.state_group = None event.old_state_events = None - event.state_events = state + event.state_events = {(s.type, s.state_key): s for s in state} + defer.returnValue(False) return - state_groups = yield self.store.get_state_groups( - event.prev_events - ) + if hasattr(event, "outlier") and event.outlier: + event.state_group = None + event.old_state_events = None + event.state_events = None + defer.returnValue(False) + return - state = {} - state_sets = {} - for group in state_groups: - for s in group.state: - state.setdefault((s.type, s.state_key), []).append(s) - - state_sets.setdefault( - (s.type, s.state_key), - set() - ).add(s.event_id) - - unconflicted_state = { - k: state[k].pop() for k, v in state_sets.items() - if len(v) == 1 - } - - conflicted_state = { - k: state[k] - for k, v in state_sets.items() - if len(v) > 1 - } - - new_state = {} - new_state.update(unconflicted_state) - for key, events in conflicted_state.items(): - new_state[key] = yield self.resolve(events) + new_state = yield self.resolve_state_groups(event.prev_events) event.old_state_events = new_state @@ -176,8 +157,63 @@ class StateHandler(object): event.state_group = None event.state_events = new_state + defer.returnValue(hasattr(event, "state_key")) + @defer.inlineCallbacks - def resolve(self, events): + def get_current_state(self, room_id, event_type=None, state_key=""): + # FIXME: HACK! + pdus = yield self.store.get_latest_pdus_in_context(room_id) + + event_ids = [encode_event_id(p.pdu_id, p.origin) for p in pdus] + + res = self.resolve_state_groups(event_ids) + + if event_type: + defer.returnValue(res.get((event_type, state_key))) + return + + defer.returnValue(res.values()) + + @defer.inlineCallbacks + @log_function + def resolve_state_groups(self, event_ids): + state_groups = yield self.store.get_state_groups( + event_ids + ) + + state = {} + for group in state_groups: + for s in group.state: + state.setdefault( + (s.type, s.state_key), + {} + )[s.event_id] = s + + unconflicted_state = { + k: v.values()[0] for k, v in state.items() + if len(v.values()) == 1 + } + + conflicted_state = { + k: v.values() + for k, v in state.items() + if len(v.values()) > 1 + } + + try: + new_state = {} + new_state.update(unconflicted_state) + for key, events in conflicted_state.items(): + new_state[key] = yield self._resolve_state_events(events) + except: + logger.exception("Failed to resolve state") + raise + + defer.returnValue(new_state) + + @defer.inlineCallbacks + @log_function + def _resolve_state_events(self, events): curr_events = events new_powers_deferreds = [] diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index d70467dcd..b1cb0185a 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -277,6 +277,12 @@ class PduStore(SQLBaseStore): (context, depth) ) + def get_latest_pdus_in_context(self, context): + return self.runInteraction( + self._get_latest_pdus_in_context, + context + ) + def _get_latest_pdus_in_context(self, txn, context): """Get's a list of the most current pdus for a given context. This is used when we are sending a Pdu and need to fill out the `prev_pdus` diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 9496c935a..0aa979c9f 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -63,6 +63,9 @@ class StateStore(SQLBaseStore): ) def _store_state_groups_txn(self, txn, event): + if not event.state_events: + return + state_group = event.state_group if not state_group: state_group = self._simple_insert_txn( From b3b19614962d78e7851299ff1f7b41706ced3d00 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 19:37:41 +0100 Subject: [PATCH 014/265] Fix bug where people could join private rooms --- synapse/api/auth.py | 86 +++++++++++++++++++--------------- synapse/handlers/federation.py | 10 +++- synapse/state.py | 12 +++-- 3 files changed, 63 insertions(+), 45 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 50ce7eb4c..93a353330 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -49,12 +49,12 @@ class Auth(object): """ try: if hasattr(event, "room_id"): - if not event.old_state_events: + if event.old_state_events is None: # Oh, we don't know what the state of the room was, so we # are trusting that this is allowed (at least for now) defer.returnValue(True) - if hasattr(event, "outlier") and event.outlier: + if hasattr(event, "outlier") and event.outlier is True: # TODO (erikj): Auth for outliers is done differently. defer.returnValue(True) @@ -65,8 +65,12 @@ class Auth(object): defer.returnValue(True) if event.type == RoomMemberEvent.TYPE: - yield self._can_replace_state(event) - allowed = yield self.is_membership_change_allowed(event) + self._can_replace_state(event) + allowed = self.is_membership_change_allowed(event) + if allowed: + logger.debug("Allowing! %s", event) + else: + logger.debug("Denying! %s", event) defer.returnValue(allowed) return @@ -77,24 +81,28 @@ class Auth(object): # TODO (erikj): This really only should be called for *new* # state yield self._can_add_state(event) - yield self._can_replace_state(event) + self._can_replace_state(event) else: yield self._can_send_event(event) if event.type == RoomPowerLevelsEvent.TYPE: - yield self._check_power_levels(event) + self._check_power_levels(event) if event.type == RoomRedactionEvent.TYPE: - yield self._check_redaction(event) + self._check_redaction(event) + + logger.debug("Allowing! %s", event) defer.returnValue(True) else: raise AuthError(500, "Unknown event: %s" % event) except AuthError as e: logger.info("Event auth check failed on event %s with msg: %s", event, e.msg) + logger.info("Denying! %s", event) if raises: raise e + defer.returnValue(False) @defer.inlineCallbacks @@ -126,7 +134,7 @@ class Auth(object): user_id, room_id, repr(member) )) - @defer.inlineCallbacks + @log_function def is_membership_change_allowed(self, event): target_user_id = event.state_key @@ -159,11 +167,23 @@ class Auth(object): ) ban_level, kick_level, redact_level = ( - yield self._get_ops_level_from_event_state( + self._get_ops_level_from_event_state( event ) ) + logger.debug( + "is_membership_change_allowed: %s", + { + "caller_in_room": caller_in_room, + "target_in_room": target_in_room, + "membership": membership, + "join_rule": join_rule, + "target_user_id": target_user_id, + "event.user_id": event.user_id, + } + ) + if Membership.INVITE == membership: # TODO (erikj): We should probably handle this more intelligently # PRIVATE join rules. @@ -183,10 +203,7 @@ class Auth(object): elif join_rule == JoinRules.PUBLIC: pass elif join_rule == JoinRules.INVITE: - if ( - not caller or caller.membership not in - [Membership.INVITE, Membership.JOIN] - ): + if not caller_in_room: raise AuthError(403, "You are not invited to this room.") else: # TODO (erikj): may_join list @@ -218,7 +235,7 @@ class Auth(object): else: raise AuthError(500, "Unknown membership %s" % membership) - defer.returnValue(True) + return True def _get_power_level_from_event_state(self, event, user_id): key = (RoomPowerLevelsEvent.TYPE, "", ) @@ -359,17 +376,7 @@ class Auth(object): defer.returnValue(True) - @defer.inlineCallbacks def _can_replace_state(self, event): - current_state = yield self.store.get_current_state( - event.room_id, - event.type, - event.state_key, - ) - - if current_state: - current_state = current_state[0] - user_level = self._get_power_level_from_event_state( event, event.user_id, @@ -383,6 +390,10 @@ class Auth(object): logger.debug( "Checking power level for %s, %s", event.user_id, user_level ) + + key = (event.type, event.state_key, ) + current_state = event.old_state_events.get(key) + if current_state and hasattr(current_state, "required_power_level"): req = current_state.required_power_level @@ -393,19 +404,20 @@ class Auth(object): "You don't have permission to change that state" ) - @defer.inlineCallbacks def _check_redaction(self, event): - user_level = yield self.store.get_power_level( - event.room_id, - event.user_id, - ) - user_level = self._get_power_level_from_event_state( event, event.user_id, ) - _, _, redact_level = yield self.store.get_ops_levels(event.room_id) + if user_level: + user_level = int(user_level) + else: + user_level = 0 + + _, _, redact_level = self.store._get_ops_level_from_event_state( + event.room_id + ) if not redact_level: redact_level = 50 @@ -416,7 +428,6 @@ class Auth(object): "You don't have permission to redact events" ) - @defer.inlineCallbacks def _check_power_levels(self, event): for k, v in event.content.items(): if k == "default": @@ -436,19 +447,16 @@ class Auth(object): except: raise SynapseError(400, "Not a valid power level: %s" % (v,)) - current_state = yield self.store.get_current_state( - event.room_id, - event.type, - event.state_key, - ) + key = (event.type, event.state_key, ) + current_state = event.old_state_events.get(key) if not current_state: return else: current_state = current_state[0] - user_level = yield self.store.get_power_level( - event.room_id, + user_level = self._get_power_level_from_event_state( + event, event.user_id, ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 70790aaa7..8c80a3716 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -269,12 +269,12 @@ class FederationHandler(BaseHandler): del self.room_queues[room_id] for p in room_queue: - p.outlier = True yield self.on_receive_pdu(p, backfilled=False) defer.returnValue(True) @defer.inlineCallbacks + @log_function def on_make_join_request(self, context, user_id): event = self.event_factory.create_event( etype=RoomMemberEvent.TYPE, @@ -289,15 +289,21 @@ class FederationHandler(BaseHandler): ) snapshot.fill_out_prev_events(event) + yield self.state_handler.annotate_state_groups(event) + yield self.auth.check(event, None, raises=True) + pdu = self.pdu_codec.pdu_from_event(event) defer.returnValue(pdu) @defer.inlineCallbacks + @log_function def on_send_join_request(self, origin, pdu): event = self.pdu_codec.event_from_pdu(pdu) - is_new_state= yield self.state_handler.annotate_state_groups(event) + event.outlier = False + + is_new_state = yield self.state_handler.annotate_state_groups(event) yield self.auth.check(event, None, raises=True) # FIXME (erikj): All this is duplicated above :( diff --git a/synapse/state.py b/synapse/state.py index 24685c6fb..c062cef75 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -22,6 +22,7 @@ from synapse.federation.pdu_codec import encode_event_id from collections import namedtuple +import copy import logging import hashlib @@ -143,13 +144,13 @@ class StateHandler(object): if hasattr(event, "outlier") and event.outlier: event.state_group = None event.old_state_events = None - event.state_events = None + event.state_events = {} defer.returnValue(False) return new_state = yield self.resolve_state_groups(event.prev_events) - event.old_state_events = new_state + event.old_state_events = copy.deepcopy(new_state) if hasattr(event, "state_key"): new_state[(event.type, event.state_key)] = event @@ -164,9 +165,12 @@ class StateHandler(object): # FIXME: HACK! pdus = yield self.store.get_latest_pdus_in_context(room_id) - event_ids = [encode_event_id(p.pdu_id, p.origin) for p in pdus] + event_ids = [ + encode_event_id(pdu_id, origin) + for pdu_id, origin, _ in pdus + ] - res = self.resolve_state_groups(event_ids) + res = yield self.resolve_state_groups(event_ids) if event_type: defer.returnValue(res.get((event_type, state_key))) From 8afbece68319728e20c3b32c2f949fd1745d405e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 19:41:32 +0100 Subject: [PATCH 015/265] Remove signatures from pdu when computing hashes to use for prev pdus, make sure is_state is a boolean. --- synapse/crypto/event_signing.py | 6 +++++- synapse/federation/units.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index a236f7d70..d3b501c6e 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -22,6 +22,9 @@ from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json, verify_signed_json import hashlib +import logging + +logger = logging.getLogger(__name__) def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): @@ -48,7 +51,7 @@ def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): def _compute_content_hash(pdu, hash_algorithm): pdu_json = pdu.get_dict() #TODO: Make "age_ts" key internal - pdu_json.pop("age_ts") + pdu_json.pop("age_ts", None) pdu_json.pop("unsigned", None) pdu_json.pop("signatures", None) hashes = pdu_json.pop("hashes", {}) @@ -60,6 +63,7 @@ def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256): tmp_pdu = Pdu(**pdu.get_dict()) tmp_pdu = prune_pdu(tmp_pdu) pdu_json = tmp_pdu.get_dict() + pdu_json.pop("signatures", None) pdu_json_bytes = encode_canonical_json(pdu_json) hashed = hash_algorithm(pdu_json_bytes) return (hashed.name, hashed.digest()) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index b81e16251..b779d259b 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -101,7 +101,7 @@ class Pdu(JsonEncodedObject): super(Pdu, self).__init__( destinations=destinations, - is_state=is_state, + is_state=bool(is_state), prev_pdus=prev_pdus, outlier=outlier, hashes=hashes, From bf8cdda2f598acce5a8694fe6482fc929c53715b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Oct 2014 20:10:34 +0100 Subject: [PATCH 016/265] It doesn't want a dict --- synapse/handlers/federation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8c80a3716..b575986fc 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -104,7 +104,6 @@ class FederationHandler(BaseHandler): if state: state = [self.pdu_codec.event_from_pdu(p) for p in state] - state = {(e.type, e.state_key): e for e in state} is_new_state = yield self.state_handler.annotate_state_groups( event, @@ -250,7 +249,6 @@ class FederationHandler(BaseHandler): # FIXME: Auth these. is_new_state = yield self.state_handler.annotate_state_groups( e, - state=state ) yield self.store.persist_event( From eea3a29699aba75a02fefe18d0eeba119a845b04 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 20:36:04 +0100 Subject: [PATCH 017/265] Add script to hash exisitng history --- scripts/hash_history.py | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 scripts/hash_history.py diff --git a/scripts/hash_history.py b/scripts/hash_history.py new file mode 100644 index 000000000..bdad530af --- /dev/null +++ b/scripts/hash_history.py @@ -0,0 +1,69 @@ +from synapse.storage.pdu import PduStore +from synapse.storage.signatures import SignatureStore +from synapse.storage._base import SQLBaseStore +from synapse.federation.units import Pdu +from synapse.crypto.event_signing import ( + add_event_pdu_content_hash, compute_pdu_event_reference_hash +) +from synapse.api.events.utils import prune_pdu +from syutil.base64util import encode_base64, decode_base64 +from syutil.jsonutil import encode_canonical_json +import sqlite3 +import sys + +class Store(object): + _get_pdu_tuples = PduStore.__dict__["_get_pdu_tuples"] + _get_pdu_content_hashes_txn = SignatureStore.__dict__["_get_pdu_content_hashes_txn"] + _get_prev_pdu_hashes_txn = SignatureStore.__dict__["_get_prev_pdu_hashes_txn"] + _get_pdu_origin_signatures_txn = SignatureStore.__dict__["_get_pdu_origin_signatures_txn"] + _store_pdu_content_hash_txn = SignatureStore.__dict__["_store_pdu_content_hash_txn"] + _store_pdu_reference_hash_txn = SignatureStore.__dict__["_store_pdu_reference_hash_txn"] + _store_prev_pdu_hash_txn = SignatureStore.__dict__["_store_prev_pdu_hash_txn"] + _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] + + +store = Store() + + +def select_pdus(cursor): + cursor.execute( + "SELECT pdu_id, origin FROM pdus ORDER BY depth ASC" + ) + + ids = cursor.fetchall() + + pdu_tuples = store._get_pdu_tuples(cursor, ids) + + pdus = [Pdu.from_pdu_tuple(p) for p in pdu_tuples] + + reference_hashes = {} + + for pdu in pdus: + try: + if pdu.prev_pdus: + print "PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus + for pdu_id, origin, hashes in pdu.prev_pdus: + ref_alg, ref_hsh = reference_hashes[(pdu_id, origin)] + hashes[ref_alg] = encode_base64(ref_hsh) + store._store_prev_pdu_hash_txn(cursor, pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh) + print "SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus + pdu = add_event_pdu_content_hash(pdu) + ref_alg, ref_hsh = compute_pdu_event_reference_hash(pdu) + reference_hashes[(pdu.pdu_id, pdu.origin)] = (ref_alg, ref_hsh) + store._store_pdu_reference_hash_txn(cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh) + + for alg, hsh_base64 in pdu.hashes.items(): + print alg, hsh_base64 + store._store_pdu_content_hash_txn(cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64)) + + except: + print "FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus + +def main(): + conn = sqlite3.connect(sys.argv[1]) + cursor = conn.cursor() + select_pdus(cursor) + conn.commit() + +if __name__=='__main__': + main() From 5e2236f9ffe3a66bbe0ff37b1793e8fa59a1c475 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 27 Oct 2014 11:19:15 +0000 Subject: [PATCH 018/265] fix pyflakes warnings --- synapse/crypto/event_signing.py | 8 ++++---- synapse/federation/units.py | 2 ++ synapse/storage/signatures.py | 2 -- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index d3b501c6e..61edd2c6f 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -35,12 +35,12 @@ def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" - computed_hash = _compute_content_hash(pdu, hash_algortithm) + computed_hash = _compute_content_hash(pdu, hash_algorithm) if computed_hash.name not in pdu.hashes: raise Exception("Algorithm %s not in hashes %s" % ( computed_hash.name, list(pdu.hashes) )) - message_hash_base64 = hashes[computed_hash.name] + message_hash_base64 = pdu.hashes[computed_hash.name] try: message_hash_bytes = decode_base64(message_hash_base64) except: @@ -54,7 +54,7 @@ def _compute_content_hash(pdu, hash_algorithm): pdu_json.pop("age_ts", None) pdu_json.pop("unsigned", None) pdu_json.pop("signatures", None) - hashes = pdu_json.pop("hashes", {}) + pdu_json.pop("hashes", None) pdu_json_bytes = encode_canonical_json(pdu_json) return hash_algorithm(pdu_json_bytes) @@ -73,7 +73,7 @@ def sign_event_pdu(pdu, signature_name, signing_key): tmp_pdu = Pdu(**pdu.get_dict()) tmp_pdu = prune_pdu(tmp_pdu) pdu_json = tmp_pdu.get_dict() - pdu_jdon = sign_json(pdu_json, signature_name, signing_key) + pdu_json = sign_json(pdu_json, signature_name, signing_key) pdu.signatures = pdu_json["signatures"] return pdu diff --git a/synapse/federation/units.py b/synapse/federation/units.py index b779d259b..adc338564 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -155,6 +155,8 @@ class Pdu(JsonEncodedObject): return Pdu( prev_pdus=prev_pdus, + hashes=hashes, + signatures=signatures, **args ) else: diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 85eec7ffb..82be946d3 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -15,8 +15,6 @@ from _base import SQLBaseStore -from twisted.internet import defer - class SignatureStore(SQLBaseStore): """Persistence for PDU signatures and hashes""" From c372929ab6246370d3da8e864f36fcbe2e48123b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Oct 2014 16:31:39 +0000 Subject: [PATCH 019/265] Remove duplicate import --- synapse/state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index ec98667c5..cc6a7db96 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,7 +18,6 @@ from twisted.internet import defer from synapse.federation.pdu_codec import encode_event_id, decode_event_id from synapse.util.logutils import log_function -from synapse.federation.pdu_codec import encode_event_id from collections import namedtuple From 8e358ef35a8e2640176003cdb2396a9b4d71fbce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 10:34:05 +0000 Subject: [PATCH 020/265] Add timer to LoggingTransaction --- synapse/storage/_base.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 65a86e905..e839704a8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -23,6 +23,7 @@ from synapse.util.logutils import log_function import collections import copy import json +import time logger = logging.getLogger(__name__) @@ -61,9 +62,15 @@ class LoggingTransaction(object): # TODO(paul): Here would be an excellent place to put some timing # measurements, and log (warning?) slow queries. - return object.__getattribute__(self, "txn").execute( - sql, *args, **kwargs - ) + + start = time.clock() * 1000 + try: + return object.__getattribute__(self, "txn").execute( + sql, *args, **kwargs + ) + finally: + end = time.clock() * 1000 + sql_logger.debug("[SQL time] %f", end - start) class SQLBaseStore(object): From 967ce43b59e90f36c21f3a493bc81ab4e57dae69 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 10:53:11 +0000 Subject: [PATCH 021/265] Clean up LoggingTransaction --- synapse/storage/_base.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e839704a8..d3e874188 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,14 +39,11 @@ class LoggingTransaction(object): def __init__(self, txn): object.__setattr__(self, "txn", txn) - def __getattribute__(self, name): - if name == "execute": - return object.__getattribute__(self, "execute") - - return getattr(object.__getattribute__(self, "txn"), name) + def __getattr__(self, name): + return getattr(self.txn, name) def __setattr__(self, name, value): - setattr(object.__getattribute__(self, "txn"), name, value) + setattr(self.txn, name, value) def execute(self, sql, *args, **kwargs): # TODO(paul): Maybe use 'info' and 'debug' for values? @@ -60,12 +57,9 @@ class LoggingTransaction(object): # Don't let logging failures stop SQL from working pass - # TODO(paul): Here would be an excellent place to put some timing - # measurements, and log (warning?) slow queries. - start = time.clock() * 1000 try: - return object.__getattribute__(self, "txn").execute( + return self.txn.execute( sql, *args, **kwargs ) finally: From da1dda3e1d9d3272527d35c23162c4baf7339d74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 11:18:04 +0000 Subject: [PATCH 022/265] Add transaction level logging and timing information. Add a _simple_delete method --- synapse/storage/__init__.py | 3 +- synapse/storage/_base.py | 74 ++++++++++++++++++++++++++------- synapse/storage/directory.py | 1 + synapse/storage/pdu.py | 13 +++++- synapse/storage/registration.py | 7 +++- synapse/storage/room.py | 2 + synapse/storage/state.py | 1 + synapse/storage/stream.py | 5 ++- synapse/storage/transactions.py | 6 +++ 9 files changed, 91 insertions(+), 21 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 15a72d0cd..a50e19349 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -109,6 +109,7 @@ class DataStore(RoomMemberStore, RoomStore, try: yield self.runInteraction( + "persist_event", self._persist_pdu_event_txn, pdu=pdu, event=event, @@ -394,7 +395,7 @@ class DataStore(RoomMemberStore, RoomStore, prev_state_pdu=prev_state_pdu, ) - return self.runInteraction(_snapshot) + return self.runInteraction("snapshot_room", _snapshot) class Snapshot(object): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d3e874188..119221697 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -29,15 +29,17 @@ import time logger = logging.getLogger(__name__) sql_logger = logging.getLogger("synapse.storage.SQL") +transaction_logger = logging.getLogger("synapse.storage.txn") class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging to the .execute() method.""" - __slots__ = ["txn"] + __slots__ = ["txn", "name"] - def __init__(self, txn): + def __init__(self, txn, name): object.__setattr__(self, "txn", txn) + object.__setattr__(self, "name", name) def __getattr__(self, name): return getattr(self.txn, name) @@ -47,12 +49,15 @@ class LoggingTransaction(object): def execute(self, sql, *args, **kwargs): # TODO(paul): Maybe use 'info' and 'debug' for values? - sql_logger.debug("[SQL] %s", sql) + sql_logger.debug("[SQL] {%s} %s", self.name, sql) try: if args and args[0]: values = args[0] - sql_logger.debug("[SQL values] " + - ", ".join(("<%s>",) * len(values)), *values) + sql_logger.debug( + "[SQL values] {%s} " + ", ".join(("<%s>",) * len(values)), + self.name, + *values + ) except: # Don't let logging failures stop SQL from working pass @@ -64,10 +69,11 @@ class LoggingTransaction(object): ) finally: end = time.clock() * 1000 - sql_logger.debug("[SQL time] %f", end - start) + sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) class SQLBaseStore(object): + _TXN_ID = 0 def __init__(self, hs): self.hs = hs @@ -75,10 +81,24 @@ class SQLBaseStore(object): self.event_factory = hs.get_event_factory() self._clock = hs.get_clock() - def runInteraction(self, func, *args, **kwargs): + def runInteraction(self, desc, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" def inner_func(txn, *args, **kwargs): - return func(LoggingTransaction(txn), *args, **kwargs) + start = time.clock() * 1000 + txn_id = str(SQLBaseStore._TXN_ID) + SQLBaseStore._TXN_ID += 1 + + name = "%s-%s" % (desc, txn_id, ) + + transaction_logger.debug("[TXN START] {%s}", name) + try: + return func(LoggingTransaction(txn, name), *args, **kwargs) + finally: + end = time.clock() * 1000 + transaction_logger.debug( + "[TXN END] {%s} %f", + name, end - start + ) return self._db_pool.runInteraction(inner_func, *args, **kwargs) @@ -114,7 +134,7 @@ class SQLBaseStore(object): else: return cursor.fetchall() - return self.runInteraction(interaction) + return self.runInteraction("_execute", interaction) def _execute_and_decode(self, query, *args): return self._execute(self.cursor_to_dict, query, *args) @@ -131,6 +151,7 @@ class SQLBaseStore(object): or_replace : bool; if True performs an INSERT OR REPLACE """ return self.runInteraction( + "_simple_insert", self._simple_insert_txn, table, values, or_replace=or_replace, or_ignore=or_ignore, ) @@ -168,6 +189,7 @@ class SQLBaseStore(object): statement returns no rows """ return self._simple_selectupdate_one( + "_simple_select_one", table, keyvalues, retcols=retcols, allow_none=allow_none ) @@ -217,7 +239,7 @@ class SQLBaseStore(object): txn.execute(sql, keyvalues.values()) return txn.fetchall() - res = yield self.runInteraction(func) + res = yield self.runInteraction("_simple_select_onecol", func) defer.returnValue([r[0] for r in res]) @@ -240,7 +262,7 @@ class SQLBaseStore(object): txn.execute(sql, keyvalues.values()) return self.cursor_to_dict(txn) - return self.runInteraction(func) + return self.runInteraction("_simple_select_list", func) def _simple_update_one(self, table, keyvalues, updatevalues, retcols=None): @@ -308,7 +330,7 @@ class SQLBaseStore(object): raise StoreError(500, "More than one row matched") return ret - return self.runInteraction(func) + return self.runInteraction("_simple_selectupdate_one", func) def _simple_delete_one(self, table, keyvalues): """Executes a DELETE query on the named table, expecting to delete a @@ -320,7 +342,7 @@ class SQLBaseStore(object): """ sql = "DELETE FROM %s WHERE %s" % ( table, - " AND ".join("%s = ?" % (k) for k in keyvalues) + " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) def func(txn): @@ -329,7 +351,25 @@ class SQLBaseStore(object): raise StoreError(404, "No row found") if txn.rowcount > 1: raise StoreError(500, "more than one row matched") - return self.runInteraction(func) + return self.runInteraction("_simple_delete_one", func) + + def _simple_delete(self, table, keyvalues): + """Executes a DELETE query on the named table. + + Args: + table : string giving the table name + keyvalues : dict of column names and values to select the row with + """ + + return self.runInteraction("_simple_delete", self._simple_delete_txn) + + def _simple_delete_txn(self, txn, table, keyvalues): + sql = "DELETE FROM %s WHERE %s" % ( + table, + " AND ".join("%s = ?" % (k, ) for k in keyvalues) + ) + + return txn.execute(sql, keyvalues.values()) def _simple_max_id(self, table): """Executes a SELECT query on the named table, expecting to return the @@ -347,7 +387,7 @@ class SQLBaseStore(object): return 0 return max_id - return self.runInteraction(func) + return self.runInteraction("_simple_max_id", func) def _parse_event_from_row(self, row_dict): d = copy.deepcopy({k: v for k, v in row_dict.items()}) @@ -371,7 +411,9 @@ class SQLBaseStore(object): ) def _parse_events(self, rows): - return self.runInteraction(self._parse_events_txn, rows) + return self.runInteraction( + "_parse_events", self._parse_events_txn, rows + ) def _parse_events_txn(self, txn, rows): events = [self._parse_event_from_row(r) for r in rows] diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 52373a28a..d6a7113b9 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -95,6 +95,7 @@ class DirectoryStore(SQLBaseStore): def delete_room_alias(self, room_alias): return self.runInteraction( + "delete_room_alias", self._delete_room_alias_txn, room_alias, ) diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 9bdc831fd..4a4341907 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -47,7 +47,7 @@ class PduStore(SQLBaseStore): """ return self.runInteraction( - self._get_pdu_tuple, pdu_id, origin + "get_pdu", self._get_pdu_tuple, pdu_id, origin ) def _get_pdu_tuple(self, txn, pdu_id, origin): @@ -108,6 +108,7 @@ class PduStore(SQLBaseStore): """ return self.runInteraction( + "get_current_state_for_context", self._get_current_state_for_context, context ) @@ -156,6 +157,7 @@ class PduStore(SQLBaseStore): """ return self.runInteraction( + "mark_pdu_as_processed", self._mark_as_processed, pdu_id, pdu_origin ) @@ -165,6 +167,7 @@ class PduStore(SQLBaseStore): def get_all_pdus_from_context(self, context): """Get a list of all PDUs for a given context.""" return self.runInteraction( + "get_all_pdus_from_context", self._get_all_pdus_from_context, context, ) @@ -192,6 +195,7 @@ class PduStore(SQLBaseStore): list: A list of PduTuples """ return self.runInteraction( + "get_backfill", self._get_backfill, context, pdu_list, limit ) @@ -253,6 +257,7 @@ class PduStore(SQLBaseStore): context (str) """ return self.runInteraction( + "get_min_depth_for_context", self._get_min_depth_for_context, context ) @@ -291,6 +296,7 @@ class PduStore(SQLBaseStore): def get_latest_pdus_in_context(self, context): return self.runInteraction( + "get_latest_pdus_in_context", self._get_latest_pdus_in_context, context ) @@ -370,6 +376,7 @@ class PduStore(SQLBaseStore): """ return self.runInteraction( + "is_pdu_new", self._is_pdu_new, pdu_id=pdu_id, origin=origin, @@ -523,6 +530,7 @@ class StatePduStore(SQLBaseStore): def get_unresolved_state_tree(self, new_state_pdu): return self.runInteraction( + "get_unresolved_state_tree", self._get_unresolved_state_tree, new_state_pdu ) @@ -562,6 +570,7 @@ class StatePduStore(SQLBaseStore): def update_current_state(self, pdu_id, origin, context, pdu_type, state_key): return self.runInteraction( + "update_current_state", self._update_current_state, pdu_id, origin, context, pdu_type, state_key ) @@ -601,6 +610,7 @@ class StatePduStore(SQLBaseStore): """ return self.runInteraction( + "get_current_state_pdu", self._get_current_state_pdu, context, pdu_type, state_key ) @@ -660,6 +670,7 @@ class StatePduStore(SQLBaseStore): bool: True if the new_pdu clobbered the current state, False if not """ return self.runInteraction( + "handle_new_state", self._handle_new_state, new_pdu ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 719806f82..a2ca6f9a6 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -62,8 +62,10 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if the user_id could not be registered. """ - yield self.runInteraction(self._register, user_id, token, - password_hash) + yield self.runInteraction( + "register", + self._register, user_id, token, password_hash + ) def _register(self, txn, user_id, token, password_hash): now = int(self.clock.time()) @@ -100,6 +102,7 @@ class RegistrationStore(SQLBaseStore): StoreError if no user was found. """ return self.runInteraction( + "get_user_by_token", self._query_for_auth, token ) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 8cd46334c..7e48ce9cc 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -150,6 +150,7 @@ class RoomStore(SQLBaseStore): def get_power_level(self, room_id, user_id): return self.runInteraction( + "get_power_level", self._get_power_level, room_id, user_id, ) @@ -183,6 +184,7 @@ class RoomStore(SQLBaseStore): def get_ops_levels(self, room_id): return self.runInteraction( + "get_ops_levels", self._get_ops_levels, room_id, ) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 0aa979c9f..e08acd640 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -59,6 +59,7 @@ class StateStore(SQLBaseStore): def store_state_groups(self, event): return self.runInteraction( + "store_state_groups", self._store_state_groups_txn, event ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index d61f90993..8f7f61d29 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -309,7 +309,10 @@ class StreamStore(SQLBaseStore): defer.returnValue(ret) def get_room_events_max_id(self): - return self.runInteraction(self._get_room_events_max_id_txn) + return self.runInteraction( + "get_room_events_max_id", + self._get_room_events_max_id_txn + ) def _get_room_events_max_id_txn(self, txn): txn.execute( diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 2ba8e30ef..908014d38 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -42,6 +42,7 @@ class TransactionStore(SQLBaseStore): """ return self.runInteraction( + "get_received_txn_response", self._get_received_txn_response, transaction_id, origin ) @@ -73,6 +74,7 @@ class TransactionStore(SQLBaseStore): """ return self.runInteraction( + "set_received_txn_response", self._set_received_txn_response, transaction_id, origin, code, response_dict ) @@ -106,6 +108,7 @@ class TransactionStore(SQLBaseStore): """ return self.runInteraction( + "prep_send_transaction", self._prep_send_transaction, transaction_id, destination, origin_server_ts, pdu_list ) @@ -161,6 +164,7 @@ class TransactionStore(SQLBaseStore): response_json (str) """ return self.runInteraction( + "delivered_txn", self._delivered_txn, transaction_id, destination, code, response_dict ) @@ -186,6 +190,7 @@ class TransactionStore(SQLBaseStore): list: A list of `ReceivedTransactionsTable.EntryType` """ return self.runInteraction( + "get_transactions_after", self._get_transactions_after, transaction_id, destination ) @@ -216,6 +221,7 @@ class TransactionStore(SQLBaseStore): list: A list of PduTuple """ return self.runInteraction( + "get_pdus_after_transaction", self._get_pdus_after_transaction, transaction_id, destination ) From 2d1dfb3b34583a4de7e1e53f685c2564a7fc731f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 16:42:35 +0000 Subject: [PATCH 023/265] Begin implementing all the PDU storage stuff in Events land --- synapse/api/events/__init__.py | 4 +- synapse/federation/pdu_codec.py | 11 +- synapse/storage/__init__.py | 72 +++++++--- synapse/storage/_base.py | 53 +++++--- synapse/storage/event_federation.py | 143 ++++++++++++++++++++ synapse/storage/schema/event_edges.sql | 51 +++++++ synapse/storage/schema/event_signatures.sql | 65 +++++++++ synapse/storage/schema/im.sql | 1 + synapse/storage/signatures.py | 127 +++++++++++++++++ 9 files changed, 485 insertions(+), 42 deletions(-) create mode 100644 synapse/storage/event_federation.py create mode 100644 synapse/storage/schema/event_edges.sql create mode 100644 synapse/storage/schema/event_signatures.sql diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index a5a55742e..b855811b9 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -71,7 +71,9 @@ class SynapseEvent(JsonEncodedObject): "outlier", "power_level", "redacted", - "prev_pdus", + "prev_events", + "hashes", + "signatures", ] required_keys = [ diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 991aae2a5..2cd591410 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -47,7 +47,10 @@ class PduCodec(object): kwargs["event_id"] = encode_event_id(pdu.pdu_id, pdu.origin) kwargs["room_id"] = pdu.context kwargs["etype"] = pdu.pdu_type - kwargs["prev_pdus"] = pdu.prev_pdus + kwargs["prev_events"] = [ + encode_event_id(i, o) + for i, o in pdu.prev_pdus + ] if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): kwargs["prev_state"] = encode_event_id( @@ -78,8 +81,8 @@ class PduCodec(object): d["context"] = event.room_id d["pdu_type"] = event.type - if hasattr(event, "prev_pdus"): - d["prev_pdus"] = event.prev_pdus + if hasattr(event, "prev_events"): + d["prev_pdus"] = [decode_event_id(e) for e in event.prev_events] if hasattr(event, "prev_state"): d["prev_state_id"], d["prev_state_origin"] = ( @@ -92,7 +95,7 @@ class PduCodec(object): kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() - if k not in ["event_id", "room_id", "type"] + if k not in ["event_id", "room_id", "type", "prev_events"] }) if "origin_server_ts" not in kwargs: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a50e19349..678de0cf5 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -40,6 +40,7 @@ from .stream import StreamStore from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore +from .event_federation import EventFederationStore from .state import StateStore from .signatures import SignatureStore @@ -69,6 +70,7 @@ SCHEMAS = [ "redactions", "state", "signatures", + "event_edges", ] @@ -83,10 +85,12 @@ class _RollbackButIsFineException(Exception): """ pass + class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, PduStore, StatePduStore, TransactionStore, - DirectoryStore, KeyStore, StateStore, SignatureStore): + DirectoryStore, KeyStore, StateStore, SignatureStore, + EventFederationStore, ): def __init__(self, hs): super(DataStore, self).__init__(hs) @@ -230,6 +234,10 @@ class DataStore(RoomMemberStore, RoomStore, elif event.type == RoomRedactionEvent.TYPE: self._store_redaction(txn, event) + outlier = False + if hasattr(event, "outlier"): + outlier = event.outlier + vals = { "topological_ordering": event.depth, "event_id": event.event_id, @@ -237,20 +245,20 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "content": json.dumps(event.content), "processed": True, + "outlier": outlier, + "depth": event.depth, } if stream_ordering is not None: vals["stream_ordering"] = stream_ordering - if hasattr(event, "outlier"): - vals["outlier"] = event.outlier - else: - vals["outlier"] = False - unrec = { k: v for k, v in event.get_full_dict().items() - if k not in vals.keys() and k not in ["redacted", "redacted_because"] + if k not in vals.keys() and k not in [ + "redacted", "redacted_because", "signatures", "hashes", + "prev_events", + ] } vals["unrecognized_keys"] = json.dumps(unrec) @@ -264,6 +272,14 @@ class DataStore(RoomMemberStore, RoomStore, ) raise _RollbackButIsFineException("_persist_event") + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + self._store_state_groups_txn(txn, event) is_state = hasattr(event, "state_key") and event.state_key is not None @@ -291,6 +307,28 @@ class DataStore(RoomMemberStore, RoomStore, } ) + signatures = event.signatures.get(event.origin, {}) + + for key_id, signature_base64 in signatures.items(): + signature_bytes = decode_base64(signature_base64) + self._store_event_origin_signature_txn( + txn, event.event_id, key_id, signature_bytes, + ) + + for prev_event_id, prev_hashes in event.prev_events: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_event_hash_txn( + txn, event.event_id, prev_event_id, alg, hash_bytes + ) + + (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) + self._store_pdu_reference_hash_txn( + txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes + ) + + self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) + def _store_redaction(self, txn, event): txn.execute( "INSERT OR IGNORE INTO redactions " @@ -373,7 +411,7 @@ class DataStore(RoomMemberStore, RoomStore, """ def _snapshot(txn): membership_state = self._get_room_member(txn, user_id, room_id) - prev_pdus = self._get_latest_pdus_in_context( + prev_events = self._get_latest_events_in_room( txn, room_id ) @@ -388,7 +426,7 @@ class DataStore(RoomMemberStore, RoomStore, store=self, room_id=room_id, user_id=user_id, - prev_pdus=prev_pdus, + prev_events=prev_events, membership_state=membership_state, state_type=state_type, state_key=state_key, @@ -404,7 +442,7 @@ class Snapshot(object): store (DataStore): The datastore. room_id (RoomId): The room of the snapshot. user_id (UserId): The user this snapshot is for. - prev_pdus (list): The list of PDU ids this snapshot is after. + prev_events (list): The list of event ids this snapshot is after. membership_state (RoomMemberEvent): The current state of the user in the room. state_type (str, optional): State type captured by the snapshot @@ -413,29 +451,29 @@ class Snapshot(object): the previous value of the state type and key in the room. """ - def __init__(self, store, room_id, user_id, prev_pdus, + def __init__(self, store, room_id, user_id, prev_events, membership_state, state_type=None, state_key=None, prev_state_pdu=None): self.store = store self.room_id = room_id self.user_id = user_id - self.prev_pdus = prev_pdus + self.prev_events = prev_events self.membership_state = membership_state self.state_type = state_type self.state_key = state_key self.prev_state_pdu = prev_state_pdu def fill_out_prev_events(self, event): - if hasattr(event, "prev_pdus"): + if hasattr(event, "prev_events"): return - event.prev_pdus = [ + event.prev_events = [ (p_id, origin, hashes) - for p_id, origin, hashes, _ in self.prev_pdus + for p_id, origin, hashes, _ in self.prev_events ] - if self.prev_pdus: - event.depth = max([int(v) for _, _, _, v in self.prev_pdus]) + 1 + if self.prev_events: + event.depth = max([int(v) for _, _, _, v in self.prev_events]) + 1 else: event.depth = 0 diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 119221697..30732caa8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -193,7 +193,6 @@ class SQLBaseStore(object): table, keyvalues, retcols=retcols, allow_none=allow_none ) - @defer.inlineCallbacks def _simple_select_one_onecol(self, table, keyvalues, retcol, allow_none=False): """Executes a SELECT query on the named table, which is expected to @@ -204,19 +203,41 @@ class SQLBaseStore(object): keyvalues : dict of column names and values to select the row with retcol : string giving the name of the column to return """ - ret = yield self._simple_select_one( + return self.runInteraction( + "_simple_select_one_onecol_txn", + self._simple_select_one_onecol_txn, + table, keyvalues, retcol, allow_none=allow_none, + ) + + def _simple_select_one_onecol_txn(self, txn, table, keyvalues, retcol, + allow_none=False): + ret = self._simple_select_onecol_txn( + txn, table=table, keyvalues=keyvalues, - retcols=[retcol], - allow_none=allow_none + retcols=retcol, ) if ret: - defer.returnValue(ret[retcol]) + return ret[retcol] else: - defer.returnValue(None) + if allow_none: + return None + else: + raise StoreError(404, "No row found") + + def _simple_select_onecol_txn(self, txn, table, keyvalues, retcol): + sql = "SELECT %(retcol)s FROM %(table)s WHERE %(where)s" % { + "retcol": retcol, + "table": table, + "where": " AND ".join("%s = ?" % k for k in keyvalues.keys()), + } + + txn.execute(sql, keyvalues.values()) + + return [r[0] for r in txn.fetchall()] + - @defer.inlineCallbacks def _simple_select_onecol(self, table, keyvalues, retcol): """Executes a SELECT query on the named table, which returns a list comprising of the values of the named column from the selected rows. @@ -229,19 +250,11 @@ class SQLBaseStore(object): Returns: Deferred: Results in a list """ - sql = "SELECT %(retcol)s FROM %(table)s WHERE %(where)s" % { - "retcol": retcol, - "table": table, - "where": " AND ".join("%s = ?" % k for k in keyvalues.keys()), - } - - def func(txn): - txn.execute(sql, keyvalues.values()) - return txn.fetchall() - - res = yield self.runInteraction("_simple_select_onecol", func) - - defer.returnValue([r[0] for r in res]) + return self.runInteraction( + "_simple_select_onecol", + self._simple_select_onecol_txn, + table, keyvalues, retcol + ) def _simple_select_list(self, table, keyvalues, retcols): """Executes a SELECT query on the named table, which may return zero or diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py new file mode 100644 index 000000000..27ad9aea4 --- /dev/null +++ b/synapse/storage/event_federation.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore +from twisted.internet import defer + +import logging + + +logger = logging.getLogger(__name__) + + +class EventFederationStore(SQLBaseStore): + + def _get_latest_events_in_room(self, txn, room_id): + self._simple_select_onecol_txn( + txn, + table="event_forward_extremities", + keyvalues={ + "room_id": room_id, + }, + retcol="event_id", + ) + + results = [] + for pdu_id, origin, depth in txn.fetchall(): + hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin) + sha256_bytes = hashes["sha256"] + prev_hashes = {"sha256": encode_base64(sha256_bytes)} + results.append((pdu_id, origin, prev_hashes, depth)) + + def _get_min_depth_interaction(self, txn, room_id): + min_depth = self._simple_select_one_onecol_txn( + txn, + table="room_depth", + keyvalues={"room_id": room_id,}, + retcol="min_depth", + allow_none=True, + ) + + return int(min_depth) if min_depth is not None else None + + def _update_min_depth_for_room_txn(self, txn, room_id, depth): + min_depth = self._get_min_depth_interaction(txn, room_id) + + do_insert = depth < min_depth if min_depth else True + + if do_insert: + self._simple_insert_txn( + txn, + table="room_depth", + values={ + "room_id": room_id, + "min_depth": depth, + }, + or_replace=True, + ) + + def _handle_prev_events(self, txn, outlier, event_id, prev_events, + room_id): + for e_id in prev_events: + # TODO (erikj): This could be done as a bulk insert + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event_id, + "prev_event": e_id, + "room_id": room_id, + } + ) + + # Update the extremities table if this is not an outlier. + if not outlier: + for e_id in prev_events: + # TODO (erikj): This could be done as a bulk insert + self._simple_delete_txn( + txn, + table="event_forward_extremities", + keyvalues={ + "event_id": e_id, + "room_id": room_id, + } + ) + + + + # We only insert as a forward extremity the new pdu if there are no + # other pdus that reference it as a prev pdu + query = ( + "INSERT INTO %(table)s (event_id, room_id) " + "SELECT ?, ? WHERE NOT EXISTS (" + "SELECT 1 FROM %(event_edges)s WHERE " + "prev_event_id = ? " + ")" + ) % { + "table": "event_forward_extremities", + "event_edges": "event_edges", + } + + logger.debug("query: %s", query) + + txn.execute(query, (event_id, room_id, event_id)) + + # Insert all the prev_pdus as a backwards thing, they'll get + # deleted in a second if they're incorrect anyway. + for e_id in prev_events: + # TODO (erikj): This could be done as a bulk insert + self._simple_insert_txn( + txn, + table="event_backward_extremities", + values={ + "event_id": e_id, + "room_id": room_id, + } + ) + + # Also delete from the backwards extremities table all ones that + # reference pdus that we have already seen + query = ( + "DELETE FROM %(event_back)s as b WHERE EXISTS (" + "SELECT 1 FROM %(events)s AS events " + "WHERE " + "b.event_id = events.event_id " + "AND not events.outlier " + ")" + ) % { + "event_back": "event_backward_extremities", + "events": "events", + } + txn.execute(query) \ No newline at end of file diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql new file mode 100644 index 000000000..6a28314ec --- /dev/null +++ b/synapse/storage/schema/event_edges.sql @@ -0,0 +1,51 @@ + +CREATE TABLE IF NOT EXISTS event_forward_extremities( + event_id TEXT, + room_id TEXT, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); +-- + +CREATE TABLE IF NOT EXISTS event_backward_extremities( + event_id TEXT, + room_id TEXT, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); +-- + +CREATE TABLE IF NOT EXISTS event_edges( + event_id TEXT, + prev_event_id TEXT, + room_id TEXT, + CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); +CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); +-- + + +CREATE TABLE IF NOT EXISTS room_depth( + room_id TEXT, + min_depth INTEGER, + CONSTRAINT uniqueness UNIQUE (room_id) +); + +CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); +-- + +create TABLE IF NOT EXISTS event_destinations( + event_id TEXT, + destination TEXT, + delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered + CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); +-- \ No newline at end of file diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/event_signatures.sql new file mode 100644 index 000000000..5491c7ece --- /dev/null +++ b/synapse/storage/schema/event_signatures.sql @@ -0,0 +1,65 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_content_hashes ( + event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_reference_hashes ( + event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_origin_signatures ( + event_id TEXT, + origin TEXT, + key_id TEXT, + signature BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, key_id) +); + +CREATE INDEX IF NOT EXISTS event_origin_signatures_id ON event_origin_signatures ( + event_id +); + + +CREATE TABLE IF NOT EXISTS event_edge_hashes( + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE ( + event_id, prev_event_id, algorithm + ) +); + +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( + event_id +); diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 3aa83f5c8..8d6f65599 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -23,6 +23,7 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, + depth INTEGER DEFAULT 0 NOT NULL, CONSTRAINT ev_uniq UNIQUE (event_id) ); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 82be946d3..b8f8fd44c 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -153,3 +153,130 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }) + + ## Events ## + + def _get_event_content_hashes_txn(self, txn, event_id): + """Get all the hashes for a given Event. + Args: + txn (cursor): + event_id (str): Id for the Event. + Returns: + A dict of algorithm -> hash. + """ + query = ( + "SELECT algorithm, hash" + " FROM event_content_hashes" + " WHERE event_id = ?" + ) + txn.execute(query, (event_id, )) + return dict(txn.fetchall()) + + def _store_event_content_hash_txn(self, txn, event_id, algorithm, + hash_bytes): + """Store a hash for a Event + Args: + txn (cursor): + event_id (str): Id for the Event. + algorithm (str): Hashing algorithm. + hash_bytes (bytes): Hash function output bytes. + """ + self._simple_insert_txn(txn, "event_content_hashes", { + "event_id": event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) + + def _get_event_reference_hashes_txn(self, txn, event_id): + """Get all the hashes for a given PDU. + Args: + txn (cursor): + event_id (str): Id for the Event. + Returns: + A dict of algorithm -> hash. + """ + query = ( + "SELECT algorithm, hash" + " FROM event_reference_hashes" + " WHERE event_id = ?" + ) + txn.execute(query, (event_id, )) + return dict(txn.fetchall()) + + def _store_event_reference_hash_txn(self, txn, event_id, algorithm, + hash_bytes): + """Store a hash for a PDU + Args: + txn (cursor): + event_id (str): Id for the Event. + algorithm (str): Hashing algorithm. + hash_bytes (bytes): Hash function output bytes. + """ + self._simple_insert_txn(txn, "event_reference_hashes", { + "event_id": event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) + + + def _get_event_origin_signatures_txn(self, txn, event_id): + """Get all the signatures for a given PDU. + Args: + txn (cursor): + event_id (str): Id for the Event. + Returns: + A dict of key_id -> signature_bytes. + """ + query = ( + "SELECT key_id, signature" + " FROM event_origin_signatures" + " WHERE event_id = ? " + ) + txn.execute(query, (event_id, )) + return dict(txn.fetchall()) + + def _store_event_origin_signature_txn(self, txn, event_id, origin, key_id, + signature_bytes): + """Store a signature from the origin server for a PDU. + Args: + txn (cursor): + event_id (str): Id for the Event. + origin (str): origin of the Event. + key_id (str): Id for the signing key. + signature (bytes): The signature. + """ + self._simple_insert_txn(txn, "event_origin_signatures", { + "event_id": event_id, + "origin": origin, + "key_id": key_id, + "signature": buffer(signature_bytes), + }) + + def _get_prev_event_hashes_txn(self, txn, event_id): + """Get all the hashes for previous PDUs of a PDU + Args: + txn (cursor): + event_id (str): Id for the Event. + Returns: + dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. + """ + query = ( + "SELECT prev_event_id, algorithm, hash" + " FROM event_edge_hashes" + " WHERE event_id = ?" + ) + txn.execute(query, (event_id, )) + results = {} + for prev_event_id, algorithm, hash_bytes in txn.fetchall(): + hashes = results.setdefault(prev_event_id, {}) + hashes[algorithm] = hash_bytes + return results + + def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id, + algorithm, hash_bytes): + self._simple_insert_txn(txn, "event_edge_hashes", { + "event_id": event_id, + "prev_event_id": prev_event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) \ No newline at end of file From a10c2ec88d98abe035a60ab0027c1914d4ad7d77 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Oct 2014 17:15:32 +0000 Subject: [PATCH 024/265] Don't reference PDU when persisting event --- synapse/storage/__init__.py | 5 +++-- synapse/storage/event_federation.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 678de0cf5..f89e51869 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -322,9 +322,10 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, prev_event_id, alg, hash_bytes ) + # TODO (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) - self._store_pdu_reference_hash_txn( - txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes + self._store_event_reference_hash_txn( + txn, event.event_id, ref_alg, ref_hash_bytes ) self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 27ad9aea4..7688fc550 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import SQLBaseStore -from twisted.internet import defer +from syutil.base64util import encode_base64 import logging @@ -36,7 +36,7 @@ class EventFederationStore(SQLBaseStore): results = [] for pdu_id, origin, depth in txn.fetchall(): - hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin) + hashes = self._get_prev_event_hashes_txn(txn, pdu_id, origin) sha256_bytes = hashes["sha256"] prev_hashes = {"sha256": encode_base64(sha256_bytes)} results.append((pdu_id, origin, prev_hashes, depth)) From e7858b6d7ef37849a3d2d5004743cdd21ec330a8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Oct 2014 16:59:24 +0000 Subject: [PATCH 025/265] Start filling out and using new events tables --- synapse/federation/pdu_codec.py | 12 +++- synapse/handlers/_base.py | 4 ++ synapse/handlers/federation.py | 96 +++++++++++++++----------- synapse/state.py | 11 +-- synapse/storage/__init__.py | 45 +++++++----- synapse/storage/_base.py | 33 +++++++-- synapse/storage/event_federation.py | 49 ++++++++----- synapse/storage/schema/event_edges.sql | 8 +-- 8 files changed, 162 insertions(+), 96 deletions(-) diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 2cd591410..dccbccb85 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -48,8 +48,8 @@ class PduCodec(object): kwargs["room_id"] = pdu.context kwargs["etype"] = pdu.pdu_type kwargs["prev_events"] = [ - encode_event_id(i, o) - for i, o in pdu.prev_pdus + (encode_event_id(i, o), s) + for i, o, s in pdu.prev_pdus ] if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): @@ -82,7 +82,13 @@ class PduCodec(object): d["pdu_type"] = event.type if hasattr(event, "prev_events"): - d["prev_pdus"] = [decode_event_id(e) for e in event.prev_events] + def f(e, s): + i, o = decode_event_id(e, self.server_name) + return i, o, s + d["prev_pdus"] = [ + f(e, s) + for e, s in event.prev_events + ] if hasattr(event, "prev_state"): d["prev_state_id"], d["prev_state_origin"] = ( diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index cd6c35f19..787a01efc 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -16,6 +16,8 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError +from synapse.util.async import run_on_reactor + class BaseHandler(object): def __init__(self, hs): @@ -45,6 +47,8 @@ class BaseHandler(object): @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], extra_users=[], suppress_auth=False): + yield run_on_reactor() + snapshot.fill_out_prev_events(event) yield self.state_handler.annotate_state_groups(event) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b575986fc..5f86ed03f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -22,6 +22,7 @@ from synapse.api.constants import Membership from synapse.util.logutils import log_function from synapse.federation.pdu_codec import PduCodec, encode_event_id from synapse.api.errors import SynapseError +from synapse.util.async import run_on_reactor from twisted.internet import defer, reactor @@ -81,6 +82,8 @@ class FederationHandler(BaseHandler): processing. """ + yield run_on_reactor() + pdu = self.pdu_codec.pdu_from_event(event) if not hasattr(pdu, "destinations") or not pdu.destinations: @@ -102,6 +105,8 @@ class FederationHandler(BaseHandler): self.room_queues[event.room_id].append(pdu) return + logger.debug("Processing event: %s", event.event_id) + if state: state = [self.pdu_codec.event_from_pdu(p) for p in state] @@ -216,58 +221,65 @@ class FederationHandler(BaseHandler): assert(event.state_key == joinee) assert(event.room_id == room_id) + event.outlier = False + self.room_queues[room_id] = [] - event.event_id = self.event_factory.create_event_id() - event.content = content - - state = yield self.replication_layer.send_join( - target_host, - self.pdu_codec.pdu_from_event(event) - ) - - state = [self.pdu_codec.event_from_pdu(p) for p in state] - - logger.debug("do_invite_join state: %s", state) - - is_new_state = yield self.state_handler.annotate_state_groups( - event, - state=state - ) - try: - yield self.store.store_room( - room_id=room_id, - room_creator_user_id="", - is_public=False - ) - except: - # FIXME - pass + event.event_id = self.event_factory.create_event_id() + event.content = content - for e in state: - # FIXME: Auth these. - is_new_state = yield self.state_handler.annotate_state_groups( - e, + state = yield self.replication_layer.send_join( + target_host, + self.pdu_codec.pdu_from_event(event) ) + state = [self.pdu_codec.event_from_pdu(p) for p in state] + + logger.debug("do_invite_join state: %s", state) + + is_new_state = yield self.state_handler.annotate_state_groups( + event, + state=state + ) + + logger.debug("do_invite_join event: %s", event) + + try: + yield self.store.store_room( + room_id=room_id, + room_creator_user_id="", + is_public=False + ) + except: + # FIXME + pass + + for e in state: + # FIXME: Auth these. + e.outlier = True + + yield self.state_handler.annotate_state_groups( + e, + ) + + yield self.store.persist_event( + e, + backfilled=False, + is_new_state=False + ) + yield self.store.persist_event( - e, + event, backfilled=False, - is_new_state=False + is_new_state=is_new_state ) + finally: + room_queue = self.room_queues[room_id] + del self.room_queues[room_id] - yield self.store.persist_event( - event, - backfilled=False, - is_new_state=is_new_state - ) - - room_queue = self.room_queues[room_id] - del self.room_queues[room_id] - - for p in room_queue: - yield self.on_receive_pdu(p, backfilled=False) + for p in room_queue: + yield self.on_receive_pdu(p, backfilled=False) defer.returnValue(True) diff --git a/synapse/state.py b/synapse/state.py index cc6a7db96..993c4f18d 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -143,7 +143,9 @@ class StateHandler(object): defer.returnValue(False) return - new_state = yield self.resolve_state_groups(event.prev_events) + new_state = yield self.resolve_state_groups( + [e for e, _ in event.prev_events] + ) event.old_state_events = copy.deepcopy(new_state) @@ -157,12 +159,11 @@ class StateHandler(object): @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): - # FIXME: HACK! - pdus = yield self.store.get_latest_pdus_in_context(room_id) + events = yield self.store.get_latest_events_in_room(room_id) event_ids = [ - encode_event_id(pdu_id, origin) - for pdu_id, origin, _ in pdus + e_id + for e_id, _ in events ] res = yield self.resolve_state_groups(event_ids) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f89e51869..d75c36683 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -71,6 +71,7 @@ SCHEMAS = [ "state", "signatures", "event_edges", + "event_signatures", ] @@ -134,7 +135,8 @@ class DataStore(RoomMemberStore, RoomStore, "type", "room_id", "content", - "unrecognized_keys" + "unrecognized_keys", + "depth", ], allow_none=allow_none, ) @@ -263,7 +265,12 @@ class DataStore(RoomMemberStore, RoomStore, vals["unrecognized_keys"] = json.dumps(unrec) try: - self._simple_insert_txn(txn, "events", vals) + self._simple_insert_txn( + txn, + "events", + vals, + or_replace=(not outlier), + ) except: logger.warn( "Failed to persist, probably duplicate: %s", @@ -307,13 +314,14 @@ class DataStore(RoomMemberStore, RoomStore, } ) - signatures = event.signatures.get(event.origin, {}) + if hasattr(event, "signatures"): + signatures = event.signatures.get(event.origin, {}) - for key_id, signature_base64 in signatures.items(): - signature_bytes = decode_base64(signature_base64) - self._store_event_origin_signature_txn( - txn, event.event_id, key_id, signature_bytes, - ) + for key_id, signature_base64 in signatures.items(): + signature_bytes = decode_base64(signature_base64) + self._store_event_origin_signature_txn( + txn, event.event_id, event.origin, key_id, signature_bytes, + ) for prev_event_id, prev_hashes in event.prev_events: for alg, hash_base64 in prev_hashes.items(): @@ -323,10 +331,10 @@ class DataStore(RoomMemberStore, RoomStore, ) # TODO - (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) - self._store_event_reference_hash_txn( - txn, event.event_id, ref_alg, ref_hash_bytes - ) + # (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) + # self._store_event_reference_hash_txn( + # txn, event.event_id, ref_alg, ref_hash_bytes + # ) self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) @@ -412,9 +420,7 @@ class DataStore(RoomMemberStore, RoomStore, """ def _snapshot(txn): membership_state = self._get_room_member(txn, user_id, room_id) - prev_events = self._get_latest_events_in_room( - txn, room_id - ) + prev_events = self._get_latest_events_in_room(txn, room_id) if state_type is not None and state_key is not None: prev_state_pdu = self._get_current_state_pdu( @@ -469,12 +475,12 @@ class Snapshot(object): return event.prev_events = [ - (p_id, origin, hashes) - for p_id, origin, hashes, _ in self.prev_events + (event_id, hashes) + for event_id, hashes, _ in self.prev_events ] if self.prev_events: - event.depth = max([int(v) for _, _, _, v in self.prev_events]) + 1 + event.depth = max([int(v) for _, _, v in self.prev_events]) + 1 else: event.depth = 0 @@ -533,9 +539,10 @@ def prepare_database(db_conn): db_conn.commit() else: - sql_script = "BEGIN TRANSACTION;" + sql_script = "BEGIN TRANSACTION;\n" for sql_loc in SCHEMAS: sql_script += read_schema(sql_loc) + sql_script += "\n" sql_script += "COMMIT TRANSACTION;" c.executescript(sql_script) db_conn.commit() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 30732caa8..464b12f03 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,10 +19,12 @@ from twisted.internet import defer from synapse.api.errors import StoreError from synapse.api.events.utils import prune_event from synapse.util.logutils import log_function +from syutil.base64util import encode_base64 import collections import copy import json +import sys import time @@ -67,6 +69,9 @@ class LoggingTransaction(object): return self.txn.execute( sql, *args, **kwargs ) + except: + logger.exception("[SQL FAIL] {%s}", self.name) + raise finally: end = time.clock() * 1000 sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) @@ -85,14 +90,20 @@ class SQLBaseStore(object): """Wraps the .runInteraction() method on the underlying db_pool.""" def inner_func(txn, *args, **kwargs): start = time.clock() * 1000 - txn_id = str(SQLBaseStore._TXN_ID) - SQLBaseStore._TXN_ID += 1 + txn_id = SQLBaseStore._TXN_ID - name = "%s-%s" % (desc, txn_id, ) + # We don't really need these to be unique, so lets stop it from + # growing really large. + self._TXN_ID = (self._TXN_ID + 1) % (sys.maxint - 1) + + name = "%s-%x" % (desc, txn_id, ) transaction_logger.debug("[TXN START] {%s}", name) try: return func(LoggingTransaction(txn, name), *args, **kwargs) + except: + logger.exception("[TXN FAIL] {%s}", name) + raise finally: end = time.clock() * 1000 transaction_logger.debug( @@ -189,7 +200,6 @@ class SQLBaseStore(object): statement returns no rows """ return self._simple_selectupdate_one( - "_simple_select_one", table, keyvalues, retcols=retcols, allow_none=allow_none ) @@ -215,11 +225,11 @@ class SQLBaseStore(object): txn, table=table, keyvalues=keyvalues, - retcols=retcol, + retcol=retcol, ) if ret: - return ret[retcol] + return ret[0] else: if allow_none: return None @@ -434,6 +444,17 @@ class SQLBaseStore(object): sql = "SELECT * FROM events WHERE event_id = ?" for ev in events: + signatures = self._get_event_origin_signatures_txn( + txn, ev.event_id, + ) + + ev.signatures = { + k: encode_base64(v) for k, v in signatures.items() + } + + prev_events = self._get_latest_events_in_room(txn, ev.room_id) + ev.prev_events = [(e_id, s,) for e_id, s, _ in prev_events] + if hasattr(ev, "prev_state"): # Load previous state_content. # TODO: Should we be pulling this out above? diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7688fc550..5f94c3181 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -24,6 +24,13 @@ logger = logging.getLogger(__name__) class EventFederationStore(SQLBaseStore): + def get_latest_events_in_room(self, room_id): + return self.runInteraction( + "get_latest_events_in_room", + self._get_latest_events_in_room, + room_id, + ) + def _get_latest_events_in_room(self, txn, room_id): self._simple_select_onecol_txn( txn, @@ -34,12 +41,25 @@ class EventFederationStore(SQLBaseStore): retcol="event_id", ) + sql = ( + "SELECT e.event_id, e.depth FROM events as e " + "INNER JOIN event_forward_extremities as f " + "ON e.event_id = f.event_id " + "WHERE f.room_id = ?" + ) + + txn.execute(sql, (room_id, )) + results = [] - for pdu_id, origin, depth in txn.fetchall(): - hashes = self._get_prev_event_hashes_txn(txn, pdu_id, origin) - sha256_bytes = hashes["sha256"] - prev_hashes = {"sha256": encode_base64(sha256_bytes)} - results.append((pdu_id, origin, prev_hashes, depth)) + for event_id, depth in txn.fetchall(): + hashes = self._get_prev_event_hashes_txn(txn, event_id) + prev_hashes = { + k: encode_base64(v) for k, v in hashes.items() + if k == "sha256" + } + results.append((event_id, prev_hashes, depth)) + + return results def _get_min_depth_interaction(self, txn, room_id): min_depth = self._simple_select_one_onecol_txn( @@ -70,21 +90,21 @@ class EventFederationStore(SQLBaseStore): def _handle_prev_events(self, txn, outlier, event_id, prev_events, room_id): - for e_id in prev_events: + for e_id, _ in prev_events: # TODO (erikj): This could be done as a bulk insert self._simple_insert_txn( txn, table="event_edges", values={ "event_id": event_id, - "prev_event": e_id, + "prev_event_id": e_id, "room_id": room_id, } ) # Update the extremities table if this is not an outlier. if not outlier: - for e_id in prev_events: + for e_id, _ in prev_events: # TODO (erikj): This could be done as a bulk insert self._simple_delete_txn( txn, @@ -116,7 +136,7 @@ class EventFederationStore(SQLBaseStore): # Insert all the prev_pdus as a backwards thing, they'll get # deleted in a second if they're incorrect anyway. - for e_id in prev_events: + for e_id, _ in prev_events: # TODO (erikj): This could be done as a bulk insert self._simple_insert_txn( txn, @@ -130,14 +150,11 @@ class EventFederationStore(SQLBaseStore): # Also delete from the backwards extremities table all ones that # reference pdus that we have already seen query = ( - "DELETE FROM %(event_back)s as b WHERE EXISTS (" - "SELECT 1 FROM %(events)s AS events " + "DELETE FROM event_backward_extremities WHERE EXISTS (" + "SELECT 1 FROM events " "WHERE " - "b.event_id = events.event_id " + "event_backward_extremities.event_id = events.event_id " "AND not events.outlier " ")" - ) % { - "event_back": "event_backward_extremities", - "events": "events", - } + ) txn.execute(query) \ No newline at end of file diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index 6a28314ec..e5f768c70 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); --- + CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id TEXT, @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); --- + CREATE TABLE IF NOT EXISTS event_edges( event_id TEXT, @@ -28,7 +28,6 @@ CREATE TABLE IF NOT EXISTS event_edges( CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); --- CREATE TABLE IF NOT EXISTS room_depth( @@ -38,7 +37,7 @@ CREATE TABLE IF NOT EXISTS room_depth( ); CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); --- + create TABLE IF NOT EXISTS event_destinations( event_id TEXT, @@ -48,4 +47,3 @@ create TABLE IF NOT EXISTS event_destinations( ); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); --- \ No newline at end of file From 53216a500d2e0d815581ada87379ecece05b05a5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Oct 2014 17:02:22 +0000 Subject: [PATCH 026/265] Add a run_on_reactor function --- synapse/util/async.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/util/async.py b/synapse/util/async.py index 647ea6142..bf578f8bf 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -21,3 +21,10 @@ def sleep(seconds): d = defer.Deferred() reactor.callLater(seconds, d.callback, seconds) return d + + +def run_on_reactor(): + """ This will cause the rest of the function to be invoked upon the next + iteration of the main loop + """ + return sleep(0) \ No newline at end of file From aa80900a8e8cd9e7305a66cec336a8e150c46651 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Oct 2014 10:11:06 +0000 Subject: [PATCH 027/265] Fix SQL so that accepts we may want to persist events twice. --- synapse/storage/event_federation.py | 8 ++-- synapse/storage/signatures.py | 64 +++++++++++++++++++---------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 5f94c3181..88d09d9ba 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -99,7 +99,8 @@ class EventFederationStore(SQLBaseStore): "event_id": event_id, "prev_event_id": e_id, "room_id": room_id, - } + }, + or_ignore=True, ) # Update the extremities table if this is not an outlier. @@ -120,7 +121,7 @@ class EventFederationStore(SQLBaseStore): # We only insert as a forward extremity the new pdu if there are no # other pdus that reference it as a prev pdu query = ( - "INSERT INTO %(table)s (event_id, room_id) " + "INSERT OR IGNORE INTO %(table)s (event_id, room_id) " "SELECT ?, ? WHERE NOT EXISTS (" "SELECT 1 FROM %(event_edges)s WHERE " "prev_event_id = ? " @@ -144,7 +145,8 @@ class EventFederationStore(SQLBaseStore): values={ "event_id": e_id, "room_id": room_id, - } + }, + or_ignore=True, ) # Also delete from the backwards extremities table all ones that diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index b8f8fd44c..5e99174fc 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -181,11 +181,16 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(txn, "event_content_hashes", { - "event_id": event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) + self._simple_insert_txn( + txn, + "event_content_hashes", + { + "event_id": event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }, + or_ignore=True, + ) def _get_event_reference_hashes_txn(self, txn, event_id): """Get all the hashes for a given PDU. @@ -212,11 +217,16 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(txn, "event_reference_hashes", { - "event_id": event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) + self._simple_insert_txn( + txn, + "event_reference_hashes", + { + "event_id": event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }, + or_ignore=True, + ) def _get_event_origin_signatures_txn(self, txn, event_id): @@ -245,12 +255,17 @@ class SignatureStore(SQLBaseStore): key_id (str): Id for the signing key. signature (bytes): The signature. """ - self._simple_insert_txn(txn, "event_origin_signatures", { - "event_id": event_id, - "origin": origin, - "key_id": key_id, - "signature": buffer(signature_bytes), - }) + self._simple_insert_txn( + txn, + "event_origin_signatures", + { + "event_id": event_id, + "origin": origin, + "key_id": key_id, + "signature": buffer(signature_bytes), + }, + or_ignore=True, + ) def _get_prev_event_hashes_txn(self, txn, event_id): """Get all the hashes for previous PDUs of a PDU @@ -274,9 +289,14 @@ class SignatureStore(SQLBaseStore): def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id, algorithm, hash_bytes): - self._simple_insert_txn(txn, "event_edge_hashes", { - "event_id": event_id, - "prev_event_id": prev_event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) \ No newline at end of file + self._simple_insert_txn( + txn, + "event_edge_hashes", + { + "event_id": event_id, + "prev_event_id": prev_event_id, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }, + or_ignore=True, + ) \ No newline at end of file From d5aa9655223d5a57b76194b155a6366791dd084c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 30 Oct 2014 11:14:29 +0000 Subject: [PATCH 028/265] SYWEB-12: Add a 'Room Info' button which displays all state content. Content displayed in a modal dialog. Currently only read-only. --- webclient/app-filter.js | 11 +++++++++++ webclient/room/room-controller.js | 19 +++++++++++++++++++ webclient/room/room.html | 31 +++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/webclient/app-filter.js b/webclient/app-filter.js index 39ea1d637..f19db4141 100644 --- a/webclient/app-filter.js +++ b/webclient/app-filter.js @@ -76,6 +76,17 @@ angular.module('matrixWebClient') return filtered; }; }) +.filter('stateEventsFilter', function($sce) { + return function(events) { + var filtered = {}; + angular.forEach(events, function(value, key) { + if (value && typeof(value.state_key) === "string") { + filtered[key] = value; + } + }); + return filtered; + }; +}) .filter('unsafe', ['$sce', function($sce) { return function(text) { return $sce.trustAsHtml(text); diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 841b5cccd..37f51c4e9 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -1017,6 +1017,15 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) }); }; + $scope.openRoomInfo = function() { + var modalInstance = $modal.open({ + templateUrl: 'roomInfoTemplate.html', + controller: 'RoomInfoController', + size: 'lg', + scope: $scope + }); + }; + }]) .controller('EventInfoController', function($scope, $modalInstance) { console.log("Displaying modal dialog for >>>> " + JSON.stringify($scope.event_selected)); @@ -1026,4 +1035,14 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) console.log("Redact event >> " + JSON.stringify($scope.event_selected)); $modalInstance.close("redact"); }; +}) +.controller('RoomInfoController', function($scope, $modalInstance, $filter) { + console.log("Displaying room info."); + + $scope.submitState = function(eventType, content) { + console.log("Submitting " + eventType + " with " + content); + } + + $scope.dismiss = $modalInstance.dismiss; + }); diff --git a/webclient/room/room.html b/webclient/room/room.html index 38b6d591e..3458e9703 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -13,6 +13,34 @@ + +
[matrix]
@@ -216,6 +244,9 @@ > Video Call +
{{ feedback }} From 8e8bbb00f5a0524873b5a76c833c8a9d553c0327 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 30 Oct 2014 11:22:47 +0000 Subject: [PATCH 029/265] SYWEB-12: Store unknown state events so they are displayed in the Room Info dialog. --- webclient/components/matrix/event-handler-service.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 3b1354cde..6f251eec5 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -564,6 +564,13 @@ function(matrixService, $rootScope, $q, $timeout, mPresence) { handleRedaction(event, isLiveEvent); break; default: + // if it is a state event, then just add it in so it + // displays on the Room Info screen. + if (typeof(event.state_key) === "string") { // incls. 0-len strings + if (event.room_id) { + handleRoomDateEvent(event, isLiveEvent, false); + } + } console.log("Unable to handle event type " + event.type); console.log(JSON.stringify(event, undefined, 4)); break; From 40342af4597c5f52815629836bc37b4798c1d232 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 30 Oct 2014 11:53:28 +0000 Subject: [PATCH 030/265] SYWEB-12: Format room info dialog better. --- webclient/app.css | 25 +++++++++++++++++++++++++ webclient/room/room.html | 30 +++++++++++++----------------- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/webclient/app.css b/webclient/app.css index 20a13aad8..2dc6e8cae 100755 --- a/webclient/app.css +++ b/webclient/app.css @@ -418,6 +418,31 @@ textarea, input { margin-top: 15px; } +/*** Room Info Dialog ***/ + +.room-info { + border-collapse: collapse; + width: 100%; +} + +.room-info-event { + border-bottom: 1pt solid black; +} + +.room-info-event-meta { + padding-top: 1em; + padding-bottom: 1em; +} + +.room-info-event-content { + padding-top: 1em; + padding-bottom: 1em; +} + +.monospace { + font-family: monospace; +} + /*** Participant list ***/ #usersTableWrapper { diff --git a/webclient/room/room.html b/webclient/room/room.html index 3458e9703..65b959fe9 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -15,23 +15,19 @@ + diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index fcbcd7536..ce513990f 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -1018,6 +1018,13 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) }; $scope.openRoomInfo = function() { + var stateFilter = $filter("stateEventsFilter"); + var stateEvents = stateFilter($scope.events.rooms[$scope.room_id]); + // The modal dialog will 2-way bind this field, so we MUST make a deep + // copy of the state events else we will be *actually adjusing our view + // of the world* when fiddling with the JSON!! Apparently parse/stringify + // is faster than jQuery's extend when doing deep copies. + $scope.roomInfoStateEvents = JSON.parse(JSON.stringify(stateEvents)); var modalInstance = $modal.open({ templateUrl: 'roomInfoTemplate.html', controller: 'RoomInfoController', @@ -1040,7 +1047,9 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) console.log("Displaying room info."); $scope.submit = function(event) { - console.error("submit >>> " + JSON.stringify(event)); + if (event.content) { + console.error("submit >>> " + JSON.stringify(event)); + } }; $scope.dismiss = $modalInstance.dismiss; diff --git a/webclient/room/room.html b/webclient/room/room.html index cc79d6b77..627918223 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -16,7 +16,7 @@ + diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 59274bacc..2af1e8ae2 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -15,11 +15,21 @@ limitations under the License. */ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) -.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', - function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall) { +.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'notificationService', + function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, notificationService) { 'use strict'; var MESSAGES_PER_PAGINATION = 30; var THUMBNAIL_SIZE = 320; + + // .html needs this + $scope.containsBingWord = function(content) { + return notificationService.containsBingWord( + matrixService.config().user_id, + matrixService.config().display_name, + matrixService.config().bingWords, + content + ); + }; // Room ids. Computed and resolved in onInit $scope.room_id = undefined; From 946d02536be9e94888bdd399109c474b2cddeac2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 11:45:08 +0000 Subject: [PATCH 048/265] Remove unused functions. --- synapse/state.py | 90 ------------------------------------------------ 1 file changed, 90 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index c514b118a..f7249705c 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -102,35 +102,6 @@ class StateHandler(object): defer.returnValue(True) - @defer.inlineCallbacks - @log_function - def handle_new_state(self, new_pdu): - """ Apply conflict resolution to `new_pdu`. - - This should be called on every new state pdu, regardless of whether or - not there is a conflict. - - This function is safe against the race of it getting called with two - `PDU`s trying to update the same state. - """ - - # This needs to be done in a transaction. - - is_new = yield self._handle_new_state(new_pdu) - - logger.debug("is_new: %s %s %s", is_new, new_pdu.pdu_id, new_pdu.origin) - - if is_new: - yield self.store.update_current_state( - pdu_id=new_pdu.pdu_id, - origin=new_pdu.origin, - context=new_pdu.context, - pdu_type=new_pdu.pdu_type, - state_key=new_pdu.state_key - ) - - defer.returnValue(is_new) - @defer.inlineCallbacks @log_function def annotate_state_groups(self, event, old_state=None): @@ -267,67 +238,6 @@ class StateHandler(object): # event.sender) return event.power_level - @defer.inlineCallbacks - @log_function - def _handle_new_state(self, new_pdu): - tree, missing_branch = yield self.store.get_unresolved_state_tree( - new_pdu - ) - new_branch, current_branch = tree - - logger.debug( - "_handle_new_state new=%s, current=%s", - new_branch, current_branch - ) - - if missing_branch is not None: - # We're missing some PDUs. Fetch them. - # TODO (erikj): Limit this. - missing_prev = tree[missing_branch][-1] - - pdu_id = missing_prev.prev_state_id - origin = missing_prev.prev_state_origin - - is_missing = yield self.store.get_pdu(pdu_id, origin) is None - if not is_missing: - raise Exception("Conflict resolution failed") - - yield self._replication.get_pdu( - destination=missing_prev.origin, - pdu_origin=origin, - pdu_id=pdu_id, - outlier=True - ) - - updated_current = yield self._handle_new_state(new_pdu) - defer.returnValue(updated_current) - - if not current_branch: - # There is no current state - defer.returnValue(True) - return - - n = new_branch[-1] - c = current_branch[-1] - - common_ancestor = n.pdu_id == c.pdu_id and n.origin == c.origin - - if common_ancestor: - # We found a common ancestor! - - if len(current_branch) == 1: - # This is a direct clobber so we can just... - defer.returnValue(True) - - else: - # We didn't find a common ancestor. This is probably fine. - pass - - result = yield self._do_conflict_res( - new_branch, current_branch, common_ancestor - ) - defer.returnValue(result) - @defer.inlineCallbacks def _do_conflict_res(self, new_branch, current_branch, common_ancestor): conflict_res = [ From 20cf0b7aeb42c49bce199206130b5498b0fc293c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 31 Oct 2014 11:54:04 +0000 Subject: [PATCH 049/265] Factor out notification logic. --- .../matrix/event-handler-service.js | 25 +++++++------------ .../components/matrix/notification-service.js | 22 ++++++++++++++-- webclient/room/room-controller.js | 19 +++++++------- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index bcf805fc7..e63584510 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -212,22 +212,15 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService roomTitle = event.room_id; } - var notification = new window.Notification( - displayname + - " (" + roomTitle + ")", - { - "body": message, - "icon": member ? member.avatar_url : undefined - }); - - notification.onclick = function() { - console.log("notification.onclick() room=" + event.room_id); - $rootScope.goToPage('room/' + (event.room_id)); - }; - - $timeout(function() { - notification.close(); - }, 5 * 1000); + notificationService.showNotification( + displayname + " (" + roomTitle + ")", + message, + member ? member.avatar_url : undefined, + function() { + console.log("notification.onclick() room=" + event.room_id); + $rootScope.goToPage('room/' + event.room_id); + } + ); } } } diff --git a/webclient/components/matrix/notification-service.js b/webclient/components/matrix/notification-service.js index 782124884..9a911413c 100644 --- a/webclient/components/matrix/notification-service.js +++ b/webclient/components/matrix/notification-service.js @@ -21,7 +21,7 @@ This service manages notifications: enabling, creating and showing them. This also contains 'bing word' logic. */ angular.module('notificationService', []) -.factory('notificationService', function($rootScope) { +.factory('notificationService', ['$timeout', function($timeout) { var getLocalPartFromUserId = function(user_id) { if (!user_id) { @@ -80,7 +80,25 @@ angular.module('notificationService', []) } } return false; + }, + + showNotification: function(title, body, icon, onclick) { + var notification = new window.Notification( + title, + { + "body": body, + "icon": icon + } + ); + + if (onclick) { + notification.onclick = onclick; + } + + $timeout(function() { + notification.close(); + }, 5 * 1000); } }; -}); +}]); diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 2af1e8ae2..45638acaa 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -201,16 +201,17 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) // Notify when a user joins if ((document.hidden || matrixService.presence.unavailable === mPresence.getState()) && event.state_key !== $scope.state.user_id && "join" === event.membership) { - var notification = new window.Notification( + + notificationService.showNotification( event.content.displayname + - " (" + (matrixService.getRoomIdToAliasMapping(event.room_id) || event.room_id) + ")", // FIXME: don't leak room_ids here - { - "body": event.content.displayname + " joined", - "icon": event.content.avatar_url ? event.content.avatar_url : undefined - }); - $timeout(function() { - notification.close(); - }, 5 * 1000); + " (" + (matrixService.getRoomIdToAliasMapping(event.room_id) || event.room_id) + ")", + event.content.displayname + " joined", + event.content.avatar_url ? event.content.avatar_url : undefined, + function() { + console.log("notification.onclick() room=" + event.room_id); + $rootScope.goToPage('room/' + event.room_id); + } + ); } } } From 71ef8f0636428ec08aa4ec5394cac2cbb77fd96b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 31 Oct 2014 11:56:36 +0000 Subject: [PATCH 050/265] SYWEB-102: Fix desktop notification msg when a user with no display name joins a room. --- webclient/room/room-controller.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 45638acaa..486ead0da 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -201,11 +201,14 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) // Notify when a user joins if ((document.hidden || matrixService.presence.unavailable === mPresence.getState()) && event.state_key !== $scope.state.user_id && "join" === event.membership) { - + var userName = event.content.displayname; + if (!userName) { + userName = event.state_key; + } notificationService.showNotification( - event.content.displayname + + userName + " (" + (matrixService.getRoomIdToAliasMapping(event.room_id) || event.room_id) + ")", - event.content.displayname + " joined", + userName + " joined", event.content.avatar_url ? event.content.avatar_url : undefined, function() { console.log("notification.onclick() room=" + event.room_id); From bfa36a72b9a852130cc42fb9322f6596e89725a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 14:00:14 +0000 Subject: [PATCH 051/265] Remove PDU tables. --- synapse/federation/persistence.py | 70 --- synapse/federation/replication.py | 2 +- synapse/storage/__init__.py | 60 +- synapse/storage/pdu.py | 949 ------------------------------ synapse/storage/schema/pdu.sql | 106 ---- synapse/storage/transactions.py | 45 -- 6 files changed, 2 insertions(+), 1230 deletions(-) delete mode 100644 synapse/storage/pdu.py delete mode 100644 synapse/storage/schema/pdu.sql diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 7043fcc50..a565375e6 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -32,76 +32,6 @@ import logging logger = logging.getLogger(__name__) -class PduActions(object): - """ Defines persistence actions that relate to handling PDUs. - """ - - def __init__(self, datastore): - self.store = datastore - - @log_function - def mark_as_processed(self, pdu): - """ Persist the fact that we have fully processed the given `Pdu` - - Returns: - Deferred - """ - return self.store.mark_pdu_as_processed(pdu.pdu_id, pdu.origin) - - @defer.inlineCallbacks - @log_function - def after_transaction(self, transaction_id, destination, origin): - """ Returns all `Pdu`s that we sent to the given remote home server - after a given transaction id. - - Returns: - Deferred: Results in a list of `Pdu`s - """ - results = yield self.store.get_pdus_after_transaction( - transaction_id, - destination - ) - - defer.returnValue([Pdu.from_pdu_tuple(p) for p in results]) - - @defer.inlineCallbacks - @log_function - def get_all_pdus_from_context(self, context): - results = yield self.store.get_all_pdus_from_context(context) - defer.returnValue([Pdu.from_pdu_tuple(p) for p in results]) - - @defer.inlineCallbacks - @log_function - def backfill(self, context, pdu_list, limit): - """ For a given list of PDU id and origins return the proceeding - `limit` `Pdu`s in the given `context`. - - Returns: - Deferred: Results in a list of `Pdu`s. - """ - results = yield self.store.get_backfill( - context, pdu_list, limit - ) - - defer.returnValue([Pdu.from_pdu_tuple(p) for p in results]) - - @log_function - def is_new(self, pdu): - """ When we receive a `Pdu` from a remote home server, we want to - figure out whether it is `new`, i.e. it is not some historic PDU that - we haven't seen simply because we haven't backfilled back that far. - - Returns: - Deferred: Results in a `bool` - """ - return self.store.is_pdu_new( - pdu_id=pdu.pdu_id, - origin=pdu.origin, - context=pdu.context, - depth=pdu.depth - ) - - class TransactionActions(object): """ Defines persistence actions that relate to handling Transactions. """ diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index a0bd2e057..159af4eed 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -21,7 +21,7 @@ from twisted.internet import defer from .units import Transaction, Pdu, Edu -from .persistence import PduActions, TransactionActions +from .persistence import TransactionActions from synapse.util.logutils import log_function diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d75c36683..3faa571dd 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -37,7 +37,6 @@ from .registration import RegistrationStore from .room import RoomStore from .roommember import RoomMemberStore from .stream import StreamStore -from .pdu import StatePduStore, PduStore, PdusTable from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore @@ -60,7 +59,6 @@ logger = logging.getLogger(__name__) SCHEMAS = [ "transactions", - "pdu", "users", "profiles", "presence", @@ -89,7 +87,7 @@ class _RollbackButIsFineException(Exception): class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, - PresenceStore, PduStore, StatePduStore, TransactionStore, + PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, EventFederationStore, ): @@ -150,68 +148,12 @@ class DataStore(RoomMemberStore, RoomStore, def _persist_pdu_event_txn(self, txn, pdu=None, event=None, backfilled=False, stream_ordering=None, is_new_state=True): - if pdu is not None: - self._persist_event_pdu_txn(txn, pdu) if event is not None: return self._persist_event_txn( txn, event, backfilled, stream_ordering, is_new_state=is_new_state, ) - def _persist_event_pdu_txn(self, txn, pdu): - cols = dict(pdu.__dict__) - unrec_keys = dict(pdu.unrecognized_keys) - del cols["hashes"] - del cols["signatures"] - del cols["content"] - del cols["prev_pdus"] - cols["content_json"] = json.dumps(pdu.content) - - unrec_keys.update({ - k: v for k, v in cols.items() - if k not in PdusTable.fields - }) - - cols["unrecognized_keys"] = json.dumps(unrec_keys) - - cols["ts"] = cols.pop("origin_server_ts") - - logger.debug("Persisting: %s", repr(cols)) - - for hash_alg, hash_base64 in pdu.hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_pdu_content_hash_txn( - txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, - ) - - signatures = pdu.signatures.get(pdu.origin, {}) - - for key_id, signature_base64 in signatures.items(): - signature_bytes = decode_base64(signature_base64) - self._store_pdu_origin_signature_txn( - txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, - ) - - for prev_pdu_id, prev_origin, prev_hashes in pdu.prev_pdus: - for alg, hash_base64 in prev_hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_prev_pdu_hash_txn( - txn, pdu.pdu_id, pdu.origin, prev_pdu_id, prev_origin, alg, - hash_bytes - ) - - (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) - self._store_pdu_reference_hash_txn( - txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes - ) - - if pdu.is_state: - self._persist_state_txn(txn, pdu.prev_pdus, cols) - else: - self._persist_pdu_txn(txn, pdu.prev_pdus, cols) - - self._update_min_depth_for_context_txn(txn, pdu.context, pdu.depth) - @log_function def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None, is_new_state=True): diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py deleted file mode 100644 index 4a4341907..000000000 --- a/synapse/storage/pdu.py +++ /dev/null @@ -1,949 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from ._base import SQLBaseStore, Table, JoinHelper - -from synapse.federation.units import Pdu -from synapse.util.logutils import log_function - -from syutil.base64util import encode_base64 - -from collections import namedtuple - -import logging - - -logger = logging.getLogger(__name__) - - -class PduStore(SQLBaseStore): - """A collection of queries for handling PDUs. - """ - - def get_pdu(self, pdu_id, origin): - """Given a pdu_id and origin, get a PDU. - - Args: - txn - pdu_id (str) - origin (str) - - Returns: - PduTuple: If the pdu does not exist in the database, returns None - """ - - return self.runInteraction( - "get_pdu", self._get_pdu_tuple, pdu_id, origin - ) - - def _get_pdu_tuple(self, txn, pdu_id, origin): - res = self._get_pdu_tuples(txn, [(pdu_id, origin)]) - return res[0] if res else None - - def _get_pdu_tuples(self, txn, pdu_id_tuples): - results = [] - for pdu_id, origin in pdu_id_tuples: - txn.execute( - PduEdgesTable.select_statement("pdu_id = ? AND origin = ?"), - (pdu_id, origin) - ) - - edges = [ - (r.prev_pdu_id, r.prev_origin) - for r in PduEdgesTable.decode_results(txn.fetchall()) - ] - - edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin) - - hashes = self._get_pdu_content_hashes_txn(txn, pdu_id, origin) - signatures = self._get_pdu_origin_signatures_txn( - txn, pdu_id, origin - ) - - query = ( - "SELECT %(fields)s FROM %(pdus)s as p " - "LEFT JOIN %(state)s as s " - "ON p.pdu_id = s.pdu_id AND p.origin = s.origin " - "WHERE p.pdu_id = ? AND p.origin = ? " - ) % { - "fields": _pdu_state_joiner.get_fields( - PdusTable="p", StatePdusTable="s"), - "pdus": PdusTable.table_name, - "state": StatePdusTable.table_name, - } - - txn.execute(query, (pdu_id, origin)) - - row = txn.fetchone() - if row: - results.append(PduTuple( - PduEntry(*row), edges, hashes, signatures, edge_hashes - )) - - return results - - def get_current_state_for_context(self, context): - """Get a list of PDUs that represent the current state for a given - context - - Args: - context (str) - - Returns: - list: A list of PduTuples - """ - - return self.runInteraction( - "get_current_state_for_context", - self._get_current_state_for_context, - context - ) - - def _get_current_state_for_context(self, txn, context): - query = ( - "SELECT pdu_id, origin FROM %s WHERE context = ?" - % CurrentStateTable.table_name - ) - - logger.debug("get_current_state %s, Args=%s", query, context) - txn.execute(query, (context,)) - - res = txn.fetchall() - - logger.debug("get_current_state %d results", len(res)) - - return self._get_pdu_tuples(txn, res) - - def _persist_pdu_txn(self, txn, prev_pdus, cols): - """Inserts a (non-state) PDU into the database. - - Args: - txn, - prev_pdus (list) - **cols: The columns to insert into the PdusTable. - """ - entry = PdusTable.EntryType( - **{k: cols.get(k, None) for k in PdusTable.fields} - ) - - txn.execute(PdusTable.insert_statement(), entry) - - self._handle_prev_pdus( - txn, entry.outlier, entry.pdu_id, entry.origin, - prev_pdus, entry.context - ) - - def mark_pdu_as_processed(self, pdu_id, pdu_origin): - """Mark a received PDU as processed. - - Args: - txn - pdu_id (str) - pdu_origin (str) - """ - - return self.runInteraction( - "mark_pdu_as_processed", - self._mark_as_processed, pdu_id, pdu_origin - ) - - def _mark_as_processed(self, txn, pdu_id, pdu_origin): - txn.execute("UPDATE %s SET have_processed = 1" % PdusTable.table_name) - - def get_all_pdus_from_context(self, context): - """Get a list of all PDUs for a given context.""" - return self.runInteraction( - "get_all_pdus_from_context", - self._get_all_pdus_from_context, context, - ) - - def _get_all_pdus_from_context(self, txn, context): - query = ( - "SELECT pdu_id, origin FROM %s " - "WHERE context = ?" - ) % PdusTable.table_name - - txn.execute(query, (context,)) - - return self._get_pdu_tuples(txn, txn.fetchall()) - - def get_backfill(self, context, pdu_list, limit): - """Get a list of Pdus for a given topic that occured before (and - including) the pdus in pdu_list. Return a list of max size `limit`. - - Args: - txn - context (str) - pdu_list (list) - limit (int) - - Return: - list: A list of PduTuples - """ - return self.runInteraction( - "get_backfill", - self._get_backfill, context, pdu_list, limit - ) - - def _get_backfill(self, txn, context, pdu_list, limit): - logger.debug( - "backfill: %s, %s, %s", - context, repr(pdu_list), limit - ) - - # We seed the pdu_results with the things from the pdu_list. - pdu_results = pdu_list - - front = pdu_list - - query = ( - "SELECT prev_pdu_id, prev_origin FROM %(edges_table)s " - "WHERE context = ? AND pdu_id = ? AND origin = ? " - "LIMIT ?" - ) % { - "edges_table": PduEdgesTable.table_name, - } - - # We iterate through all pdu_ids in `front` to select their previous - # pdus. These are dumped in `new_front`. We continue until we reach the - # limit *or* new_front is empty (i.e., we've run out of things to - # select - while front and len(pdu_results) < limit: - - new_front = [] - for pdu_id, origin in front: - logger.debug( - "_backfill_interaction: i=%s, o=%s", - pdu_id, origin - ) - - txn.execute( - query, - (context, pdu_id, origin, limit - len(pdu_results)) - ) - - for row in txn.fetchall(): - logger.debug( - "_backfill_interaction: got i=%s, o=%s", - *row - ) - new_front.append(row) - - front = new_front - pdu_results += new_front - - # We also want to update the `prev_pdus` attributes before returning. - return self._get_pdu_tuples(txn, pdu_results) - - def get_min_depth_for_context(self, context): - """Get the current minimum depth for a context - - Args: - txn - context (str) - """ - return self.runInteraction( - "get_min_depth_for_context", - self._get_min_depth_for_context, context - ) - - def _get_min_depth_for_context(self, txn, context): - return self._get_min_depth_interaction(txn, context) - - def _get_min_depth_interaction(self, txn, context): - txn.execute( - "SELECT min_depth FROM %s WHERE context = ?" - % ContextDepthTable.table_name, - (context,) - ) - - row = txn.fetchone() - - return row[0] if row else None - - def _update_min_depth_for_context_txn(self, txn, context, depth): - """Update the minimum `depth` of the given context, which is the line - on which we stop backfilling backwards. - - Args: - context (str) - depth (int) - """ - min_depth = self._get_min_depth_interaction(txn, context) - - do_insert = depth < min_depth if min_depth else True - - if do_insert: - txn.execute( - "INSERT OR REPLACE INTO %s (context, min_depth) " - "VALUES (?,?)" % ContextDepthTable.table_name, - (context, depth) - ) - - def get_latest_pdus_in_context(self, context): - return self.runInteraction( - "get_latest_pdus_in_context", - self._get_latest_pdus_in_context, - context - ) - - def _get_latest_pdus_in_context(self, txn, context): - """Get's a list of the most current pdus for a given context. This is - used when we are sending a Pdu and need to fill out the `prev_pdus` - key - - Args: - txn - context - """ - query = ( - "SELECT p.pdu_id, p.origin, p.depth FROM %(pdus)s as p " - "INNER JOIN %(forward)s as f ON p.pdu_id = f.pdu_id " - "AND f.origin = p.origin " - "WHERE f.context = ?" - ) % { - "pdus": PdusTable.table_name, - "forward": PduForwardExtremitiesTable.table_name, - } - - logger.debug("get_prev query: %s", query) - - txn.execute( - query, - (context, ) - ) - - results = [] - for pdu_id, origin, depth in txn.fetchall(): - hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin) - sha256_bytes = hashes["sha256"] - prev_hashes = {"sha256": encode_base64(sha256_bytes)} - results.append((pdu_id, origin, prev_hashes, depth)) - - return results - - @defer.inlineCallbacks - def get_oldest_pdus_in_context(self, context): - """Get a list of Pdus that we haven't backfilled beyond yet (and havent - seen). This list is used when we want to backfill backwards and is the - list we send to the remote server. - - Args: - txn - context (str) - - Returns: - list: A list of PduIdTuple. - """ - results = yield self._execute( - None, - "SELECT pdu_id, origin FROM %(back)s WHERE context = ?" - % {"back": PduBackwardExtremitiesTable.table_name, }, - context - ) - - defer.returnValue([PduIdTuple(i, o) for i, o in results]) - - def is_pdu_new(self, pdu_id, origin, context, depth): - """For a given Pdu, try and figure out if it's 'new', i.e., if it's - not something we got randomly from the past, for example when we - request the current state of the room that will probably return a bunch - of pdus from before we joined. - - Args: - txn - pdu_id (str) - origin (str) - context (str) - depth (int) - - Returns: - bool - """ - - return self.runInteraction( - "is_pdu_new", - self._is_pdu_new, - pdu_id=pdu_id, - origin=origin, - context=context, - depth=depth - ) - - def _is_pdu_new(self, txn, pdu_id, origin, context, depth): - # If depth > min depth in back table, then we classify it as new. - # OR if there is nothing in the back table, then it kinda needs to - # be a new thing. - query = ( - "SELECT min(p.depth) FROM %(edges)s as e " - "INNER JOIN %(back)s as b " - "ON e.prev_pdu_id = b.pdu_id AND e.prev_origin = b.origin " - "INNER JOIN %(pdus)s as p " - "ON e.pdu_id = p.pdu_id AND p.origin = e.origin " - "WHERE p.context = ?" - ) % { - "pdus": PdusTable.table_name, - "edges": PduEdgesTable.table_name, - "back": PduBackwardExtremitiesTable.table_name, - } - - txn.execute(query, (context,)) - - min_depth, = txn.fetchone() - - if not min_depth or depth > int(min_depth): - logger.debug( - "is_new true: id=%s, o=%s, d=%s min_depth=%s", - pdu_id, origin, depth, min_depth - ) - return True - - # If this pdu is in the forwards table, then it also is a new one - query = ( - "SELECT * FROM %(forward)s WHERE pdu_id = ? AND origin = ?" - ) % { - "forward": PduForwardExtremitiesTable.table_name, - } - - txn.execute(query, (pdu_id, origin)) - - # Did we get anything? - if txn.fetchall(): - logger.debug( - "is_new true: id=%s, o=%s, d=%s was forward", - pdu_id, origin, depth - ) - return True - - logger.debug( - "is_new false: id=%s, o=%s, d=%s", - pdu_id, origin, depth - ) - - # FINE THEN. It's probably old. - return False - - @staticmethod - @log_function - def _handle_prev_pdus(txn, outlier, pdu_id, origin, prev_pdus, - context): - txn.executemany( - PduEdgesTable.insert_statement(), - [(pdu_id, origin, p[0], p[1], context) for p in prev_pdus] - ) - - # Update the extremities table if this is not an outlier. - if not outlier: - - # First, we delete the new one from the forwards extremities table. - query = ( - "DELETE FROM %s WHERE pdu_id = ? AND origin = ?" - % PduForwardExtremitiesTable.table_name - ) - txn.executemany(query, list(p[:2] for p in prev_pdus)) - - # We only insert as a forward extremety the new pdu if there are no - # other pdus that reference it as a prev pdu - query = ( - "INSERT INTO %(table)s (pdu_id, origin, context) " - "SELECT ?, ?, ? WHERE NOT EXISTS (" - "SELECT 1 FROM %(pdu_edges)s WHERE " - "prev_pdu_id = ? AND prev_origin = ?" - ")" - ) % { - "table": PduForwardExtremitiesTable.table_name, - "pdu_edges": PduEdgesTable.table_name - } - - logger.debug("query: %s", query) - - txn.execute(query, (pdu_id, origin, context, pdu_id, origin)) - - # Insert all the prev_pdus as a backwards thing, they'll get - # deleted in a second if they're incorrect anyway. - txn.executemany( - PduBackwardExtremitiesTable.insert_statement(), - [(i, o, context) for i, o, _ in prev_pdus] - ) - - # Also delete from the backwards extremities table all ones that - # reference pdus that we have already seen - query = ( - "DELETE FROM %(pdu_back)s WHERE EXISTS (" - "SELECT 1 FROM %(pdus)s AS pdus " - "WHERE " - "%(pdu_back)s.pdu_id = pdus.pdu_id " - "AND %(pdu_back)s.origin = pdus.origin " - "AND not pdus.outlier " - ")" - ) % { - "pdu_back": PduBackwardExtremitiesTable.table_name, - "pdus": PdusTable.table_name, - } - txn.execute(query) - - -class StatePduStore(SQLBaseStore): - """A collection of queries for handling state PDUs. - """ - - def _persist_state_txn(self, txn, prev_pdus, cols): - """Inserts a state PDU into the database - - Args: - txn, - prev_pdus (list) - **cols: The columns to insert into the PdusTable and StatePdusTable - """ - pdu_entry = PdusTable.EntryType( - **{k: cols.get(k, None) for k in PdusTable.fields} - ) - state_entry = StatePdusTable.EntryType( - **{k: cols.get(k, None) for k in StatePdusTable.fields} - ) - - logger.debug("Inserting pdu: %s", repr(pdu_entry)) - logger.debug("Inserting state: %s", repr(state_entry)) - - txn.execute(PdusTable.insert_statement(), pdu_entry) - txn.execute(StatePdusTable.insert_statement(), state_entry) - - self._handle_prev_pdus( - txn, - pdu_entry.outlier, pdu_entry.pdu_id, pdu_entry.origin, prev_pdus, - pdu_entry.context - ) - - def get_unresolved_state_tree(self, new_state_pdu): - return self.runInteraction( - "get_unresolved_state_tree", - self._get_unresolved_state_tree, new_state_pdu - ) - - @log_function - def _get_unresolved_state_tree(self, txn, new_pdu): - current = self._get_current_interaction( - txn, - new_pdu.context, new_pdu.pdu_type, new_pdu.state_key - ) - - ReturnType = namedtuple( - "StateReturnType", ["new_branch", "current_branch"] - ) - return_value = ReturnType([new_pdu], []) - - if not current: - logger.debug("get_unresolved_state_tree No current state.") - return (return_value, None) - - return_value.current_branch.append(current) - - enum_branches = self._enumerate_state_branches( - txn, new_pdu, current - ) - - missing_branch = None - for branch, prev_state, state in enum_branches: - if state: - return_value[branch].append(state) - else: - # We don't have prev_state :( - missing_branch = branch - break - - return (return_value, missing_branch) - - def update_current_state(self, pdu_id, origin, context, pdu_type, - state_key): - return self.runInteraction( - "update_current_state", - self._update_current_state, - pdu_id, origin, context, pdu_type, state_key - ) - - def _update_current_state(self, txn, pdu_id, origin, context, pdu_type, - state_key): - query = ( - "INSERT OR REPLACE INTO %(curr)s (%(fields)s) VALUES (%(qs)s)" - ) % { - "curr": CurrentStateTable.table_name, - "fields": CurrentStateTable.get_fields_string(), - "qs": ", ".join(["?"] * len(CurrentStateTable.fields)) - } - - query_args = CurrentStateTable.EntryType( - pdu_id=pdu_id, - origin=origin, - context=context, - pdu_type=pdu_type, - state_key=state_key - ) - - txn.execute(query, query_args) - - def get_current_state_pdu(self, context, pdu_type, state_key): - """For a given context, pdu_type, state_key 3-tuple, return what is - currently considered the current state. - - Args: - txn - context (str) - pdu_type (str) - state_key (str) - - Returns: - PduEntry - """ - - return self.runInteraction( - "get_current_state_pdu", - self._get_current_state_pdu, context, pdu_type, state_key - ) - - def _get_current_state_pdu(self, txn, context, pdu_type, state_key): - return self._get_current_interaction(txn, context, pdu_type, state_key) - - def _get_current_interaction(self, txn, context, pdu_type, state_key): - logger.debug( - "_get_current_interaction %s %s %s", - context, pdu_type, state_key - ) - - fields = _pdu_state_joiner.get_fields( - PdusTable="p", StatePdusTable="s") - - current_query = ( - "SELECT %(fields)s FROM %(state)s as s " - "INNER JOIN %(pdus)s as p " - "ON s.pdu_id = p.pdu_id AND s.origin = p.origin " - "INNER JOIN %(curr)s as c " - "ON s.pdu_id = c.pdu_id AND s.origin = c.origin " - "WHERE s.context = ? AND s.pdu_type = ? AND s.state_key = ? " - ) % { - "fields": fields, - "curr": CurrentStateTable.table_name, - "state": StatePdusTable.table_name, - "pdus": PdusTable.table_name, - } - - txn.execute( - current_query, - (context, pdu_type, state_key) - ) - - row = txn.fetchone() - - result = PduEntry(*row) if row else None - - if not result: - logger.debug("_get_current_interaction not found") - else: - logger.debug( - "_get_current_interaction found %s %s", - result.pdu_id, result.origin - ) - - return result - - def handle_new_state(self, new_pdu): - """Actually perform conflict resolution on the new_pdu on the - assumption we have all the pdus required to perform it. - - Args: - new_pdu - - Returns: - bool: True if the new_pdu clobbered the current state, False if not - """ - return self.runInteraction( - "handle_new_state", - self._handle_new_state, new_pdu - ) - - def _handle_new_state(self, txn, new_pdu): - logger.debug( - "handle_new_state %s %s", - new_pdu.pdu_id, new_pdu.origin - ) - - current = self._get_current_interaction( - txn, - new_pdu.context, new_pdu.pdu_type, new_pdu.state_key - ) - - is_current = False - - if (not current or not current.prev_state_id - or not current.prev_state_origin): - # Oh, we don't have any state for this yet. - is_current = True - elif (current.pdu_id == new_pdu.prev_state_id - and current.origin == new_pdu.prev_state_origin): - # Oh! A direct clobber. Just do it. - is_current = True - else: - ## - # Ok, now loop through until we get to a common ancestor. - max_new = int(new_pdu.power_level) - max_current = int(current.power_level) - - enum_branches = self._enumerate_state_branches( - txn, new_pdu, current - ) - for branch, prev_state, state in enum_branches: - if not state: - raise RuntimeError( - "Could not find state_pdu %s %s" % - ( - prev_state.prev_state_id, - prev_state.prev_state_origin - ) - ) - - if branch == 0: - max_new = max(int(state.depth), max_new) - else: - max_current = max(int(state.depth), max_current) - - is_current = max_new > max_current - - if is_current: - logger.debug("handle_new_state make current") - - # Right, this is a new thing, so woo, just insert it. - txn.execute( - "INSERT OR REPLACE INTO %(curr)s (%(fields)s) VALUES (%(qs)s)" - % { - "curr": CurrentStateTable.table_name, - "fields": CurrentStateTable.get_fields_string(), - "qs": ", ".join(["?"] * len(CurrentStateTable.fields)) - }, - CurrentStateTable.EntryType( - *(new_pdu.__dict__[k] for k in CurrentStateTable.fields) - ) - ) - else: - logger.debug("handle_new_state not current") - - logger.debug("handle_new_state done") - - return is_current - - @log_function - def _enumerate_state_branches(self, txn, pdu_a, pdu_b): - branch_a = pdu_a - branch_b = pdu_b - - while True: - if (branch_a.pdu_id == branch_b.pdu_id - and branch_a.origin == branch_b.origin): - # Woo! We found a common ancestor - logger.debug("_enumerate_state_branches Found common ancestor") - break - - do_branch_a = ( - hasattr(branch_a, "prev_state_id") and - branch_a.prev_state_id - ) - - do_branch_b = ( - hasattr(branch_b, "prev_state_id") and - branch_b.prev_state_id - ) - - logger.debug( - "do_branch_a=%s, do_branch_b=%s", - do_branch_a, do_branch_b - ) - - if do_branch_a and do_branch_b: - do_branch_a = int(branch_a.depth) > int(branch_b.depth) - - if do_branch_a: - pdu_tuple = PduIdTuple( - branch_a.prev_state_id, - branch_a.prev_state_origin - ) - - prev_branch = branch_a - - logger.debug("getting branch_a prev %s", pdu_tuple) - branch_a = self._get_pdu_tuple(txn, *pdu_tuple) - if branch_a: - branch_a = Pdu.from_pdu_tuple(branch_a) - - logger.debug("branch_a=%s", branch_a) - - yield (0, prev_branch, branch_a) - - if not branch_a: - break - elif do_branch_b: - pdu_tuple = PduIdTuple( - branch_b.prev_state_id, - branch_b.prev_state_origin - ) - - prev_branch = branch_b - - logger.debug("getting branch_b prev %s", pdu_tuple) - branch_b = self._get_pdu_tuple(txn, *pdu_tuple) - if branch_b: - branch_b = Pdu.from_pdu_tuple(branch_b) - - logger.debug("branch_b=%s", branch_b) - - yield (1, prev_branch, branch_b) - - if not branch_b: - break - else: - break - - -class PdusTable(Table): - table_name = "pdus" - - fields = [ - "pdu_id", - "origin", - "context", - "pdu_type", - "ts", - "depth", - "is_state", - "content_json", - "unrecognized_keys", - "outlier", - "have_processed", - ] - - EntryType = namedtuple("PdusEntry", fields) - - -class PduDestinationsTable(Table): - table_name = "pdu_destinations" - - fields = [ - "pdu_id", - "origin", - "destination", - "delivered_ts", - ] - - EntryType = namedtuple("PduDestinationsEntry", fields) - - -class PduEdgesTable(Table): - table_name = "pdu_edges" - - fields = [ - "pdu_id", - "origin", - "prev_pdu_id", - "prev_origin", - "context" - ] - - EntryType = namedtuple("PduEdgesEntry", fields) - - -class PduForwardExtremitiesTable(Table): - table_name = "pdu_forward_extremities" - - fields = [ - "pdu_id", - "origin", - "context", - ] - - EntryType = namedtuple("PduForwardExtremitiesEntry", fields) - - -class PduBackwardExtremitiesTable(Table): - table_name = "pdu_backward_extremities" - - fields = [ - "pdu_id", - "origin", - "context", - ] - - EntryType = namedtuple("PduBackwardExtremitiesEntry", fields) - - -class ContextDepthTable(Table): - table_name = "context_depth" - - fields = [ - "context", - "min_depth", - ] - - EntryType = namedtuple("ContextDepthEntry", fields) - - -class StatePdusTable(Table): - table_name = "state_pdus" - - fields = [ - "pdu_id", - "origin", - "context", - "pdu_type", - "state_key", - "power_level", - "prev_state_id", - "prev_state_origin", - ] - - EntryType = namedtuple("StatePdusEntry", fields) - - -class CurrentStateTable(Table): - table_name = "current_state" - - fields = [ - "pdu_id", - "origin", - "context", - "pdu_type", - "state_key", - ] - - EntryType = namedtuple("CurrentStateEntry", fields) - -_pdu_state_joiner = JoinHelper(PdusTable, StatePdusTable) - - -# TODO: These should probably be put somewhere more sensible -PduIdTuple = namedtuple("PduIdTuple", ("pdu_id", "origin")) - -PduEntry = _pdu_state_joiner.EntryType -""" We are always interested in the join of the PdusTable and StatePdusTable, -rather than just the PdusTable. - -This does not include a prev_pdus key. -""" - -PduTuple = namedtuple( - "PduTuple", - ("pdu_entry", "prev_pdu_list", "hashes", "signatures", "edge_hashes") -) -""" This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent -the `prev_pdus` key of a PDU. -""" diff --git a/synapse/storage/schema/pdu.sql b/synapse/storage/schema/pdu.sql deleted file mode 100644 index 16e111a56..000000000 --- a/synapse/storage/schema/pdu.sql +++ /dev/null @@ -1,106 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Stores pdus and their content -CREATE TABLE IF NOT EXISTS pdus( - pdu_id TEXT, - origin TEXT, - context TEXT, - pdu_type TEXT, - ts INTEGER, - depth INTEGER DEFAULT 0 NOT NULL, - is_state BOOL, - content_json TEXT, - unrecognized_keys TEXT, - outlier BOOL NOT NULL, - have_processed BOOL, - CONSTRAINT pdu_id_origin UNIQUE (pdu_id, origin) -); - --- Stores what the current state pdu is for a given (context, pdu_type, key) tuple -CREATE TABLE IF NOT EXISTS state_pdus( - pdu_id TEXT, - origin TEXT, - context TEXT, - pdu_type TEXT, - state_key TEXT, - power_level TEXT, - prev_state_id TEXT, - prev_state_origin TEXT, - CONSTRAINT pdu_id_origin UNIQUE (pdu_id, origin) - CONSTRAINT prev_pdu_id_origin UNIQUE (prev_state_id, prev_state_origin) -); - -CREATE TABLE IF NOT EXISTS current_state( - pdu_id TEXT, - origin TEXT, - context TEXT, - pdu_type TEXT, - state_key TEXT, - CONSTRAINT pdu_id_origin UNIQUE (pdu_id, origin) - CONSTRAINT uniqueness UNIQUE (context, pdu_type, state_key) ON CONFLICT REPLACE -); - --- Stores where each pdu we want to send should be sent and the delivery status. -create TABLE IF NOT EXISTS pdu_destinations( - pdu_id TEXT, - origin TEXT, - destination TEXT, - delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, destination) ON CONFLICT REPLACE -); - -CREATE TABLE IF NOT EXISTS pdu_forward_extremities( - pdu_id TEXT, - origin TEXT, - context TEXT, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, context) ON CONFLICT REPLACE -); - -CREATE TABLE IF NOT EXISTS pdu_backward_extremities( - pdu_id TEXT, - origin TEXT, - context TEXT, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, context) ON CONFLICT REPLACE -); - -CREATE TABLE IF NOT EXISTS pdu_edges( - pdu_id TEXT, - origin TEXT, - prev_pdu_id TEXT, - prev_origin TEXT, - context TEXT, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, prev_pdu_id, prev_origin, context) -); - -CREATE TABLE IF NOT EXISTS context_depth( - context TEXT, - min_depth INTEGER, - CONSTRAINT uniqueness UNIQUE (context) -); - -CREATE INDEX IF NOT EXISTS context_depth_context ON context_depth(context); - - -CREATE INDEX IF NOT EXISTS pdu_id ON pdus(pdu_id, origin); - -CREATE INDEX IF NOT EXISTS dests_id ON pdu_destinations (pdu_id, origin); --- CREATE INDEX IF NOT EXISTS dests ON pdu_destinations (destination); - -CREATE INDEX IF NOT EXISTS pdu_extrem_context ON pdu_forward_extremities(context); -CREATE INDEX IF NOT EXISTS pdu_extrem_id ON pdu_forward_extremities(pdu_id, origin); - -CREATE INDEX IF NOT EXISTS pdu_edges_id ON pdu_edges(pdu_id, origin); - -CREATE INDEX IF NOT EXISTS pdu_b_extrem_context ON pdu_backward_extremities(context); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 6624348fd..ea6790078 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -14,7 +14,6 @@ # limitations under the License. from ._base import SQLBaseStore, Table -from .pdu import PdusTable from collections import namedtuple @@ -207,50 +206,6 @@ class TransactionStore(SQLBaseStore): return ReceivedTransactionsTable.decode_results(txn.fetchall()) - def get_pdus_after_transaction(self, transaction_id, destination): - """For a given local transaction_id that we sent to a given destination - home server, return a list of PDUs that were sent to that destination - after it. - - Args: - txn - transaction_id (str) - destination (str) - - Returns - list: A list of PduTuple - """ - return self.runInteraction( - "get_pdus_after_transaction", - self._get_pdus_after_transaction, - transaction_id, destination - ) - - def _get_pdus_after_transaction(self, txn, transaction_id, destination): - - # Query that first get's all transaction_ids with an id greater than - # the one given from the `sent_transactions` table. Then JOIN on this - # from the `tx->pdu` table to get a list of (pdu_id, origin) that - # specify the pdus that were sent in those transactions. - query = ( - "SELECT pdu_id, pdu_origin FROM %(tx_pdu)s as tp " - "INNER JOIN %(sent_tx)s as st " - "ON tp.transaction_id = st.transaction_id " - "AND tp.destination = st.destination " - "WHERE st.id > (" - "SELECT id FROM %(sent_tx)s " - "WHERE transaction_id = ? AND destination = ?" - ) % { - "tx_pdu": TransactionsToPduTable.table_name, - "sent_tx": SentTransactions.table_name, - } - - txn.execute(query, (transaction_id, destination)) - - pdus = PdusTable.decode_results(txn.fetchall()) - - return self._get_pdu_tuples(txn, pdus) - class ReceivedTransactionsTable(Table): table_name = "received_transactions" From 2aa79f4fbe64b8fe88dae5414cc7f81a851fbb23 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 31 Oct 2014 14:26:51 +0000 Subject: [PATCH 052/265] Added model-service.js to store model data. --- webclient/app.js | 1 + .../matrix/event-handler-service.js | 4 +- webclient/components/matrix/model-service.js | 98 +++++++++++++++++++ webclient/index.html | 1 + 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 webclient/components/matrix/model-service.js diff --git a/webclient/app.js b/webclient/app.js index c091f8c6c..17b2bb6e8 100644 --- a/webclient/app.js +++ b/webclient/app.js @@ -31,6 +31,7 @@ var matrixWebClient = angular.module('matrixWebClient', [ 'eventStreamService', 'eventHandlerService', 'notificationService', + 'modelService', 'infinite-scroll', 'ui.bootstrap', 'monospaced.elastic' diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index e63584510..84b2a220e 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -27,8 +27,8 @@ Typically, this service will store events or broadcast them to any listeners if typically all the $on method would do is update its own $scope. */ angular.module('eventHandlerService', []) -.factory('eventHandlerService', ['matrixService', '$rootScope', '$q', '$timeout', 'mPresence', 'notificationService', -function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService) { +.factory('eventHandlerService', ['matrixService', '$rootScope', '$q', '$timeout', 'mPresence', 'notificationService', 'modelService', +function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService, modelService) { var ROOM_CREATE_EVENT = "ROOM_CREATE_EVENT"; var MSG_EVENT = "MSG_EVENT"; var MEMBER_EVENT = "MEMBER_EVENT"; diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js new file mode 100644 index 000000000..c10e5185a --- /dev/null +++ b/webclient/components/matrix/model-service.js @@ -0,0 +1,98 @@ +/* +Copyright 2014 OpenMarket Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +'use strict'; + +/* +This service serves as the entry point for all models in the app. If access to +underlying data in a room is required, then this service should be used as the +dependency. +*/ +// NB: This is more explicit than linking top-level models to $rootScope +// in that by adding this service as a dep you are clearly saying "this X +// needs access to the underlying data store", rather than polluting the +// $rootScope. +angular.module('modelService', []) +.factory('modelService', ['matrixService', function(matrixService) { + + /***** Room Object *****/ + var Room = function Room(room_id) { + this.room_id = room_id; + this.old_room_state = RoomState(); + this.current_room_state = RoomState(); + }; + Room.prototype = { + leave: function leave() { + return matrixService.leave(this.room_id); + } + }; + + /***** Room State Object *****/ + var RoomState = function RoomState() { + // list of RoomMember + this.members = []; + // state events, the key is a compound of event type + state_key + this.state_events = {}; + // earliest token + this.pagination_token = ""; + }; + RoomState.prototype = { + // get a state event for this room from this.state_events. State events + // are unique per type+state_key tuple, with a lot of events using 0-len + // state keys. To make it not Really Annoying to access, this method is + // provided which can just be given the type and it will return the + // 0-len event by default. + state: function state(type, state_key) { + if (!state_key) { + return this.state_events[type]; + } + return this.state_events[type + state_key]; + }, + + storeState: function storeState(event) { + this.state_events[event.type + event.state_key] = event; + } + }; + + /***** Room Member Object *****/ + var RoomMember = function RoomMember() { + this.event = {}; // the m.room.member event representing the RoomMember. + this.user = undefined; // the User + }; + + /***** User Object *****/ + var User = function User() { + this.event = {}; // the m.presence event representing the User. + }; + + // rooms are stored here when they come in. + var rooms = { + // roomid: + }; + + console.log("Models inited."); + + return { + + getRoom: function(roomId) { + if(!rooms[roomId]) { + rooms[roomId] = new Room(roomId); + } + return rooms[roomId]; + } + + }; +}]); diff --git a/webclient/index.html b/webclient/index.html index bc011a6c7..965981f7d 100644 --- a/webclient/index.html +++ b/webclient/index.html @@ -42,6 +42,7 @@ + From 2f39dc19a26cca25305d10654916d7413a56a23a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 14:27:14 +0000 Subject: [PATCH 053/265] Remove more references to dead PDU tables --- synapse/federation/pdu_codec.py | 4 +- synapse/state.py | 23 +++------- synapse/storage/__init__.py | 9 ---- synapse/storage/schema/signatures.sql | 66 --------------------------- 4 files changed, 8 insertions(+), 94 deletions(-) delete mode 100644 synapse/storage/schema/signatures.sql diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 6d3128629..d4c896e16 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -32,11 +32,11 @@ class PduCodec(object): self.hs = hs def encode_event_id(self, local, domain): - return EventID.create(local, domain, self.hs).to_string() + return local def decode_event_id(self, event_id): e_id = self.hs.parse_eventid(event_id) - return e_id.localpart, e_id.domain + return event_id, e_id.domain def event_from_pdu(self, pdu): kwargs = {} diff --git a/synapse/state.py b/synapse/state.py index f7249705c..2548deed2 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -77,29 +77,18 @@ class StateHandler(object): snapshot.fill_out_prev_events(event) yield self.annotate_state_groups(event) - current_state = snapshot.prev_state_pdu + if event.old_state_events: + current_state = event.old_state_events.get( + (event.type, event.state_key) + ) - if current_state: - event.prev_state = EventID.create( - current_state.pdu_id, current_state.origin, self.hs - ).to_string() + if current_state: + event.prev_state = current_state.event_id # TODO check current_state to see if the min power level is less # than the power level of the user # power_level = self._get_power_level_for_event(event) - e_id = self.hs.parse_eventid(event.event_id) - pdu_id = e_id.localpart - origin = e_id.domain - - yield self.store.update_current_state( - pdu_id=pdu_id, - origin=origin, - context=key.context, - pdu_type=key.type, - state_key=key.state_key - ) - defer.returnValue(True) @defer.inlineCallbacks diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 3faa571dd..c2560f604 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,6 @@ SCHEMAS = [ "keys", "redactions", "state", - "signatures", "event_edges", "event_signatures", ] @@ -364,13 +363,6 @@ class DataStore(RoomMemberStore, RoomStore, membership_state = self._get_room_member(txn, user_id, room_id) prev_events = self._get_latest_events_in_room(txn, room_id) - if state_type is not None and state_key is not None: - prev_state_pdu = self._get_current_state_pdu( - txn, room_id, state_type, state_key - ) - else: - prev_state_pdu = None - return Snapshot( store=self, room_id=room_id, @@ -379,7 +371,6 @@ class DataStore(RoomMemberStore, RoomStore, membership_state=membership_state, state_type=state_type, state_key=state_key, - prev_state_pdu=prev_state_pdu, ) return self.runInteraction("snapshot_room", _snapshot) diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql deleted file mode 100644 index 1c45a51be..000000000 --- a/synapse/storage/schema/signatures.sql +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE TABLE IF NOT EXISTS pdu_content_hashes ( - pdu_id TEXT, - origin TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) -); - -CREATE INDEX IF NOT EXISTS pdu_content_hashes_id ON pdu_content_hashes ( - pdu_id, origin -); - -CREATE TABLE IF NOT EXISTS pdu_reference_hashes ( - pdu_id TEXT, - origin TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm) -); - -CREATE INDEX IF NOT EXISTS pdu_reference_hashes_id ON pdu_reference_hashes ( - pdu_id, origin -); - -CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( - pdu_id TEXT, - origin TEXT, - key_id TEXT, - signature BLOB, - CONSTRAINT uniqueness UNIQUE (pdu_id, origin, key_id) -); - -CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( - pdu_id, origin -); - -CREATE TABLE IF NOT EXISTS pdu_edge_hashes( - pdu_id TEXT, - origin TEXT, - prev_pdu_id TEXT, - prev_origin TEXT, - algorithm TEXT, - hash BLOB, - CONSTRAINT uniqueness UNIQUE ( - pdu_id, origin, prev_pdu_id, prev_origin, algorithm - ) -); - -CREATE INDEX IF NOT EXISTS pdu_edge_hashes_id ON pdu_edge_hashes( - pdu_id, origin -); From 394f77c3ff2d849b4ec69595f40ebfcc84ed1d31 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 31 Oct 2014 14:50:31 +0000 Subject: [PATCH 054/265] Parse /initialSync data and populate the new data structures. --- .../matrix/event-handler-service.js | 12 +++++++++ webclient/components/matrix/model-service.js | 25 ++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 84b2a220e..a8d89834a 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -86,6 +86,18 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } $rootScope.events.rooms[room_id].membership = room.membership; } + + // ========================================= + var __room = modelService.getRoom(room_id); + if (room) { // /initialSync data + __room.current_room_state.storeStateEvents(room.state); + __room.current_room_state.pagination_token = room.messages.end; + + __room.old_room_state.storeStateEvents(room.state); + __room.old_room_state.pagination_token = room.messages.start; + + __room.addMessages(room.messages.chunk); + } }; var resetRoomMessages = function(room_id) { diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index c10e5185a..0ba23cb76 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -31,10 +31,22 @@ angular.module('modelService', []) /***** Room Object *****/ var Room = function Room(room_id) { this.room_id = room_id; - this.old_room_state = RoomState(); - this.current_room_state = RoomState(); + this.old_room_state = new RoomState(); + this.current_room_state = new RoomState(); + this.messages = []; // events which can be displayed on the UI. TODO move? }; Room.prototype = { + addMessages: function addMessages(events, toFront) { + for (var i=0; i Date: Fri, 31 Oct 2014 15:16:43 +0000 Subject: [PATCH 055/265] Hook into more of event-handler-service and mimic its functions for now. --- .../matrix/event-handler-service.js | 164 +++++++++++------- webclient/components/matrix/model-service.js | 34 +++- 2 files changed, 130 insertions(+), 68 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index a8d89834a..589554ec0 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -96,7 +96,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService __room.old_room_state.storeStateEvents(room.state); __room.old_room_state.pagination_token = room.messages.start; - __room.addMessages(room.messages.chunk); + __room.addMessageEvents(room.messages.chunk); } }; @@ -108,6 +108,26 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // Generic method to handle events data var handleRoomDateEvent = function(event, isLiveEvent, addToRoomMessages) { + var __room = modelService.getRoom(event.room_id); + if (addToRoomMessages) { + __room.addMessageEvent(event, !isLiveEvent); + } + if (isLiveEvent) { + __room.current_room_state.storeStateEvent(event); + } + else { + var eventTs = event.origin_server_ts; + var storedEvent = __room.current_room_state.getStateEvent(event.type, event.state_key); + if (storedEvent) { + if (storedEvent.origin_server_ts < eventTs) { + // the incoming event is newer, use it. + __room.current_room_state.storeStateEvent(event); + } + } + } + + // ===================================== + // Add topic changes as if they were a room message if (addToRoomMessages) { if (isLiveEvent) { @@ -144,6 +164,70 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var handleRoomAliases = function(event, isLiveEvent) { matrixService.createRoomIdToAliasMapping(event.room_id, event.content.aliases[0]); }; + + var displayNotification = function(event) { + if (window.Notification && event.user_id != matrixService.config().user_id) { + var shouldBing = notificationService.containsBingWord( + matrixService.config().user_id, + matrixService.config().display_name, + matrixService.config().bingWords, + event.content.body + ); + + // Ideally we would notify only when the window is hidden (i.e. document.hidden = true). + // + // However, Chrome on Linux and OSX currently returns document.hidden = false unless the window is + // explicitly showing a different tab. So we need another metric to determine hiddenness - we + // simply use idle time. If the user has been idle enough that their presence goes to idle, then + // we also display notifs when things happen. + // + // This is far far better than notifying whenever anything happens anyway, otherwise you get spammed + // to death with notifications when the window is in the foreground, which is horrible UX (especially + // if you have not defined any bingers and so get notified for everything). + var isIdle = (document.hidden || matrixService.presence.unavailable === mPresence.getState()); + + // We need a way to let people get notifications for everything, if they so desire. The way to do this + // is to specify zero bingwords. + var bingWords = matrixService.config().bingWords; + if (bingWords === undefined || bingWords.length === 0) { + shouldBing = true; + } + + if (shouldBing && isIdle) { + console.log("Displaying notification for "+JSON.stringify(event)); + var member = getMember(event.room_id, event.user_id); + var displayname = getUserDisplayName(event.room_id, event.user_id); + + var message = event.content.body; + if (event.content.msgtype === "m.emote") { + message = "* " + displayname + " " + message; + } + else if (event.content.msgtype === "m.image") { + message = displayname + " sent an image."; + } + + var roomTitle = matrixService.getRoomIdToAliasMapping(event.room_id); + var theRoom = $rootScope.events.rooms[event.room_id]; + if (!roomTitle && theRoom && theRoom["m.room.name"] && theRoom["m.room.name"].content) { + roomTitle = theRoom["m.room.name"].content.name; + } + + if (!roomTitle) { + roomTitle = event.room_id; + } + + notificationService.showNotification( + displayname + " (" + roomTitle + ")", + message, + member ? member.avatar_url : undefined, + function() { + console.log("notification.onclick() room=" + event.room_id); + $rootScope.goToPage('room/' + event.room_id); + } + ); + } + } + }; var handleMessage = function(event, isLiveEvent) { // Check for empty event content @@ -156,6 +240,21 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // empty json object is a redacted event, so ignore. return; } + + // ======================= + + var __room = modelService.getRoom(event.room_id); + + if (event.user_id !== matrixService.config().user_id) { + __room.addMessageEvent(event, !isLiveEvent); + } + else { + // we may have locally echoed this, so we should replace the event + // instead of just adding. + __room.addOrReplaceMessageEvent(event, !isLiveEvent); + } + + // ======================= if (isLiveEvent) { if (event.user_id === matrixService.config().user_id && @@ -172,69 +271,10 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } else { $rootScope.events.rooms[event.room_id].messages.push(event); + displayNotification(event); } - if (window.Notification && event.user_id != matrixService.config().user_id) { - var shouldBing = notificationService.containsBingWord( - matrixService.config().user_id, - matrixService.config().display_name, - matrixService.config().bingWords, - event.content.body - ); - - // Ideally we would notify only when the window is hidden (i.e. document.hidden = true). - // - // However, Chrome on Linux and OSX currently returns document.hidden = false unless the window is - // explicitly showing a different tab. So we need another metric to determine hiddenness - we - // simply use idle time. If the user has been idle enough that their presence goes to idle, then - // we also display notifs when things happen. - // - // This is far far better than notifying whenever anything happens anyway, otherwise you get spammed - // to death with notifications when the window is in the foreground, which is horrible UX (especially - // if you have not defined any bingers and so get notified for everything). - var isIdle = (document.hidden || matrixService.presence.unavailable === mPresence.getState()); - - // We need a way to let people get notifications for everything, if they so desire. The way to do this - // is to specify zero bingwords. - var bingWords = matrixService.config().bingWords; - if (bingWords === undefined || bingWords.length === 0) { - shouldBing = true; - } - - if (shouldBing && isIdle) { - console.log("Displaying notification for "+JSON.stringify(event)); - var member = getMember(event.room_id, event.user_id); - var displayname = getUserDisplayName(event.room_id, event.user_id); - - var message = event.content.body; - if (event.content.msgtype === "m.emote") { - message = "* " + displayname + " " + message; - } - else if (event.content.msgtype === "m.image") { - message = displayname + " sent an image."; - } - - var roomTitle = matrixService.getRoomIdToAliasMapping(event.room_id); - var theRoom = $rootScope.events.rooms[event.room_id]; - if (!roomTitle && theRoom && theRoom["m.room.name"] && theRoom["m.room.name"].content) { - roomTitle = theRoom["m.room.name"].content.name; - } - - if (!roomTitle) { - roomTitle = event.room_id; - } - - notificationService.showNotification( - displayname + " (" + roomTitle + ")", - message, - member ? member.avatar_url : undefined, - function() { - console.log("notification.onclick() room=" + event.room_id); - $rootScope.goToPage('room/' + event.room_id); - } - ); - } - } + } else { $rootScope.events.rooms[event.room_id].messages.unshift(event); diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index 0ba23cb76..6e1576cd6 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -36,15 +36,33 @@ angular.module('modelService', []) this.messages = []; // events which can be displayed on the UI. TODO move? }; Room.prototype = { - addMessages: function addMessages(events, toFront) { + addMessageEvents: function addMessageEvents(events, toFront) { for (var i=0; i= 0; i--) { + var storedEvent = this.messages[i]; + if (storedEvent.event_id === event.event_id) { + // It's clobbering time! + this.messages[i] = event; + return; } } + this.addMessageEvent(event, toFront); }, leave: function leave() { @@ -81,6 +99,10 @@ angular.module('modelService', []) for (var i=0; i Date: Fri, 31 Oct 2014 15:35:39 +0000 Subject: [PATCH 056/265] Make prev_event signing work again. --- synapse/crypto/event_signing.py | 13 ++++++++++++- synapse/storage/__init__.py | 11 +++++------ synapse/storage/event_federation.py | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 61edd2c6f..07e383e22 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -16,11 +16,12 @@ from synapse.federation.units import Pdu -from synapse.api.events.utils import prune_pdu +from synapse.api.events.utils import prune_pdu, prune_event from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json, verify_signed_json +import copy import hashlib import logging @@ -69,6 +70,16 @@ def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256): return (hashed.name, hashed.digest()) +def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): + tmp_event = copy.deepcopy(event) + tmp_event = prune_event(tmp_event) + event_json = tmp_event.get_dict() + event_json.pop("signatures", None) + event_json_bytes = encode_canonical_json(event_json) + hashed = hash_algorithm(event_json_bytes) + return (hashed.name, hashed.digest()) + + def sign_event_pdu(pdu, signature_name, signing_key): tmp_pdu = Pdu(**pdu.get_dict()) tmp_pdu = prune_pdu(tmp_pdu) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c2560f604..31a0022d5 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -46,7 +46,7 @@ from .signatures import SignatureStore from syutil.base64util import decode_base64 -from synapse.crypto.event_signing import compute_pdu_event_reference_hash +from synapse.crypto.event_signing import compute_event_reference_hash import json @@ -271,11 +271,10 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, prev_event_id, alg, hash_bytes ) - # TODO - # (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu) - # self._store_event_reference_hash_txn( - # txn, event.event_id, ref_alg, ref_hash_bytes - # ) + (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) + self._store_event_reference_hash_txn( + txn, event.event_id, ref_alg, ref_hash_bytes + ) self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 8357071db..dcc116bad 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -69,7 +69,7 @@ class EventFederationStore(SQLBaseStore): results = [] for event_id, depth in txn.fetchall(): - hashes = self._get_prev_event_hashes_txn(txn, event_id) + hashes = self._get_event_reference_hashes_txn(txn, event_id) prev_hashes = { k: encode_base64(v) for k, v in hashes.items() if k == "sha256" From fb3a01fa3a2603751ac427f90597d921c8662417 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 16:04:27 +0000 Subject: [PATCH 057/265] Remove unused sql file. --- synapse/storage/schema/edge_pdus.sql | 31 ---------------------------- 1 file changed, 31 deletions(-) delete mode 100644 synapse/storage/schema/edge_pdus.sql diff --git a/synapse/storage/schema/edge_pdus.sql b/synapse/storage/schema/edge_pdus.sql deleted file mode 100644 index 8a0086806..000000000 --- a/synapse/storage/schema/edge_pdus.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright 2014 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -CREATE TABLE IF NOT EXISTS context_edge_pdus( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- twistar requires this - pdu_id TEXT, - origin TEXT, - context TEXT, - CONSTRAINT context_edge_pdu_id_origin UNIQUE (pdu_id, origin) -); - -CREATE TABLE IF NOT EXISTS origin_edge_pdus( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- twistar requires this - pdu_id TEXT, - origin TEXT, - CONSTRAINT origin_edge_pdu_id_origin UNIQUE (pdu_id, origin) -); - -CREATE INDEX IF NOT EXISTS context_edge_pdu_id ON context_edge_pdus(pdu_id, origin); -CREATE INDEX IF NOT EXISTS origin_edge_pdu_id ON origin_edge_pdus(pdu_id, origin); From b0f0b7b75e2887be546ac7c5e1d3dabec8d7052c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 31 Oct 2014 16:22:15 +0000 Subject: [PATCH 058/265] room.html now displays messages from model-service. Add debugging fields. Hook up the room member *at the time* to the message so it can display the right historical member info. --- .../components/matrix/event-handler-service.js | 9 ++++++++- webclient/components/matrix/model-service.js | 16 ++++++++++------ webclient/index.html | 2 +- webclient/room/room-controller.js | 11 +++++++---- webclient/room/room.html | 10 +++++----- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 589554ec0..a511b394b 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -96,7 +96,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService __room.old_room_state.storeStateEvents(room.state); __room.old_room_state.pagination_token = room.messages.start; - __room.addMessageEvents(room.messages.chunk); + $rootScope["debug_"+room_id] = __room; } }; @@ -589,6 +589,10 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // Store how far back we've paginated $rootScope.events.rooms[room_id].pagination.earliest_token = messages.end; + + var __room = modelService.getRoom(room_id); + __room.old_room_state.pagination_token = messages.end; + } else { // InitialSync returns messages in chronological order @@ -597,6 +601,9 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } // Store where to start pagination $rootScope.events.rooms[room_id].pagination.earliest_token = messages.start; + + var __room = modelService.getRoom(room_id); + __room.old_room_state.pagination_token = messages.start; } }, diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index 6e1576cd6..d47ac6b2a 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -33,7 +33,7 @@ angular.module('modelService', []) this.room_id = room_id; this.old_room_state = new RoomState(); this.current_room_state = new RoomState(); - this.messages = []; // events which can be displayed on the UI. TODO move? + this.events = []; // events which can be displayed on the UI. TODO move? }; Room.prototype = { addMessageEvents: function addMessageEvents(events, toFront) { @@ -43,22 +43,26 @@ angular.module('modelService', []) }, addMessageEvent: function addMessageEvent(event, toFront) { + // every message must reference the RoomMember which made it *at + // that time* so things like display names display correctly. if (toFront) { - this.messages.unshift(event); + event.room_member = this.old_room_state.getStateEvent("m.room.member", event.user_id); + this.events.unshift(event); } else { - this.messages.push(event); + event.room_member = this.current_room_state.getStateEvent("m.room.member", event.user_id); + this.events.push(event); } }, addOrReplaceMessageEvent: function addOrReplaceMessageEvent(event, toFront) { // Start looking from the tail since the first goal of this function // is to find a message among the latest ones - for (var i = this.messages.length - 1; i >= 0; i--) { - var storedEvent = this.messages[i]; + for (var i = this.events.length - 1; i >= 0; i--) { + var storedEvent = this.events[i]; if (storedEvent.event_id === event.event_id) { // It's clobbering time! - this.messages[i] = event; + this.events[i] = event; return; } } diff --git a/webclient/index.html b/webclient/index.html index 965981f7d..3ed968a5e 100644 --- a/webclient/index.html +++ b/webclient/index.html @@ -13,7 +13,7 @@ - + diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 486ead0da..7125c98e0 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -15,8 +15,8 @@ limitations under the License. */ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) -.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'notificationService', - function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, notificationService) { +.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'notificationService', 'modelService', + function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, notificationService, modelService) { 'use strict'; var MESSAGES_PER_PAGINATION = 30; var THUMBNAIL_SIZE = 320; @@ -254,11 +254,11 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) $scope.state.paginating = true; } - console.log("paginateBackMessages from " + $rootScope.events.rooms[$scope.room_id].pagination.earliest_token + " for " + numItems); + console.log("paginateBackMessages from " + $scope.room.old_room_state.pagination_token + " for " + numItems); var originalTopRow = $("#messageTable>tbody>tr:first")[0]; // Paginate events from the point in cache - matrixService.paginateBackMessages($scope.room_id, $rootScope.events.rooms[$scope.room_id].pagination.earliest_token, numItems).then( + matrixService.paginateBackMessages($scope.room_id, $scope.room.old_room_state.pagination_token, numItems).then( function(response) { eventHandlerService.handleRoomMessages($scope.room_id, response.data, false, 'b'); @@ -717,6 +717,9 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) var onInit2 = function() { console.log("onInit2"); + // ============================= + $scope.room = modelService.getRoom($scope.room_id); + // ============================= // Scroll down as soon as possible so that we point to the last message // if it already exists in memory diff --git a/webclient/room/room.html b/webclient/room/room.html index 5265f42dd..8c1e45dfd 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -123,10 +123,10 @@ ng-style="{ 'visibility': state.messages_visibility }" keep-scroll> - +
-
{{ msg.user_id | mUserDisplayName: room_id }}
+
{{ msg.user_id | mUserDisplayName: room_id }}
{{ (msg.origin_server_ts) | date:'MMM d HH:mm' }} @@ -134,7 +134,7 @@
+ ng-hide="room.events[$index - 1].user_id === msg.user_id || msg.user_id === state.user_id"/>
@@ -204,7 +204,7 @@
+ ng-hide="room.events[$index - 1].user_id === msg.user_id || msg.user_id !== state.user_id"/>
From 80b2710e6f9b7520f49f82b117634294c63fdb1e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 17:08:36 +0000 Subject: [PATCH 059/265] Remove unused signature storage methods --- synapse/storage/signatures.py | 139 +--------------------------------- 1 file changed, 1 insertion(+), 138 deletions(-) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 5e99174fc..b4b3d5d7e 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -17,144 +17,7 @@ from _base import SQLBaseStore class SignatureStore(SQLBaseStore): - """Persistence for PDU signatures and hashes""" - - def _get_pdu_content_hashes_txn(self, txn, pdu_id, origin): - """Get all the hashes for a given PDU. - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - Returns: - A dict of algorithm -> hash. - """ - query = ( - "SELECT algorithm, hash" - " FROM pdu_content_hashes" - " WHERE pdu_id = ? and origin = ?" - ) - txn.execute(query, (pdu_id, origin)) - return dict(txn.fetchall()) - - def _store_pdu_content_hash_txn(self, txn, pdu_id, origin, algorithm, - hash_bytes): - """Store a hash for a PDU - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - algorithm (str): Hashing algorithm. - hash_bytes (bytes): Hash function output bytes. - """ - self._simple_insert_txn(txn, "pdu_content_hashes", { - "pdu_id": pdu_id, - "origin": origin, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) - - def _get_pdu_reference_hashes_txn(self, txn, pdu_id, origin): - """Get all the hashes for a given PDU. - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - Returns: - A dict of algorithm -> hash. - """ - query = ( - "SELECT algorithm, hash" - " FROM pdu_reference_hashes" - " WHERE pdu_id = ? and origin = ?" - ) - txn.execute(query, (pdu_id, origin)) - return dict(txn.fetchall()) - - def _store_pdu_reference_hash_txn(self, txn, pdu_id, origin, algorithm, - hash_bytes): - """Store a hash for a PDU - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - algorithm (str): Hashing algorithm. - hash_bytes (bytes): Hash function output bytes. - """ - self._simple_insert_txn(txn, "pdu_reference_hashes", { - "pdu_id": pdu_id, - "origin": origin, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) - - - def _get_pdu_origin_signatures_txn(self, txn, pdu_id, origin): - """Get all the signatures for a given PDU. - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - Returns: - A dict of key_id -> signature_bytes. - """ - query = ( - "SELECT key_id, signature" - " FROM pdu_origin_signatures" - " WHERE pdu_id = ? and origin = ?" - ) - txn.execute(query, (pdu_id, origin)) - return dict(txn.fetchall()) - - def _store_pdu_origin_signature_txn(self, txn, pdu_id, origin, key_id, - signature_bytes): - """Store a signature from the origin server for a PDU. - Args: - txn (cursor): - pdu_id (str): Id for the PDU. - origin (str): origin of the PDU. - key_id (str): Id for the signing key. - signature (bytes): The signature. - """ - self._simple_insert_txn(txn, "pdu_origin_signatures", { - "pdu_id": pdu_id, - "origin": origin, - "key_id": key_id, - "signature": buffer(signature_bytes), - }) - - def _get_prev_pdu_hashes_txn(self, txn, pdu_id, origin): - """Get all the hashes for previous PDUs of a PDU - Args: - txn (cursor): - pdu_id (str): Id of the PDU. - origin (str): Origin of the PDU. - Returns: - dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. - """ - query = ( - "SELECT prev_pdu_id, prev_origin, algorithm, hash" - " FROM pdu_edge_hashes" - " WHERE pdu_id = ? and origin = ?" - ) - txn.execute(query, (pdu_id, origin)) - results = {} - for prev_pdu_id, prev_origin, algorithm, hash_bytes in txn.fetchall(): - hashes = results.setdefault((prev_pdu_id, prev_origin), {}) - hashes[algorithm] = hash_bytes - return results - - def _store_prev_pdu_hash_txn(self, txn, pdu_id, origin, prev_pdu_id, - prev_origin, algorithm, hash_bytes): - self._simple_insert_txn(txn, "pdu_edge_hashes", { - "pdu_id": pdu_id, - "origin": origin, - "prev_pdu_id": prev_pdu_id, - "prev_origin": prev_origin, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }) - - ## Events ## + """Persistence for event signatures and hashes""" def _get_event_content_hashes_txn(self, txn, event_id): """Get all the hashes for a given Event. From ecabff7eb49ea799d9f52fad1e05f1f9a4b31e1c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 17:08:52 +0000 Subject: [PATCH 060/265] Sign evnets --- synapse/crypto/event_signing.py | 20 ++++++++++++++++++++ synapse/storage/__init__.py | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 07e383e22..cb2db01c0 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -94,3 +94,23 @@ def verify_signed_event_pdu(pdu, signature_name, verify_key): tmp_pdu = prune_pdu(tmp_pdu) pdu_json = tmp_pdu.get_dict() verify_signed_json(pdu_json, signature_name, verify_key) + + +def add_hashes_and_signatures(event, signature_name, signing_key, + hash_algorithm=hashlib.sha256): + tmp_event = copy.deepcopy(event) + tmp_event = prune_event(tmp_event) + redact_json = tmp_event.get_dict() + redact_json.pop("signatures", None) + redact_json = sign_json(redact_json, signature_name, signing_key) + event.signatures = redact_json["signatures"] + + event_json = event.get_full_dict() + #TODO: We need to sign the JSON that is going out via fedaration. + event_json.pop("age_ts", None) + event_json.pop("unsigned", None) + event_json.pop("signatures", None) + event_json.pop("hashes", None) + event_json_bytes = encode_canonical_json(event_json) + hashed = hash_algorithm(event_json_bytes) + event.hashes[hashed.name] = encode_base64(hashed.digest()) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 31a0022d5..1f39a4094 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -255,6 +255,12 @@ class DataStore(RoomMemberStore, RoomStore, } ) + for hash_alg, hash_base64 in event.hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_event_content_hash_txn( + txn, event.event_id, hash_alg, hash_bytes, + ) + if hasattr(event, "signatures"): signatures = event.signatures.get(event.origin, {}) From f21960ec9d346d53a21369b8179cc30365654517 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 31 Oct 2014 17:13:27 +0000 Subject: [PATCH 061/265] Replace lots of .events.rooms[room_id] with .room --- .../matrix/event-handler-service.js | 6 ++++++ webclient/components/matrix/model-service.js | 2 +- webclient/room/room-controller.js | 19 +++++++++---------- webclient/room/room.html | 11 ++++++----- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index a511b394b..f8a91ee0a 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -289,6 +289,8 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService }; var handleRoomMember = function(event, isLiveEvent, isStateEvent) { + var __room = modelService.getRoom(event.room_id); + // add membership changes as if they were a room message if something interesting changed // Exception: Do not do this if the event is a room state event because such events already come @@ -317,6 +319,9 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService else { $rootScope.events.rooms[event.room_id].messages.unshift(event); } + // ============ + + __room.addMessageEvent(event, !isLiveEvent); } } @@ -324,6 +329,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // Do not care of events that come when paginating back if (isStateEvent || isLiveEvent) { $rootScope.events.rooms[event.room_id].members[event.state_key] = event; + __room.current_room_state.members[event.state_key] = event; } $rootScope.$broadcast(MEMBER_EVENT, event, isLiveEvent, isStateEvent); diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index d47ac6b2a..6167a84c6 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -77,7 +77,7 @@ angular.module('modelService', []) /***** Room State Object *****/ var RoomState = function RoomState() { // list of RoomMember - this.members = []; + this.members = {}; // state events, the key is a compound of event type + state_key this.state_events = {}; this.pagination_token = ""; diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 7125c98e0..b5f2b6041 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -64,7 +64,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) return; }; - var nameEvent = $rootScope.events.rooms[$scope.room_id]['m.room.name']; + var nameEvent = $scope.room.current_room_state.state_events['m.room.name']; if (nameEvent) { $scope.name.newNameText = nameEvent.content.name; } @@ -105,7 +105,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) console.log("Warning: Already editing topic."); return; } - var topicEvent = $rootScope.events.rooms[$scope.room_id]['m.room.topic']; + var topicEvent = $scope.room.current_room_state.state_events['m.room.topic']; if (topicEvent) { $scope.topic.newTopicText = topicEvent.content.topic; } @@ -492,7 +492,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) var room_id = matrixService.getAliasToRoomIdMapping(room_alias); console.log("joining " + room_alias + " id=" + room_id); - if ($rootScope.events.rooms[room_id]) { + if ($scope.room) { // TODO actually check that you = join // don't send a join event for a room you're already in. $location.url("room/" + room_alias); } @@ -629,7 +629,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) }; $('#mainInput').val(''); - $rootScope.events.rooms[$scope.room_id].messages.push(echoMessage); + $scope.room.addMessageEvent(echoMessage); scrollToBottom(); } @@ -732,9 +732,9 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) var needsToJoin = true; // The room members is available in the data fetched by initialSync - if ($rootScope.events.rooms[$scope.room_id]) { + if ($scope.room) { - var messages = $rootScope.events.rooms[$scope.room_id].messages; + var messages = $scope.room.events; if (0 === messages.length || (1 === messages.length && "m.room.member" === messages[0].type && "invite" === messages[0].content.membership && $scope.state.user_id === messages[0].state_key)) { @@ -746,7 +746,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) $scope.state.first_pagination = false; } - var members = $rootScope.events.rooms[$scope.room_id].members; + var members = $scope.room.current_room_state.members; // Update the member list for (var i in members) { @@ -1042,8 +1042,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) state_key: "" }; - var stateFilter = $filter("stateEventsFilter"); - var stateEvents = stateFilter($scope.events.rooms[$scope.room_id]); + var stateEvents = $scope.room.current_room_state.state_events; // The modal dialog will 2-way bind this field, so we MUST make a deep // copy of the state events else we will be *actually adjusing our view // of the world* when fiddling with the JSON!! Apparently parse/stringify @@ -1062,7 +1061,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) console.log("Displaying modal dialog for >>>> " + JSON.stringify($scope.event_selected)); $scope.redact = function() { console.log("User level = "+$scope.pow($scope.room_id, $scope.state.user_id)+ - " Redact level = "+$scope.events.rooms[$scope.room_id]["m.room.ops_levels"].content.redact_level); + " Redact level = "+$scope.room.current_room_state.state_events["m.room.ops_levels"].content.redact_level); console.log("Redact event >> " + JSON.stringify($scope.event_selected)); $modalInstance.close("redact"); }; diff --git a/webclient/room/room.html b/webclient/room/room.html index 8c1e45dfd..1cec8ac70 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -6,7 +6,7 @@
@@ -18,7 +18,8 @@
- {{ key }} + {{ event.type }} + ({{event.state_key}})
{{ (event.origin_server_ts) | date:'MMM d HH:mm' }}
@@ -68,13 +69,13 @@
- -
+
- {{ events.rooms[room_id]['m.room.topic'].content.topic | limitTo: 200}} + {{ room.current_room_state.state_events['m.room.topic'].content.topic | limitTo: 200}}
From 13fad062399bf7398d8a2c48d5ffa1e6a3d8a2cb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 31 Oct 2014 17:23:01 +0000 Subject: [PATCH 062/265] Bump version numbers and change log --- CHANGES.rst | 16 ++++++++++++++++ VERSION | 2 +- synapse/__init__.py | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 08efbbf24..78c178baf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,19 @@ +Changes in synapse 0.4.2 (2014-10-31) +===================================== + +Homeserver: + * Fix bugs where we did not notify users of correct presence updates. + * Fix bug where we did not handle sub second event stream timeouts. + +Webclient: + * Add ability to click on messages to see JSON. + * Add ability to redact messages. + * Add ability to view and edit all room state JSON. + * Handle incoming redactions. + * Improve feedback on errors. + * Fix bugs in mobile CSS. + * Fix bugs with desktop notifications. + Changes in synapse 0.4.1 (2014-10-17) ===================================== Webclient: diff --git a/VERSION b/VERSION index 267577d47..2b7c5ae01 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.1 +0.4.2 diff --git a/synapse/__init__.py b/synapse/__init__.py index 7067188c5..23ae5f003 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.4.1" +__version__ = "0.4.2" From 5ab9929cbb3dd7b908011888e79fdb7bf54a5296 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 09:44:20 +0000 Subject: [PATCH 063/265] Prevent EventStreamService from knowing too much about the EventHandlerService by changing the contract to just be a single initialSync response callback. Leave it up the handler to deal with splitting out information from /initialSync. --- .../matrix/event-handler-service.js | 22 +++++++++++++++++-- .../components/matrix/event-stream-service.js | 19 ---------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index f8a91ee0a..c11391a6d 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -613,9 +613,27 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } }, - handleInitialSyncDone: function(initialSyncData) { + handleInitialSyncDone: function(response) { console.log("# handleInitialSyncDone"); - initialSyncDeferred.resolve(initialSyncData); + + var rooms = response.data.rooms; + for (var i = 0; i < rooms.length; ++i) { + var room = rooms[i]; + + this.initRoom(room); + + if ("messages" in room) { + this.handleRoomMessages(room.room_id, room.messages, false); + } + + if ("state" in room) { + this.handleEvents(room.state, false, true); + } + } + var presence = response.data.presence; + this.handleEvents(presence, false); + + initialSyncDeferred.resolve(response); }, // Returns a promise that resolves when the initialSync request has been processed diff --git a/webclient/components/matrix/event-stream-service.js b/webclient/components/matrix/event-stream-service.js index 05469a3de..c03f0b953 100644 --- a/webclient/components/matrix/event-stream-service.js +++ b/webclient/components/matrix/event-stream-service.js @@ -109,25 +109,6 @@ angular.module('eventStreamService', []) // without requiring to make an additional request matrixService.initialSync(30, false).then( function(response) { - var rooms = response.data.rooms; - for (var i = 0; i < rooms.length; ++i) { - var room = rooms[i]; - - eventHandlerService.initRoom(room); - - if ("messages" in room) { - eventHandlerService.handleRoomMessages(room.room_id, room.messages, false); - } - - if ("state" in room) { - eventHandlerService.handleEvents(room.state, false, true); - } - } - - var presence = response.data.presence; - eventHandlerService.handleEvents(presence, false); - - // Initial sync is done eventHandlerService.handleInitialSyncDone(response); // Start event streaming from that point From 1c6825cc7a3e4446eea54905d5a97008b56c1138 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 10:16:28 +0000 Subject: [PATCH 064/265] Use python logger, not the twisted logger --- synapse/handlers/register.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 88eb51a8e..7df9d9b82 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -15,7 +15,6 @@ """Contains functions for registering clients.""" from twisted.internet import defer -from twisted.python import log from synapse.types import UserID from synapse.api.errors import ( @@ -129,7 +128,7 @@ class RegistrationHandler(BaseHandler): try: threepid = yield self._threepid_from_creds(c) except: - log.err() + logger.exception("Couldn't validate 3pid") raise RegistrationError(400, "Couldn't validate 3pid") if not threepid: From 395bb64b2677465cdb6bd2dc2d659dd59f583e8a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 10:23:14 +0000 Subject: [PATCH 065/265] Keep matrixService stateless and make matrixFilter use modelService. --- .../matrix/event-handler-service.js | 21 +++++++ webclient/components/matrix/matrix-filter.js | 15 ++--- webclient/components/matrix/matrix-service.js | 57 +++++-------------- webclient/components/matrix/model-service.js | 5 +- webclient/room/room-controller.js | 10 ++-- 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index c11391a6d..dc00d83f3 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -705,6 +705,27 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService return getMember(room_id, user_id); }, + /** + * Return the power level of an user in a particular room + * @param {String} room_id the room id + * @param {String} user_id the user id + * @returns {Number} a value between 0 and 10 + */ + getUserPowerLevel: function(room_id, user_id) { + var powerLevel = 0; + var room = $rootScope.events.rooms[room_id]; + if (room && room["m.room.power_levels"]) { + if (user_id in room["m.room.power_levels"].content) { + powerLevel = room["m.room.power_levels"].content[user_id]; + } + else { + // Use the room default user power + powerLevel = room["m.room.power_levels"].content["default"]; + } + } + return powerLevel; + }, + /** * Return the display name of an user acccording to data already downloaded * @param {String} room_id the room id diff --git a/webclient/components/matrix/matrix-filter.js b/webclient/components/matrix/matrix-filter.js index 3d64a569a..e4b42a2ed 100644 --- a/webclient/components/matrix/matrix-filter.js +++ b/webclient/components/matrix/matrix-filter.js @@ -19,23 +19,24 @@ angular.module('matrixFilter', []) // Compute the room name according to information we have -.filter('mRoomName', ['$rootScope', 'matrixService', 'eventHandlerService', function($rootScope, matrixService, eventHandlerService) { +.filter('mRoomName', ['$rootScope', 'matrixService', 'eventHandlerService', 'modelService', +function($rootScope, matrixService, eventHandlerService, modelService) { return function(room_id) { var roomName; // If there is an alias, use it // TODO: only one alias is managed for now var alias = matrixService.getRoomIdToAliasMapping(room_id); - - var room = $rootScope.events.rooms[room_id]; + var room = modelService.getRoom(room_id).current_room_state; + if (room) { // Get name from room state date - var room_name_event = room["m.room.name"]; + var room_name_event = room.state("m.room.name"); // Determine if it is a public room var isPublicRoom = false; - if (room["m.room.join_rules"] && room["m.room.join_rules"].content) { - isPublicRoom = ("public" === room["m.room.join_rules"].content.join_rule); + if (room.state("m.room.join_rules") && room.state("m.room.join_rules").content) { + isPublicRoom = ("public" === room.state("m.room.join_rules").content.join_rule); } if (room_name_event) { @@ -44,7 +45,7 @@ angular.module('matrixFilter', []) else if (alias) { roomName = alias; } - else if (room.members && !isPublicRoom) { // Do not rename public room + else if (room.members.length > 0 && !isPublicRoom) { // Do not rename public room var user_id = matrixService.config().user_id; // Else, build the name from its users diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index 1840cf46c..d089bf601 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -724,57 +724,30 @@ angular.module('matrixService', []) //console.log("looking for roomId for " + alias + "; found: " + roomId); return roomId; }, - - /****** Power levels management ******/ - - /** - * Return the power level of an user in a particular room - * @param {String} room_id the room id - * @param {String} user_id the user id - * @returns {Number} a value between 0 and 10 - */ - getUserPowerLevel: function(room_id, user_id) { - var powerLevel = 0; - var room = $rootScope.events.rooms[room_id]; - if (room && room["m.room.power_levels"]) { - if (user_id in room["m.room.power_levels"].content) { - powerLevel = room["m.room.power_levels"].content[user_id]; - } - else { - // Use the room default user power - powerLevel = room["m.room.power_levels"].content["default"]; - } - } - return powerLevel; - }, /** * Change or reset the power level of a user * @param {String} room_id the room id * @param {String} user_id the user id - * @param {Number} powerLevel a value between 0 and 10 + * @param {Number} powerLevel The desired power level. * If undefined, the user power level will be reset, ie he will use the default room user power level + * @param event The existing m.room.power_levels event if one exists. * @returns {promise} an $http promise */ - setUserPowerLevel: function(room_id, user_id, powerLevel) { - - // Hack: currently, there is no home server API so do it by hand by updating - // the current m.room.power_levels of the room and send it to the server - var room = $rootScope.events.rooms[room_id]; - if (room && room["m.room.power_levels"]) { - var content = angular.copy(room["m.room.power_levels"].content); - content[user_id] = powerLevel; - - var path = "/rooms/$room_id/state/m.room.power_levels"; - path = path.replace("$room_id", encodeURIComponent(room_id)); - - return doRequest("PUT", path, undefined, content); + setUserPowerLevel: function(room_id, user_id, powerLevel, event) { + var content = {}; + if (event) { + // if there is an existing event, copy the content as it contains + // the power level values for other members which we do not want + // to modify. + content = angular.copy(event.content); } - - // The room does not exist or does not contain power_levels data - var deferred = $q.defer(); - deferred.reject({data:{error: "Invalid room: " + room_id}}); - return deferred.promise; + content[user_id] = powerLevel; + + var path = "/rooms/$room_id/state/m.room.power_levels"; + path = path.replace("$room_id", encodeURIComponent(room_id)); + + return doRequest("PUT", path, undefined, content); }, getTurnServer: function() { diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index 6167a84c6..26349f146 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -89,8 +89,11 @@ angular.module('modelService', []) // provided which can just be given the type and it will return the // 0-len event by default. state: function state(type, state_key) { + if (!type) { + return undefined; // event type MUST be specified + } if (!state_key) { - return this.state_events[type]; + return this.state_events[type]; // treat as 0-len state key } return this.state_events[type + state_key]; }, diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index b5f2b6041..38158ffc7 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -404,7 +404,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) var updateUserPowerLevel = function(user_id) { var member = $scope.members[user_id]; if (member) { - member.powerLevel = matrixService.getUserPowerLevel($scope.room_id, user_id); + member.powerLevel = eventHandlerService.getUserPowerLevel($scope.room_id, user_id); normaliseMembersPowerLevels(); } @@ -576,7 +576,8 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) powerLevel = parseInt(matches[3]); } if (powerLevel !== NaN) { - promise = matrixService.setUserPowerLevel($scope.room_id, user_id, powerLevel); + var powerLevelEvent = $scope.room.current_room_state.state("m.room.power_levels"); + promise = matrixService.setUserPowerLevel($scope.room_id, user_id, powerLevel, powerLevelEvent); } } } @@ -591,7 +592,8 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) if (args) { var matches = args.match(/^(\S+)$/); if (matches) { - promise = matrixService.setUserPowerLevel($scope.room_id, args, undefined); + var powerLevelEvent = $scope.room.current_room_state.state("m.room.power_levels"); + promise = matrixService.setUserPowerLevel($scope.room_id, args, undefined, powerLevelEvent); } } @@ -1005,7 +1007,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) $scope.event_selected = content; // scope this so the template can check power levels and enable/disable // buttons - $scope.pow = matrixService.getUserPowerLevel; + $scope.pow = eventHandlerService.getUserPowerLevel; var modalInstance = $modal.open({ templateUrl: 'eventInfoTemplate.html', From 53da1099d132e96f9b4933a3eab1d34ba62c47d4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 10:33:38 +0000 Subject: [PATCH 066/265] Make call stuff use modelService. --- webclient/app-controller.js | 6 +++--- webclient/components/matrix/matrix-call.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/webclient/app-controller.js b/webclient/app-controller.js index e4b7cd286..2d82a42cf 100644 --- a/webclient/app-controller.js +++ b/webclient/app-controller.js @@ -21,8 +21,8 @@ limitations under the License. 'use strict'; angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'eventStreamService']) -.controller('MatrixWebClientController', ['$scope', '$location', '$rootScope', '$timeout', '$animate', 'matrixService', 'mPresence', 'eventStreamService', 'eventHandlerService', 'matrixPhoneService', - function($scope, $location, $rootScope, $timeout, $animate, matrixService, mPresence, eventStreamService, eventHandlerService, matrixPhoneService) { +.controller('MatrixWebClientController', ['$scope', '$location', '$rootScope', '$timeout', '$animate', 'matrixService', 'mPresence', 'eventStreamService', 'eventHandlerService', 'matrixPhoneService', 'modelService', + function($scope, $location, $rootScope, $timeout, $animate, matrixService, mPresence, eventStreamService, eventHandlerService, matrixPhoneService, modelService) { // Check current URL to avoid to display the logout button on the login page $scope.location = $location.path(); @@ -117,7 +117,7 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even return; } - var roomMembers = angular.copy($rootScope.events.rooms[$rootScope.currentCall.room_id].members); + var roomMembers = angular.copy(modelService.getRoom($rootScope.currentCall.room_id).current_room_state.members); delete roomMembers[matrixService.config().user_id]; $rootScope.currentCall.user_id = Object.keys(roomMembers)[0]; diff --git a/webclient/components/matrix/matrix-call.js b/webclient/components/matrix/matrix-call.js index 3e8811e5f..5a2807c75 100644 --- a/webclient/components/matrix/matrix-call.js +++ b/webclient/components/matrix/matrix-call.js @@ -46,7 +46,7 @@ var isWebRTCSupported = function () { }; angular.module('MatrixCall', []) -.factory('MatrixCall', ['matrixService', 'matrixPhoneService', '$rootScope', '$timeout', function MatrixCallFactory(matrixService, matrixPhoneService, $rootScope, $timeout) { +.factory('MatrixCall', ['matrixService', 'matrixPhoneService', 'modelService', '$rootScope', '$timeout', function MatrixCallFactory(matrixService, matrixPhoneService, modelService, $rootScope, $timeout) { $rootScope.isWebRTCSupported = isWebRTCSupported(); var MatrixCall = function(room_id) { @@ -213,7 +213,7 @@ angular.module('MatrixCall', []) var self = this; - var roomMembers = $rootScope.events.rooms[this.room_id].members; + var roomMembers = modelService.getRoom(this.room_id).current_room_state.members; if (roomMembers[matrixService.config().user_id].membership != 'join') { console.log("We need to join the room before we can accept this call"); matrixService.join(this.room_id).then(function() { From 9024a196583d6abf7a23ea3c7a90501f0d9dad16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:31:47 +0000 Subject: [PATCH 067/265] Remove dead code. --- synapse/state.py | 84 ------------------------------------------------ 1 file changed, 84 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 2548deed2..f4efc287c 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -65,12 +65,6 @@ class StateHandler(object): if not hasattr(event, "state_key"): return - key = KeyStateTuple( - event.room_id, - event.type, - _get_state_key_from_event(event) - ) - # Now I need to fill out the prev state and work out if it has auth # (w.r.t. to power levels) @@ -85,10 +79,6 @@ class StateHandler(object): if current_state: event.prev_state = current_state.event_id - # TODO check current_state to see if the min power level is less - # than the power level of the user - # power_level = self._get_power_level_for_event(event) - defer.returnValue(True) @defer.inlineCallbacks @@ -221,77 +211,3 @@ class StateHandler(object): ).hexdigest() )[0] ) - - def _get_power_level_for_event(self, event): - # return self._persistence.get_power_level_for_user(event.room_id, - # event.sender) - return event.power_level - - @defer.inlineCallbacks - def _do_conflict_res(self, new_branch, current_branch, common_ancestor): - conflict_res = [ - self._do_power_level_conflict_res, - self._do_chain_length_conflict_res, - self._do_hash_conflict_res, - ] - - for algo in conflict_res: - new_res, curr_res = yield defer.maybeDeferred( - algo, - new_branch, current_branch, common_ancestor - ) - - if new_res < curr_res: - defer.returnValue(False) - elif new_res > curr_res: - defer.returnValue(True) - - raise Exception("Conflict resolution failed.") - - @defer.inlineCallbacks - def _do_power_level_conflict_res(self, new_branch, current_branch, - common_ancestor): - new_powers_deferreds = [] - for e in new_branch[:-1] if common_ancestor else new_branch: - if hasattr(e, "user_id"): - new_powers_deferreds.append( - self.store.get_power_level(e.context, e.user_id) - ) - - current_powers_deferreds = [] - for e in current_branch[:-1] if common_ancestor else current_branch: - if hasattr(e, "user_id"): - current_powers_deferreds.append( - self.store.get_power_level(e.context, e.user_id) - ) - - new_powers = yield defer.gatherResults( - new_powers_deferreds, - consumeErrors=True - ) - - current_powers = yield defer.gatherResults( - current_powers_deferreds, - consumeErrors=True - ) - - max_power_new = max(new_powers) - max_power_current = max(current_powers) - - defer.returnValue( - (max_power_new, max_power_current) - ) - - def _do_chain_length_conflict_res(self, new_branch, current_branch, - common_ancestor): - return (len(new_branch), len(current_branch)) - - def _do_hash_conflict_res(self, new_branch, current_branch, - common_ancestor): - new_str = "".join([p.pdu_id + p.origin for p in new_branch]) - c_str = "".join([p.pdu_id + p.origin for p in current_branch]) - - return ( - hashlib.sha1(new_str).hexdigest(), - hashlib.sha1(c_str).hexdigest() - ) From 82a6b83524ed2f7cb12bd2fc43a2651558c392dd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:32:12 +0000 Subject: [PATCH 068/265] Don't assume event has hashes key already --- synapse/crypto/event_signing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index cb2db01c0..0e8bc7eb6 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -113,4 +113,6 @@ def add_hashes_and_signatures(event, signature_name, signing_key, event_json.pop("hashes", None) event_json_bytes = encode_canonical_json(event_json) hashed = hash_algorithm(event_json_bytes) + if not hasattr(event, "hashes"): + event.hashes = {} event.hashes[hashed.name] = encode_base64(hashed.digest()) From 0a8b026ccf2d2472bb14303eaf5af8b8cf0efa5d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:32:42 +0000 Subject: [PATCH 069/265] Add 'origin' key to events --- synapse/api/events/factory.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 750096c61..9134c82ef 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -69,6 +69,10 @@ class EventFactory(object): kwargs["type"] = etype if "event_id" not in kwargs: kwargs["event_id"] = self.create_event_id() + kwargs["origin"] = self.hs.hostname + else: + ev_id = self.hs.parse_eventid(kwargs["event_id"]) + kwargs["origin"] = ev_id.domain if "origin_server_ts" not in kwargs: kwargs["origin_server_ts"] = int(self.clock.time_msec()) From 7249785bcbe86a4e1c7f6b4f4f5b0601f8a8c47a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:33:28 +0000 Subject: [PATCH 070/265] Sign events we create. --- synapse/handlers/_base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 787a01efc..28b64565a 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,6 +18,8 @@ from synapse.api.errors import LimitExceededError from synapse.util.async import run_on_reactor +from synapse.crypto.event_signing import add_hashes_and_signatures + class BaseHandler(object): def __init__(self, hs): @@ -32,6 +34,9 @@ class BaseHandler(object): self.clock = hs.get_clock() self.hs = hs + self.signing_key = hs.config.signing_key[0] + self.server_name = hs.hostname + def ratelimit(self, user_id): time_now = self.clock.time() allowed, time_allowed = self.ratelimiter.send_message( @@ -53,6 +58,10 @@ class BaseHandler(object): yield self.state_handler.annotate_state_groups(event) + yield add_hashes_and_signatures( + event, self.server_name, self.signing_key + ) + if not suppress_auth: yield self.auth.check(event, snapshot, raises=True) From f139c02e95e55f793c86ae6b7ad079d93aae0754 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:34:49 +0000 Subject: [PATCH 071/265] Formatting --- synapse/storage/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1f39a4094..6b8fed450 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -199,7 +199,10 @@ class DataStore(RoomMemberStore, RoomStore, k: v for k, v in event.get_full_dict().items() if k not in vals.keys() and k not in [ - "redacted", "redacted_because", "signatures", "hashes", + "redacted", + "redacted_because", + "signatures", + "hashes", "prev_events", ] } From d59aa6af25472a15e6258f2fd9cbd1ac4c05702d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 11:35:19 +0000 Subject: [PATCH 072/265] For now, don't store txn -> pdu mappings. --- synapse/federation/persistence.py | 1 - synapse/storage/transactions.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index a565375e6..b04fbb417 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -88,7 +88,6 @@ class TransactionActions(object): transaction.transaction_id, transaction.destination, transaction.origin_server_ts, - [(p["pdu_id"], p["origin"]) for p in transaction.pdus] ) @log_function diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index ea6790078..00d0f4808 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -89,7 +89,7 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (code, response_json, transaction_id, origin)) def prep_send_transaction(self, transaction_id, destination, - origin_server_ts, pdu_list): + origin_server_ts): """Persists an outgoing transaction and calculates the values for the previous transaction id list. @@ -100,7 +100,6 @@ class TransactionStore(SQLBaseStore): transaction_id (str) destination (str) origin_server_ts (int) - pdu_list (list) Returns: list: A list of previous transaction ids. @@ -109,11 +108,11 @@ class TransactionStore(SQLBaseStore): return self.runInteraction( "prep_send_transaction", self._prep_send_transaction, - transaction_id, destination, origin_server_ts, pdu_list + transaction_id, destination, origin_server_ts ) def _prep_send_transaction(self, txn, transaction_id, destination, - origin_server_ts, pdu_list): + origin_server_ts): # First we find out what the prev_txs should be. # Since we know that we are only sending one transaction at a time, From 6bc1dc40208fd1c49591dbc8cb2b9a341b96f576 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 11:44:39 +0000 Subject: [PATCH 073/265] Make recents[controller/filter/html] use modelService. This breaks functionality whilst both events.rooms and modelService are in use. --- .../matrix/event-handler-service.js | 46 +++++++++++++------ webclient/components/matrix/model-service.js | 10 ++++ webclient/recents/recents-controller.js | 7 ++- webclient/recents/recents-filter.js | 16 +++++-- webclient/recents/recents.html | 12 ++--- 5 files changed, 66 insertions(+), 25 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index dc00d83f3..9d33b6715 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -86,18 +86,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } $rootScope.events.rooms[room_id].membership = room.membership; } - - // ========================================= - var __room = modelService.getRoom(room_id); - if (room) { // /initialSync data - __room.current_room_state.storeStateEvents(room.state); - __room.current_room_state.pagination_token = room.messages.end; - - __room.old_room_state.storeStateEvents(room.state); - __room.old_room_state.pagination_token = room.messages.start; - - $rootScope["debug_"+room_id] = __room; - } }; var resetRoomMessages = function(room_id) { @@ -621,12 +609,44 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var room = rooms[i]; this.initRoom(room); + + // FIXME: This is ming: the HS should be sending down the m.room.member + // event for the invite in .state but it isn't, so fudge it for now. + if (room.inviter && room.membership === "invite") { + var me = matrixService.config().user_id; + var fakeEvent = { + event_id: "__FAKE__" + room.room_id, + user_id: room.inviter, + origin_server_ts: 0, + room_id: room.room_id, + state_key: me, + type: "m.room.member", + content: { + membership: "invite" + } + }; + if (!room.state) { + room.state = []; + } + room.state.push(fakeEvent); + console.log("RECV /initialSync invite >> "+JSON.stringify(fakeEvent)); + } + + // ========================================= + var __room = modelService.getRoom(room.room_id); + __room.current_room_state.storeStateEvents(room.state); + __room.old_room_state.storeStateEvents(room.state); + + + if ("messages" in room) { this.handleRoomMessages(room.room_id, room.messages, false); + __room.current_room_state.pagination_token = room.messages.end; + __room.old_room_state.pagination_token = room.messages.start; } - if ("state" in room) { + if ("state" in room) { // TODO FIXME remove this. this.handleEvents(room.state, false, true); } } diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index 26349f146..ff6abfe42 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -100,9 +100,15 @@ angular.module('modelService', []) storeStateEvent: function storeState(event) { this.state_events[event.type + event.state_key] = event; + if (event.type === "m.room.member") { + this.members[event.state_key] = event; + } }, storeStateEvents: function storeState(events) { + if (!events) { + return; + } for (var i=0; i - - From 92e2ff49853922a5e81ac67ccf1a95f2278d5415 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 13:48:08 +0000 Subject: [PATCH 077/265] Fix bug which prevented room name invites appearing correctly. --- .../matrix/event-handler-service.js | 4 +- webclient/components/matrix/matrix-filter.js | 125 ++++++------------ webclient/recents/recents.html | 2 +- 3 files changed, 46 insertions(+), 85 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 953ce839e..a872e3f43 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -408,7 +408,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } } console.log("Unable to handle event type " + event.type); - console.log(JSON.stringify(event, undefined, 4)); + // console.log(JSON.stringify(event, undefined, 4)); break; } } @@ -475,7 +475,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService room.state = []; } room.state.push(fakeEvent); - console.log("RECV /initialSync invite >> "+JSON.stringify(fakeEvent)); + console.log("RECV /initialSync invite >> "+room.room_id); } // ========================================= diff --git a/webclient/components/matrix/matrix-filter.js b/webclient/components/matrix/matrix-filter.js index e4b42a2ed..4d264e93f 100644 --- a/webclient/components/matrix/matrix-filter.js +++ b/webclient/components/matrix/matrix-filter.js @@ -19,6 +19,8 @@ angular.module('matrixFilter', []) // Compute the room name according to information we have +// TODO: It would be nice if this was stateless and had no dependencies. That would +// make the business logic here a lot easier to see. .filter('mRoomName', ['$rootScope', 'matrixService', 'eventHandlerService', 'modelService', function($rootScope, matrixService, eventHandlerService, modelService) { return function(room_id) { @@ -29,93 +31,60 @@ function($rootScope, matrixService, eventHandlerService, modelService) { var alias = matrixService.getRoomIdToAliasMapping(room_id); var room = modelService.getRoom(room_id).current_room_state; - if (room) { - // Get name from room state date - var room_name_event = room.state("m.room.name"); + var room_name_event = room.state("m.room.name"); - // Determine if it is a public room - var isPublicRoom = false; - if (room.state("m.room.join_rules") && room.state("m.room.join_rules").content) { - isPublicRoom = ("public" === room.state("m.room.join_rules").content.join_rule); - } + // Determine if it is a public room + var isPublicRoom = false; + if (room.state("m.room.join_rules") && room.state("m.room.join_rules").content) { + isPublicRoom = ("public" === room.state("m.room.join_rules").content.join_rule); + } - if (room_name_event) { - roomName = room_name_event.content.name; - } - else if (alias) { - roomName = alias; - } - else if (room.members.length > 0 && !isPublicRoom) { // Do not rename public room + if (room_name_event) { + roomName = room_name_event.content.name; + } + else if (alias) { + roomName = alias; + } + else if (Object.keys(room.members).length > 0 && !isPublicRoom) { // Do not rename public room + var user_id = matrixService.config().user_id; - var user_id = matrixService.config().user_id; - // Else, build the name from its users - // Limit the room renaming to 1:1 room - if (2 === Object.keys(room.members).length) { - for (var i in room.members) { - if (!room.members.hasOwnProperty(i)) continue; + // this is a "one to one" room and should have the name of the other user. + if (Object.keys(room.members).length === 2) { + for (var i in room.members) { + if (!room.members.hasOwnProperty(i)) continue; - var member = room.members[i]; - if (member.state_key !== user_id) { - roomName = eventHandlerService.getUserDisplayName(room_id, member.state_key); - break; - } + var member = room.members[i]; + if (member.state_key !== user_id) { + roomName = eventHandlerService.getUserDisplayName(room_id, member.state_key); + break; } } - else if (Object.keys(room.members).length <= 1) { - - var otherUserId; - - if (Object.keys(room.members)[0]) { - otherUserId = Object.keys(room.members)[0]; - // this could be an invite event (from event stream) - if (otherUserId === user_id && - room.members[user_id].content.membership === "invite") { - // this is us being invited to this room, so the - // *user_id* is the other user ID and not the state - // key. - otherUserId = room.members[user_id].user_id; - } + } + else if (Object.keys(room.members).length === 1) { + // this could be just us (self-chat) or could be the other person + // in a room if they have invited us to the room. Find out which. + var otherUserId = Object.keys(room.members)[0]; + if (otherUserId === user_id) { + // it's us, we may have been invited to this room or it could + // be a self chat. + if (room.members[otherUserId].content.membership === "invite") { + // someone invited us, use the right ID. + roomName = eventHandlerService.getUserDisplayName(room_id, room.members[otherUserId].user_id); } else { - // it's got to be an invite, or failing that a self-chat; - otherUserId = room.inviter || user_id; -/* - // XXX: This should all be unnecessary now thanks to using the /rooms//roomid API - - // The other member may be in the invite list, get all invited users - var invitedUserIDs = []; - - // XXX: *SURELY* we shouldn't have to trawl through the whole messages list to - // find invite - surely the other user should be in room.members with state invited? :/ --Matthew - for (var i in room.messages) { - var message = room.messages[i]; - if ("m.room.member" === message.type && "invite" === message.content.membership) { - // Filter out the current user - var member_id = message.state_key; - if (member_id === user_id) { - member_id = message.user_id; - } - if (member_id !== user_id) { - // Make sure there is no duplicate user - if (-1 === invitedUserIDs.indexOf(member_id)) { - invitedUserIDs.push(member_id); - } - } - } - } - - // For now, only 1:1 room needs to be renamed. It means only 1 invited user - if (1 === invitedUserIDs.length) { - otherUserId = invitedUserIDs[0]; - } -*/ + roomName = eventHandlerService.getUserDisplayName(room_id, otherUserId); } - - // Get the user display name + } + else { // it isn't us, so use their name if we know it. roomName = eventHandlerService.getUserDisplayName(room_id, otherUserId); } } + else if (Object.keys(room.members).length === 0) { + // this shouldn't be possible + console.error("0 members in room >> " + room_id); + } } + // Always show the alias in the room displayed name if (roomName && alias && alias !== roomName) { @@ -125,14 +94,6 @@ function($rootScope, matrixService, eventHandlerService, modelService) { if (undefined === roomName) { // By default, use the room ID roomName = room_id; - - // XXX: this is *INCREDIBLY* heavy logging for a function that calls every single - // time any kind of digest runs which refreshes a room name... - // commenting it out for now. - - // Log some information that lead to this leak - // console.log("Room ID leak for " + room_id); - // console.log("room object: " + JSON.stringify(room, undefined, 4)); } return roomName; diff --git a/webclient/recents/recents.html b/webclient/recents/recents.html index a11e6be53..7297e2370 100644 --- a/webclient/recents/recents.html +++ b/webclient/recents/recents.html @@ -27,7 +27,7 @@
+ {{ room.room_id | mRoomName }} - {{ room.numUsersInRoom || '1' }} {{ room.numUsersInRoom == 1 ? 'user' : 'users' }} + {{ room.recent.numUsersInRoom || '1' }} {{ room.recent.numUsersInRoom == 1 ? 'user' : 'users' }} @@ -27,11 +27,11 @@
-
- {{ room.inviter | mUserDisplayName: room.room_id }} invited you +
+ {{ room.recent.inviter | mUserDisplayName: room.room_id }} invited you
-
+
From fd535183eed4b2262109f90a9f29fe9e49b4b9a0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 12:18:22 +0000 Subject: [PATCH 074/265] Remove events.rooms[room_id] entirely from event-handler-service. Everything now uses modelService, but there are still one or two minor teething problems to fix. --- .../matrix/event-handler-service.js | 275 +++--------------- webclient/components/matrix/model-service.js | 11 + webclient/home/home-controller.js | 1 - webclient/recents/recents-filter.js | 4 +- 4 files changed, 59 insertions(+), 232 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 9d33b6715..00e3edf18 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -50,57 +50,28 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var reset = function() { initialSyncDeferred = $q.defer(); - - $rootScope.events = { - rooms: {} // will contain roomId: { messages:[], members:{userid1: event} } - }; - + $rootScope.presence = {}; eventMap = {}; }; reset(); - var initRoom = function(room_id, room) { - if (!(room_id in $rootScope.events.rooms)) { - console.log("Creating new rooms entry for " + room_id); - $rootScope.events.rooms[room_id] = { - room_id: room_id, - messages: [], - members: {}, - // Pagination information - pagination: { - earliest_token: "END" // how far back we've paginated - } - }; - } - - if (room) { // we got an existing room object from initialsync, seemingly. - // Report all other metadata of the room object (membership, inviter, visibility, ...) - for (var field in room) { - if (!room.hasOwnProperty(field)) continue; - - if (-1 === ["room_id", "messages", "state"].indexOf(field)) { // why indexOf - why not ===? --Matthew - $rootScope.events.rooms[room_id][field] = room[field]; - } - } - $rootScope.events.rooms[room_id].membership = room.membership; - } - }; - var resetRoomMessages = function(room_id) { - if ($rootScope.events.rooms[room_id]) { - $rootScope.events.rooms[room_id].messages = []; - } + var room = modelService.getRoom(room_id); + room.events = []; }; // Generic method to handle events data - var handleRoomDateEvent = function(event, isLiveEvent, addToRoomMessages) { + var handleRoomStateEvent = function(event, isLiveEvent, addToRoomMessages) { var __room = modelService.getRoom(event.room_id); if (addToRoomMessages) { + // some state events are displayed as messages, so add them. __room.addMessageEvent(event, !isLiveEvent); } + if (isLiveEvent) { + // update the current room state with the latest state __room.current_room_state.storeStateEvent(event); } else { @@ -113,39 +84,10 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } } } - - // ===================================== - - // Add topic changes as if they were a room message - if (addToRoomMessages) { - if (isLiveEvent) { - $rootScope.events.rooms[event.room_id].messages.push(event); - } - else { - $rootScope.events.rooms[event.room_id].messages.unshift(event); - } - } - - // live events always update, but non-live events only update if the - // ts is later. - var latestData = true; - if (!isLiveEvent) { - var eventTs = event.origin_server_ts; - var storedEvent = $rootScope.events.rooms[event.room_id][event.type]; - if (storedEvent) { - if (storedEvent.origin_server_ts > eventTs) { - // ignore it, we have a newer one already. - latestData = false; - } - } - } - if (latestData) { - $rootScope.events.rooms[event.room_id][event.type] = event; - } + // TODO: handle old_room_state }; var handleRoomCreate = function(event, isLiveEvent) { - // For now, we do not use the event data. Simply signal it to the app controllers $rootScope.$broadcast(ROOM_CREATE_EVENT, event, isLiveEvent); }; @@ -183,7 +125,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService if (shouldBing && isIdle) { console.log("Displaying notification for "+JSON.stringify(event)); - var member = getMember(event.room_id, event.user_id); + var member = modelService.getMember(event.room_id, event.user_id); var displayname = getUserDisplayName(event.room_id, event.user_id); var message = event.content.body; @@ -195,9 +137,9 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } var roomTitle = matrixService.getRoomIdToAliasMapping(event.room_id); - var theRoom = $rootScope.events.rooms[event.room_id]; - if (!roomTitle && theRoom && theRoom["m.room.name"] && theRoom["m.room.name"].content) { - roomTitle = theRoom["m.room.name"].content.name; + var theRoom = modelService.getRoom(event.room_id); + if (!roomTitle && theRoom.current_room_state.state("m.room.name") && theRoom.current_room_state.state("m.room.name").content) { + roomTitle = theRoom.current_room_state.state("m.room.name").content.name; } if (!roomTitle) { @@ -235,6 +177,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService if (event.user_id !== matrixService.config().user_id) { __room.addMessageEvent(event, !isLiveEvent); + displayNotification(event); } else { // we may have locally echoed this, so we should replace the event @@ -242,37 +185,8 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService __room.addOrReplaceMessageEvent(event, !isLiveEvent); } - // ======================= - - if (isLiveEvent) { - if (event.user_id === matrixService.config().user_id && - (event.content.msgtype === "m.text" || event.content.msgtype === "m.emote") ) { - // Assume we've already echoed it. So, there is a fake event in the messages list of the room - // Replace this fake event by the true one - var index = getRoomEventIndex(event.room_id, event.event_id); - if (index) { - $rootScope.events.rooms[event.room_id].messages[index] = event; - } - else { - $rootScope.events.rooms[event.room_id].messages.push(event); - } - } - else { - $rootScope.events.rooms[event.room_id].messages.push(event); - displayNotification(event); - } - - - } - else { - $rootScope.events.rooms[event.room_id].messages.unshift(event); - } - // TODO send delivery receipt if isLiveEvent - // $broadcast this, as controllers may want to do funky things such as - // scroll to the bottom, etc which cannot be expressed via simple $scope - // updates. $rootScope.$broadcast(MSG_EVENT, event, isLiveEvent); }; @@ -284,6 +198,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // Exception: Do not do this if the event is a room state event because such events already come // as room messages events. Moreover, when they come as room messages events, they are relatively ordered // with other other room messages + // FIXME this isn't true anymore, work out what this is doing. if (!isStateEvent) { // could be a membership change, display name change, etc. // Find out which one. @@ -301,14 +216,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // If there was a change we want to display, dump it in the message // list. if (memberChanges) { - if (isLiveEvent) { - $rootScope.events.rooms[event.room_id].messages.push(event); - } - else { - $rootScope.events.rooms[event.room_id].messages.unshift(event); - } - // ============ - __room.addMessageEvent(event, !isLiveEvent); } } @@ -316,7 +223,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // Use data from state event or the latest data from the stream. // Do not care of events that come when paginating back if (isStateEvent || isLiveEvent) { - $rootScope.events.rooms[event.room_id].members[event.state_key] = event; __room.current_room_state.members[event.state_key] = event; } @@ -329,30 +235,28 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService }; var handlePowerLevels = function(event, isLiveEvent) { - // Keep the latest data. Do not care of events that come when paginating back - if (!$rootScope.events.rooms[event.room_id][event.type] || isLiveEvent) { - $rootScope.events.rooms[event.room_id][event.type] = event; - $rootScope.$broadcast(POWERLEVEL_EVENT, event, isLiveEvent); - } + handleRoomStateEvent(event, isLiveEvent); + $rootScope.$broadcast(POWERLEVEL_EVENT, event, isLiveEvent); }; var handleRoomName = function(event, isLiveEvent, isStateEvent) { console.log("handleRoomName room_id: " + event.room_id + " - isLiveEvent: " + isLiveEvent + " - name: " + event.content.name); - handleRoomDateEvent(event, isLiveEvent, !isStateEvent); + handleRoomStateEvent(event, isLiveEvent, !isStateEvent); $rootScope.$broadcast(NAME_EVENT, event, isLiveEvent); }; var handleRoomTopic = function(event, isLiveEvent, isStateEvent) { console.log("handleRoomTopic room_id: " + event.room_id + " - isLiveEvent: " + isLiveEvent + " - topic: " + event.content.topic); - handleRoomDateEvent(event, isLiveEvent, !isStateEvent); + handleRoomStateEvent(event, isLiveEvent, !isStateEvent); $rootScope.$broadcast(TOPIC_EVENT, event, isLiveEvent); }; var handleCallEvent = function(event, isLiveEvent) { $rootScope.$broadcast(CALL_EVENT, event, isLiveEvent); if (event.type === 'm.call.invite') { - $rootScope.events.rooms[event.room_id].messages.push(event); + var __room = modelService.getRoom(event.room_id); + __room.addMessageEvent(event, !isLiveEvent); } }; @@ -366,8 +270,9 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // we need to remove something possibly: do we know the redacted // event ID? if (eventMap[event.redacts]) { + var __room = modelService.getRoom(event.room_id); // remove event from list of messages in this room. - var eventList = $rootScope.events.rooms[event.room_id].messages; + var eventList = __room.events; for (var i=0; i 0; i--) { - var message = room.messages[i]; - if (event_id === message.event_id) { - index = i; - break; - } - } - } - return index; - }; - - /** - * Get the member object of a room member - * @param {String} room_id the room id - * @param {String} user_id the id of the user - * @returns {undefined | Object} the member object of this user in this room if he is part of the room - */ - var getMember = function(room_id, user_id) { - var member; - - var room = $rootScope.events.rooms[room_id]; - if (room) { - member = room.members[user_id]; - } - return member; - }; /** * Return the display name of an user acccording to data already downloaded @@ -431,17 +295,17 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var displayName; // Get the user display name from the member list of the room - var member = getMember(room_id, user_id); + var member = modelService.getMember(room_id, user_id); if (member && member.content.displayname) { // Do not consider null displayname displayName = member.content.displayname; // Disambiguate users who have the same displayname in the room if (user_id !== matrixService.config().user_id) { - var room = $rootScope.events.rooms[room_id]; + var room = modelService.getRoom(room_id); - for (var member_id in room.members) { - if (room.members.hasOwnProperty(member_id) && member_id !== user_id) { - var member2 = room.members[member_id]; + for (var member_id in room.current_room_state.members) { + if (room.current_room_state.members.hasOwnProperty(member_id) && member_id !== user_id) { + var member2 = room.current_room_state.members[member_id]; if (member2.content.displayname && member2.content.displayname === displayName) { displayName = displayName + " (" + user_id + ")"; break; @@ -479,19 +343,9 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService reset(); $rootScope.$broadcast(RESET_EVENT); }, - - initRoom: function(room) { - initRoom(room.room_id, room); - }, handleEvent: function(event, isLiveEvent, isStateEvent) { - // FIXME: /initialSync on a particular room is not yet available - // So initRoom on a new room is not called. Make sure the room data is initialised here - if (event.room_id) { - initRoom(event.room_id); - } - // Avoid duplicated events // Needed for rooms where initialSync has not been done. // In this case, we do not know where to start pagination. So, it starts from the END @@ -550,7 +404,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // displays on the Room Info screen. if (typeof(event.state_key) === "string") { // incls. 0-len strings if (event.room_id) { - handleRoomDateEvent(event, isLiveEvent, false); + handleRoomStateEvent(event, isLiveEvent, false); } } console.log("Unable to handle event type " + event.type); @@ -570,8 +424,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // Handle messages from /initialSync or /messages handleRoomMessages: function(room_id, messages, isLiveEvents, dir) { - initRoom(room_id); - var events = messages.chunk; // Handles messages according to their time order @@ -582,8 +434,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } // Store how far back we've paginated - $rootScope.events.rooms[room_id].pagination.earliest_token = messages.end; - var __room = modelService.getRoom(room_id); __room.old_room_state.pagination_token = messages.end; @@ -594,8 +444,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService this.handleEvent(events[i], isLiveEvents, isLiveEvents); } // Store where to start pagination - $rootScope.events.rooms[room_id].pagination.earliest_token = messages.start; - var __room = modelService.getRoom(room_id); __room.old_room_state.pagination_token = messages.start; } @@ -607,8 +455,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var rooms = response.data.rooms; for (var i = 0; i < rooms.length; ++i) { var room = rooms[i]; - - this.initRoom(room); // FIXME: This is ming: the HS should be sending down the m.room.member // event for the invite in .state but it isn't, so fudge it for now. @@ -636,9 +482,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var __room = modelService.getRoom(room.room_id); __room.current_room_state.storeStateEvents(room.state); __room.old_room_state.storeStateEvents(room.state); - - - if ("messages" in room) { this.handleRoomMessages(room.room_id, room.messages, false); @@ -674,15 +517,13 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService getLastMessage: function(room_id, filterEcho) { var lastMessage; - var room = $rootScope.events.rooms[room_id]; - if (room) { - for (var i = room.messages.length - 1; i >= 0; i--) { - var message = room.messages[i]; + var events = modelService.getRoom(room_id).events; + for (var i = events.length - 1; i >= 0; i--) { + var message = events[i]; - if (!filterEcho || undefined === message.echo_msg_state) { - lastMessage = message; - break; - } + if (!filterEcho || undefined === message.echo_msg_state) { + lastMessage = message; + break; } } @@ -697,34 +538,22 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService getUsersCountInRoom: function(room_id) { var memberCount; - var room = $rootScope.events.rooms[room_id]; - if (room) { - memberCount = 0; + var room = modelService.getRoom(room_id); + memberCount = 0; - for (var i in room.members) { - if (!room.members.hasOwnProperty(i)) continue; + for (var i in room.current_room_state.members) { + if (!room.current_room_state.members.hasOwnProperty(i)) continue; - var member = room.members[i]; + var member = room.current_room_state.members[i]; - if ("join" === member.membership) { - memberCount = memberCount + 1; - } + if ("join" === member.membership) { + memberCount = memberCount + 1; } } return memberCount; }, - /** - * Get the member object of a room member - * @param {String} room_id the room id - * @param {String} user_id the id of the user - * @returns {undefined | Object} the member object of this user in this room if he is part of the room - */ - getMember: function(room_id, user_id) { - return getMember(room_id, user_id); - }, - /** * Return the power level of an user in a particular room * @param {String} room_id the room id @@ -733,14 +562,14 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService */ getUserPowerLevel: function(room_id, user_id) { var powerLevel = 0; - var room = $rootScope.events.rooms[room_id]; - if (room && room["m.room.power_levels"]) { - if (user_id in room["m.room.power_levels"].content) { - powerLevel = room["m.room.power_levels"].content[user_id]; + var room = modelService.getRoom(room_id).current_room_state; + if (room.state("m.room.power_levels")) { + if (user_id in room.state("m.room.power_levels").content) { + powerLevel = room.state("m.room.power_levels").content[user_id]; } else { // Use the room default user power - powerLevel = room["m.room.power_levels"].content["default"]; + powerLevel = room.state("m.room.power_levels").content["default"]; } } return powerLevel; @@ -754,18 +583,6 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService */ getUserDisplayName: function(room_id, user_id) { return getUserDisplayName(room_id, user_id); - }, - - setRoomVisibility: function(room_id, visible) { - if (!visible) { - return; - } - initRoom(room_id); - - var room = $rootScope.events.rooms[room_id]; - if (room) { - room.visibility = visible; - } } }; }]); diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index ff6abfe42..d5acd7eda 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -148,6 +148,17 @@ angular.module('modelService', []) getRooms: function() { return rooms; + }, + + /** + * Get the member object of a room member + * @param {String} room_id the room id + * @param {String} user_id the id of the user + * @returns {undefined | Object} the member object of this user in this room if he is part of the room + */ + getMember: function(room_id, user_id) { + var room = this.getRoom(room_id); + return room.current_room_state.members[user_id]; } }; diff --git a/webclient/home/home-controller.js b/webclient/home/home-controller.js index f1295560e..3a48e64ab 100644 --- a/webclient/home/home-controller.js +++ b/webclient/home/home-controller.js @@ -58,7 +58,6 @@ angular.module('HomeController', ['matrixService', 'eventHandlerService', 'Recen // Add room_alias & room_display_name members angular.extend(room, matrixService.getRoomAliasAndDisplayName(room)); - eventHandlerService.setRoomVisibility(room.room_id, "public"); } } ); diff --git a/webclient/recents/recents-filter.js b/webclient/recents/recents-filter.js index cf0f53f37..acd9f86ec 100644 --- a/webclient/recents/recents-filter.js +++ b/webclient/recents/recents-filter.js @@ -17,7 +17,7 @@ 'use strict'; angular.module('RecentsController') -.filter('orderRecents', ["matrixService", "eventHandlerService", function(matrixService, eventHandlerService) { +.filter('orderRecents', ["matrixService", "eventHandlerService", "modelService", function(matrixService, eventHandlerService, modelService) { return function(rooms) { var user_id = matrixService.config().user_id; @@ -29,7 +29,7 @@ angular.module('RecentsController') var meEvent = room.current_room_state.state("m.room.member", user_id); // Show the room only if the user has joined it or has been invited // (ie, do not show it if he has been banned) - var member = eventHandlerService.getMember(room_id, user_id); + var member = modelService.getMember(room_id, user_id); room.recent.me = member; if (member && ("invite" === member.membership || "join" === member.membership)) { if ("invite" === member.membership) { From ad6eacb3e9424902da9f83c8f106a4f0169c3108 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 13:06:58 +0000 Subject: [PATCH 075/265] Rename PDU fields to match that of events. --- synapse/api/events/utils.py | 2 +- synapse/federation/pdu_codec.py | 48 +------- synapse/federation/replication.py | 72 +++++------- synapse/federation/transport.py | 184 +++++------------------------- synapse/federation/units.py | 78 ++----------- synapse/handlers/federation.py | 12 +- 6 files changed, 80 insertions(+), 316 deletions(-) diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index 7fdf45a26..31601fd3a 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -32,7 +32,7 @@ def prune_event(event): def prune_pdu(pdu): """Removes keys that contain unrestricted and non-essential data from a PDU """ - return _prune_event_or_pdu(pdu.pdu_type, pdu) + return _prune_event_or_pdu(pdu.type, pdu) def _prune_event_or_pdu(event_type, event): # Remove all extraneous fields. diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index d4c896e16..5ec97a698 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -31,39 +31,16 @@ class PduCodec(object): self.clock = hs.get_clock() self.hs = hs - def encode_event_id(self, local, domain): - return local - - def decode_event_id(self, event_id): - e_id = self.hs.parse_eventid(event_id) - return event_id, e_id.domain - def event_from_pdu(self, pdu): kwargs = {} - kwargs["event_id"] = self.encode_event_id(pdu.pdu_id, pdu.origin) - kwargs["room_id"] = pdu.context - kwargs["etype"] = pdu.pdu_type - kwargs["prev_events"] = [ - (self.encode_event_id(i, o), s) - for i, o, s in pdu.prev_pdus - ] - - if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): - kwargs["prev_state"] = self.encode_event_id( - pdu.prev_state_id, pdu.prev_state_origin - ) + kwargs["etype"] = pdu.type kwargs.update({ k: v for k, v in pdu.get_full_dict().items() if k not in [ - "pdu_id", - "context", - "pdu_type", - "prev_pdus", - "prev_state_id", - "prev_state_origin", + "type", ] }) @@ -72,33 +49,12 @@ class PduCodec(object): def pdu_from_event(self, event): d = event.get_full_dict() - d["pdu_id"], d["origin"] = self.decode_event_id( - event.event_id - ) - d["context"] = event.room_id - d["pdu_type"] = event.type - - if hasattr(event, "prev_events"): - def f(e, s): - i, o = self.decode_event_id(e) - return i, o, s - d["prev_pdus"] = [ - f(e, s) - for e, s in event.prev_events - ] - - if hasattr(event, "prev_state"): - d["prev_state_id"], d["prev_state_origin"] = ( - self.decode_event_id(event.prev_state) - ) - if hasattr(event, "state_key"): d["is_state"] = True kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() - if k not in ["event_id", "room_id", "type", "prev_events"] }) if "origin_server_ts" not in kwargs: diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 159af4eed..838e660a4 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -111,14 +111,6 @@ class ReplicationLayer(object): """Informs the replication layer about a new PDU generated within the home server that should be transmitted to others. - This will fill out various attributes on the PDU object, e.g. the - `prev_pdus` key. - - *Note:* The home server should always call `send_pdu` even if it knows - that it does not need to be replicated to other home servers. This is - in case e.g. someone else joins via a remote home server and then - backfills. - TODO: Figure out when we should actually resolve the deferred. Args: @@ -131,18 +123,12 @@ class ReplicationLayer(object): order = self._order self._order += 1 - logger.debug("[%s] Persisting PDU", pdu.pdu_id) - - # Save *before* trying to send - # yield self.store.persist_event(pdu=pdu) - - logger.debug("[%s] Persisted PDU", pdu.pdu_id) - logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.pdu_id) + logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) # TODO, add errback, etc. self._transaction_queue.enqueue_pdu(pdu, order) - logger.debug("[%s] transaction_layer.enqueue_pdu... done", pdu.pdu_id) + logger.debug("[%s] transaction_layer.enqueue_pdu... done", pdu.event_id) @log_function def send_edu(self, destination, edu_type, content): @@ -215,7 +201,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def get_pdu(self, destination, pdu_origin, pdu_id, outlier=False): + def get_pdu(self, destination, event_id, outlier=False): """Requests the PDU with given origin and ID from the remote home server. @@ -224,7 +210,7 @@ class ReplicationLayer(object): Args: destination (str): Which home server to query pdu_origin (str): The home server that originally sent the pdu. - pdu_id (str) + event_id (str) outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if it's from an arbitary point in the context as opposed to part of the current block of PDUs. Defaults to `False` @@ -233,8 +219,9 @@ class ReplicationLayer(object): Deferred: Results in the requested PDU. """ - transaction_data = yield self.transport_layer.get_pdu( - destination, pdu_origin, pdu_id) + transaction_data = yield self.transport_layer.get_event( + destination, event_id + ) transaction = Transaction(**transaction_data) @@ -249,8 +236,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def get_state_for_context(self, destination, context, pdu_id=None, - pdu_origin=None): + def get_state_for_context(self, destination, context, event_id=None): """Requests all of the `current` state PDUs for a given context from a remote home server. @@ -263,7 +249,9 @@ class ReplicationLayer(object): """ transaction_data = yield self.transport_layer.get_context_state( - destination, context, pdu_id=pdu_id, pdu_origin=pdu_origin, + destination, + context, + event_id=event_id, ) transaction = Transaction(**transaction_data) @@ -352,10 +340,10 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_context_state_request(self, context, pdu_id, pdu_origin): - if pdu_id and pdu_origin: + def on_context_state_request(self, context, event_id): + if event_id: pdus = yield self.handler.get_state_for_pdu( - pdu_id, pdu_origin + event_id ) else: raise NotImplementedError("Specify an event") @@ -370,8 +358,8 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_pdu_request(self, pdu_origin, pdu_id): - pdu = yield self._get_persisted_pdu(pdu_id, pdu_origin) + def on_pdu_request(self, event_id): + pdu = yield self._get_persisted_pdu(event_id) if pdu: defer.returnValue( @@ -443,9 +431,8 @@ class ReplicationLayer(object): def send_join(self, destination, pdu): _, content = yield self.transport_layer.send_join( destination, - pdu.context, - pdu.pdu_id, - pdu.origin, + pdu.room_id, + pdu.event_id, pdu.get_dict(), ) @@ -457,13 +444,13 @@ class ReplicationLayer(object): defer.returnValue(pdus) @log_function - def _get_persisted_pdu(self, pdu_id, pdu_origin): + def _get_persisted_pdu(self, event_id): """ Get a PDU from the database with given origin and id. Returns: Deferred: Results in a `Pdu`. """ - return self.handler.get_persisted_pdu(pdu_id, pdu_origin) + return self.handler.get_persisted_pdu(event_id) def _transaction_from_pdus(self, pdu_list): """Returns a new Transaction containing the given PDUs suitable for @@ -487,10 +474,10 @@ class ReplicationLayer(object): @log_function def _handle_new_pdu(self, origin, pdu, backfilled=False): # We reprocess pdus when we have seen them only as outliers - existing = yield self._get_persisted_pdu(pdu.pdu_id, pdu.origin) + existing = yield self._get_persisted_pdu(pdu.event_id) if existing and (not existing.outlier or pdu.outlier): - logger.debug("Already seen pdu %s %s", pdu.pdu_id, pdu.origin) + logger.debug("Already seen pdu %s", pdu.event_id) defer.returnValue({}) return @@ -500,23 +487,22 @@ class ReplicationLayer(object): if not pdu.outlier: # We only backfill backwards to the min depth. min_depth = yield self.handler.get_min_depth_for_context( - pdu.context + pdu.room_id ) if min_depth and pdu.depth > min_depth: - for pdu_id, origin, hashes in pdu.prev_pdus: - exists = yield self._get_persisted_pdu(pdu_id, origin) + for event_id, hashes in pdu.prev_events: + exists = yield self._get_persisted_pdu(event_id) if not exists: - logger.debug("Requesting pdu %s %s", pdu_id, origin) + logger.debug("Requesting pdu %s", event_id) try: yield self.get_pdu( pdu.origin, - pdu_id=pdu_id, - pdu_origin=origin + event_id=event_id, ) - logger.debug("Processed pdu %s %s", pdu_id, origin) + logger.debug("Processed pdu %s", event_id) except: # TODO(erikj): Do some more intelligent retries. logger.exception("Failed to get PDU") @@ -524,7 +510,7 @@ class ReplicationLayer(object): # We need to get the state at this event, since we have reached # a backward extremity edge. state = yield self.get_state_for_context( - origin, pdu.context, pdu.pdu_id, pdu.origin, + origin, pdu.room_id, pdu.event_id, ) # Persist the Pdu, but don't mark it as processed yet. diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 7f01b4faa..04ad7e63a 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -72,8 +72,7 @@ class TransportLayer(object): self.received_handler = None @log_function - def get_context_state(self, destination, context, pdu_id=None, - pdu_origin=None): + def get_context_state(self, destination, context, event_id=None): """ Requests all state for a given context (i.e. room) from the given server. @@ -91,60 +90,59 @@ class TransportLayer(object): subpath = "/state/%s/" % context args = {} - if pdu_id and pdu_origin: - args["pdu_id"] = pdu_id - args["pdu_origin"] = pdu_origin + if event_id: + args["event_id"] = event_id return self._do_request_for_transaction( destination, subpath, args=args ) @log_function - def get_pdu(self, destination, pdu_origin, pdu_id): + def get_event(self, destination, event_id): """ Requests the pdu with give id and origin from the given server. Args: destination (str): The host name of the remote home server we want to get the state from. - pdu_origin (str): The home server which created the PDU. - pdu_id (str): The id of the PDU being requested. + event_id (str): The id of the event being requested. Returns: Deferred: Results in a dict received from the remote homeserver. """ - logger.debug("get_pdu dest=%s, pdu_origin=%s, pdu_id=%s", - destination, pdu_origin, pdu_id) + logger.debug("get_pdu dest=%s, event_id=%s", + destination, event_id) - subpath = "/pdu/%s/%s/" % (pdu_origin, pdu_id) + subpath = "/event/%s/" % (event_id, ) return self._do_request_for_transaction(destination, subpath) @log_function - def backfill(self, dest, context, pdu_tuples, limit): + def backfill(self, dest, context, event_tuples, limit): """ Requests `limit` previous PDUs in a given context before list of PDUs. Args: dest (str) context (str) - pdu_tuples (list) + event_tuples (list) limt (int) Returns: Deferred: Results in a dict received from the remote homeserver. """ logger.debug( - "backfill dest=%s, context=%s, pdu_tuples=%s, limit=%s", - dest, context, repr(pdu_tuples), str(limit) + "backfill dest=%s, context=%s, event_tuples=%s, limit=%s", + dest, context, repr(event_tuples), str(limit) ) - if not pdu_tuples: + if not event_tuples: + # TODO: raise? return - subpath = "/backfill/%s/" % context + subpath = "/backfill/%s/" % (context,) args = { - "v": ["%s,%s" % (i, o) for i, o in pdu_tuples], + "v": event_tuples, "limit": limit, } @@ -222,11 +220,10 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def send_join(self, destination, context, pdu_id, origin, content): - path = PREFIX + "/send_join/%s/%s/%s" % ( + def send_join(self, destination, context, event_id, content): + path = PREFIX + "/send_join/%s/%s" % ( context, - origin, - pdu_id, + event_id, ) code, content = yield self.client.put_json( @@ -242,11 +239,10 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def send_invite(self, destination, context, pdu_id, origin, content): - path = PREFIX + "/invite/%s/%s/%s" % ( + def send_invite(self, destination, context, event_id, content): + path = PREFIX + "/invite/%s/%s" % ( context, - origin, - pdu_id, + event_id, ) code, content = yield self.client.put_json( @@ -376,10 +372,10 @@ class TransportLayer(object): # data_id pair. self.server.register_path( "GET", - re.compile("^" + PREFIX + "/pdu/([^/]*)/([^/]*)/$"), + re.compile("^" + PREFIX + "/event/([^/]*)/$"), self._with_authentication( - lambda origin, content, query, pdu_origin, pdu_id: - handler.on_pdu_request(pdu_origin, pdu_id) + lambda origin, content, query, event_id: + handler.on_pdu_request(event_id) ) ) @@ -391,8 +387,7 @@ class TransportLayer(object): lambda origin, content, query, context: handler.on_context_state_request( context, - query.get("pdu_id", [None])[0], - query.get("pdu_origin", [None])[0] + query.get("event_id", [None])[0], ) ) ) @@ -442,9 +437,9 @@ class TransportLayer(object): self.server.register_path( "PUT", - re.compile("^" + PREFIX + "/send_join/([^/]*)/([^/]*)/([^/]*)$"), + re.compile("^" + PREFIX + "/send_join/([^/]*)/([^/]*)$"), self._with_authentication( - lambda origin, content, query, context, pdu_origin, pdu_id: + lambda origin, content, query, context, event_id: self._on_send_join_request( origin, content, query, ) @@ -453,9 +448,9 @@ class TransportLayer(object): self.server.register_path( "PUT", - re.compile("^" + PREFIX + "/invite/([^/]*)/([^/]*)/([^/]*)$"), + re.compile("^" + PREFIX + "/invite/([^/]*)/([^/]*)$"), self._with_authentication( - lambda origin, content, query, context, pdu_origin, pdu_id: + lambda origin, content, query, context, event_id: self._on_invite_request( origin, content, query, ) @@ -548,7 +543,7 @@ class TransportLayer(object): limit = int(limits[-1]) - versions = [v.split(",", 1) for v in v_list] + versions = v_list return self.request_handler.on_backfill_request( context, versions, limit @@ -579,120 +574,3 @@ class TransportLayer(object): ) defer.returnValue((200, content)) - - -class TransportReceivedHandler(object): - """ Callbacks used when we receive a transaction - """ - def on_incoming_transaction(self, transaction): - """ Called on PUT /send/, or on response to a request - that we sent (e.g. a backfill request) - - Args: - transaction (synapse.transaction.Transaction): The transaction that - was sent to us. - - Returns: - twisted.internet.defer.Deferred: A deferred that gets fired when - the transaction has finished being processed. - - The result should be a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - -class TransportRequestHandler(object): - """ Handlers used when someone want's data from us - """ - def on_pull_request(self, versions): - """ Called on GET /pull/?v=... - - This is hit when a remote home server wants to get all data - after a given transaction. Mainly used when a home server comes back - online and wants to get everything it has missed. - - Args: - versions (list): A list of transaction_ids that should be used to - determine what PDUs the remote side have not yet seen. - - Returns: - Deferred: Resultsin a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - def on_pdu_request(self, pdu_origin, pdu_id): - """ Called on GET /pdu/// - - Someone wants a particular PDU. This PDU may or may not have originated - from us. - - Args: - pdu_origin (str) - pdu_id (str) - - Returns: - Deferred: Resultsin a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - def on_context_state_request(self, context): - """ Called on GET /state// - - Gets hit when someone wants all the *current* state for a given - contexts. - - Args: - context (str): The name of the context that we're interested in. - - Returns: - twisted.internet.defer.Deferred: A deferred that gets fired when - the transaction has finished being processed. - - The result should be a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - def on_backfill_request(self, context, versions, limit): - """ Called on GET /backfill//?v=...&limit=... - - Gets hit when we want to backfill backwards on a given context from - the given point. - - Args: - context (str): The context to backfill - versions (list): A list of 2-tuples representing where to backfill - from, in the form `(pdu_id, origin)` - limit (int): How many pdus to return. - - Returns: - Deferred: Results in a tuple in the form of - `(response_code, respond_body)`, where `response_body` is a python - dict that will get serialized to JSON. - - On errors, the dict should have an `error` key with a brief message - of what went wrong. - """ - pass - - def on_query_request(self): - """ Called on a GET /query/ request. """ diff --git a/synapse/federation/units.py b/synapse/federation/units.py index adc338564..c94dcf64c 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -34,13 +34,13 @@ class Pdu(JsonEncodedObject): A Pdu can be classified as "state". For a given context, we can efficiently retrieve all state pdu's that haven't been clobbered. Clobbering is done - via a unique constraint on the tuple (context, pdu_type, state_key). A pdu + via a unique constraint on the tuple (context, type, state_key). A pdu is a state pdu if `is_state` is True. Example pdu:: { - "pdu_id": "78c", + "event_id": "$78c:example.com", "origin_server_ts": 1404835423000, "origin": "bar", "prev_ids": [ @@ -53,14 +53,14 @@ class Pdu(JsonEncodedObject): """ valid_keys = [ - "pdu_id", - "context", + "event_id", + "room_id", "origin", "origin_server_ts", - "pdu_type", + "type", "destinations", "transaction_id", - "prev_pdus", + "prev_events", "depth", "content", "outlier", @@ -68,8 +68,7 @@ class Pdu(JsonEncodedObject): "signatures", "is_state", # Below this are keys valid only for State Pdus. "state_key", - "prev_state_id", - "prev_state_origin", + "prev_state", "required_power_level", "user_id", ] @@ -81,18 +80,18 @@ class Pdu(JsonEncodedObject): ] required_keys = [ - "pdu_id", - "context", + "event_id", + "room_id", "origin", "origin_server_ts", - "pdu_type", + "type", "content", ] # TODO: We need to make this properly load content rather than # just leaving it as a dict. (OR DO WE?!) - def __init__(self, destinations=[], is_state=False, prev_pdus=[], + def __init__(self, destinations=[], is_state=False, prev_events=[], outlier=False, hashes={}, signatures={}, **kwargs): if is_state: for required_key in ["state_key"]: @@ -102,66 +101,13 @@ class Pdu(JsonEncodedObject): super(Pdu, self).__init__( destinations=destinations, is_state=bool(is_state), - prev_pdus=prev_pdus, + prev_events=prev_events, outlier=outlier, hashes=hashes, signatures=signatures, **kwargs ) - @classmethod - def from_pdu_tuple(cls, pdu_tuple): - """ Converts a PduTuple to a Pdu - - Args: - pdu_tuple (synapse.persistence.transactions.PduTuple): The tuple to - convert - - Returns: - Pdu - """ - if pdu_tuple: - d = copy.copy(pdu_tuple.pdu_entry._asdict()) - d["origin_server_ts"] = d.pop("ts") - - for k in d.keys(): - if d[k] is None: - del d[k] - - d["content"] = json.loads(d["content_json"]) - del d["content_json"] - - args = {f: d[f] for f in cls.valid_keys if f in d} - if "unrecognized_keys" in d and d["unrecognized_keys"]: - args.update(json.loads(d["unrecognized_keys"])) - - hashes = { - alg: encode_base64(hsh) - for alg, hsh in pdu_tuple.hashes.items() - } - - signatures = { - kid: encode_base64(sig) - for kid, sig in pdu_tuple.signatures.items() - } - - prev_pdus = [] - for prev_pdu in pdu_tuple.prev_pdu_list: - prev_hashes = pdu_tuple.edge_hashes.get(prev_pdu, {}) - prev_hashes = { - alg: encode_base64(hsh) for alg, hsh in prev_hashes.items() - } - prev_pdus.append((prev_pdu[0], prev_pdu[1], prev_hashes)) - - return Pdu( - prev_pdus=prev_pdus, - hashes=hashes, - signatures=signatures, - **args - ) - else: - return None - def __str__(self): return "(%s, %s)" % (self.__class__.__name__, repr(self.__dict__)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 18cb1d4e9..bdd28f04b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -139,7 +139,7 @@ class FederationHandler(BaseHandler): # Huh, let's try and get the current state try: yield self.replication_layer.get_state_for_context( - event.origin, event.room_id, pdu.pdu_id, pdu.origin, + event.origin, event.room_id, event.event_id, ) hosts = yield self.store.get_joined_hosts_for_room( @@ -368,11 +368,9 @@ class FederationHandler(BaseHandler): ]) @defer.inlineCallbacks - def get_state_for_pdu(self, pdu_id, pdu_origin): + def get_state_for_pdu(self, event_id): yield run_on_reactor() - event_id = EventID.create(pdu_id, pdu_origin, self.hs).to_string() - state_groups = yield self.store.get_state_groups( [event_id] ) @@ -406,7 +404,7 @@ class FederationHandler(BaseHandler): events = yield self.store.get_backfill_events( context, - [self.pdu_codec.encode_event_id(i, o) for i, o in pdu_list], + pdu_list, limit ) @@ -417,14 +415,14 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def get_persisted_pdu(self, pdu_id, origin): + def get_persisted_pdu(self, event_id): """ Get a PDU from the database with given origin and id. Returns: Deferred: Results in a `Pdu`. """ event = yield self.store.get_event( - self.pdu_codec.encode_event_id(pdu_id, origin), + event_id, allow_none=True, ) From 9b1ca64a75160b44106d5a7b4ddc96a009f56012 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 13:26:50 +0000 Subject: [PATCH 076/265] Fix bug which prevented the number of users being visible on the recents view. --- webclient/components/matrix/event-handler-service.js | 3 +-- webclient/recents/recents-filter.js | 4 ++-- webclient/recents/recents.html | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 00e3edf18..953ce839e 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -540,13 +540,12 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var room = modelService.getRoom(room_id); memberCount = 0; - for (var i in room.current_room_state.members) { if (!room.current_room_state.members.hasOwnProperty(i)) continue; var member = room.current_room_state.members[i]; - if ("join" === member.membership) { + if ("join" === member.content.membership) { memberCount = memberCount + 1; } } diff --git a/webclient/recents/recents-filter.js b/webclient/recents/recents-filter.js index acd9f86ec..db7fc5e4e 100644 --- a/webclient/recents/recents-filter.js +++ b/webclient/recents/recents-filter.js @@ -31,8 +31,8 @@ angular.module('RecentsController') // (ie, do not show it if he has been banned) var member = modelService.getMember(room_id, user_id); room.recent.me = member; - if (member && ("invite" === member.membership || "join" === member.membership)) { - if ("invite" === member.membership) { + if (member && ("invite" === member.content.membership || "join" === member.content.membership)) { + if ("invite" === member.content.membership) { room.recent.inviter = member.user_id; } // Count users here diff --git a/webclient/recents/recents.html b/webclient/recents/recents.html index 2a633ebe2..a11e6be53 100644 --- a/webclient/recents/recents.html +++ b/webclient/recents/recents.html @@ -9,7 +9,7 @@ {{ room.room_id | mRoomName }}
- + {{ room.recent.numUsersInRoom || '1' }} {{ room.recent.numUsersInRoom == 1 ? 'user' : 'users' }}
-
+
{{ room.recent.inviter | mUserDisplayName: room.room_id }} invited you
From f4e50079dea15f25a6d1ce03c5bf9f0608ecd45d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 14:22:09 +0000 Subject: [PATCH 078/265] Fix bug which prevented pagination from bumping the list down, causing infini-pagination. --- webclient/room/room.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index 1cec8ac70..8b5be66cf 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -124,7 +124,7 @@ ng-style="{ 'visibility': state.messages_visibility }" keep-scroll> -
{{ msg.user_id | mUserDisplayName: room_id }}
From 7614d8f87a445aae48ea7869f72070796218bcee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 15:02:16 +0000 Subject: [PATCH 079/265] Fix hidden event keys being incorrectly shown in the even info dialog. --- webclient/room/room-controller.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 38158ffc7..0bcc33ab4 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -1004,7 +1004,11 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) }; $scope.openJson = function(content) { - $scope.event_selected = content; + $scope.event_selected = angular.copy(content); + + // FIXME: Pre-calculated event data should be stripped in a nicer way. + $scope.event_selected.room_member = undefined; + // scope this so the template can check power levels and enable/disable // buttons $scope.pow = eventHandlerService.getUserPowerLevel; From 1071d063ab92a882eaacf71dfe05f580c65f6b75 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 15:05:35 +0000 Subject: [PATCH 080/265] Fix broken redact enable logic. --- webclient/room/room.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index 8b5be66cf..3f4c4c57b 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -6,7 +6,7 @@
From af7ae048f8868a58dc1b82a17ca701f28624b065 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 15:06:40 +0000 Subject: [PATCH 081/265] Add option to not bind to HTTPS port. This is useful if running behind an ssl load balancer --- demo/start.sh | 2 +- synapse/app/homeserver.py | 5 ++++- synapse/config/server.py | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index fc6cd6303..0530f0a26 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -32,7 +32,7 @@ for port in 8080 8081 8082; do -D --pid-file "$DIR/$port.pid" \ --manhole $((port + 1000)) \ --tls-dh-params-path "demo/demo.tls.dh" \ - $PARAMS + $PARAMS $SYNAPSE_PARAMS python -m synapse.app.homeserver \ --config-path "demo/etc/$port.config" \ diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6394bc27d..a20376b9d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -233,7 +233,10 @@ def setup(): f.namespace['hs'] = hs reactor.listenTCP(config.manhole, f, interface='127.0.0.1') - hs.start_listening(config.bind_port, config.unsecure_port) + bind_port = config.bind_port + if config.no_tls: + bind_port = None + hs.start_listening(bind_port, config.unsecure_port) if config.daemonize: print config.pid_file diff --git a/synapse/config/server.py b/synapse/config/server.py index 3afda12d5..814a4c349 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -30,6 +30,7 @@ class ServerConfig(Config): self.pid_file = self.abspath(args.pid_file) self.webclient = True self.manhole = args.manhole + self.no_tls = args.no_tls if not args.content_addr: host = args.server_name @@ -67,6 +68,8 @@ class ServerConfig(Config): server_group.add_argument("--content-addr", default=None, help="The host and scheme to use for the " "content repository") + server_group.add_argument("--no-tls", action='store_true', + help="Don't bind to the https port.") def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") From f3dbcdc7b31f693096407f9bfad0f7c2aa9b66c5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 15:17:32 +0000 Subject: [PATCH 082/265] Variable renaming, general cleanup. Don't feed state events from /initialSync twice. --- .../matrix/event-handler-service.js | 61 +++++++++---------- webclient/recents/recents-filter.js | 6 +- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index a872e3f43..7f5342975 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -22,9 +22,8 @@ not care where the event came from, it only needs enough context to be able to process them. Events may be coming from the event stream, the REST API (via direct GETs or via a pagination stream API), etc. -Typically, this service will store events or broadcast them to any listeners -(e.g. controllers) via $broadcast. Alternatively, it may update the $rootScope -if typically all the $on method would do is update its own $scope. +Typically, this service will store events and broadcast them to any listeners +(e.g. controllers) via $broadcast. */ angular.module('eventHandlerService', []) .factory('eventHandlerService', ['matrixService', '$rootScope', '$q', '$timeout', 'mPresence', 'notificationService', 'modelService', @@ -44,6 +43,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // of the app, given we never try to reap memory yet) var eventMap = {}; + // TODO: Remove this and replace with modelService.User objects. $rootScope.presence = {}; var initialSyncDeferred; @@ -64,23 +64,23 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // Generic method to handle events data var handleRoomStateEvent = function(event, isLiveEvent, addToRoomMessages) { - var __room = modelService.getRoom(event.room_id); + var room = modelService.getRoom(event.room_id); if (addToRoomMessages) { // some state events are displayed as messages, so add them. - __room.addMessageEvent(event, !isLiveEvent); + room.addMessageEvent(event, !isLiveEvent); } if (isLiveEvent) { // update the current room state with the latest state - __room.current_room_state.storeStateEvent(event); + room.current_room_state.storeStateEvent(event); } else { var eventTs = event.origin_server_ts; - var storedEvent = __room.current_room_state.getStateEvent(event.type, event.state_key); + var storedEvent = room.current_room_state.getStateEvent(event.type, event.state_key); if (storedEvent) { if (storedEvent.origin_server_ts < eventTs) { // the incoming event is newer, use it. - __room.current_room_state.storeStateEvent(event); + room.current_room_state.storeStateEvent(event); } } } @@ -173,16 +173,16 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // ======================= - var __room = modelService.getRoom(event.room_id); + var room = modelService.getRoom(event.room_id); if (event.user_id !== matrixService.config().user_id) { - __room.addMessageEvent(event, !isLiveEvent); + room.addMessageEvent(event, !isLiveEvent); displayNotification(event); } else { // we may have locally echoed this, so we should replace the event // instead of just adding. - __room.addOrReplaceMessageEvent(event, !isLiveEvent); + room.addOrReplaceMessageEvent(event, !isLiveEvent); } // TODO send delivery receipt if isLiveEvent @@ -191,7 +191,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService }; var handleRoomMember = function(event, isLiveEvent, isStateEvent) { - var __room = modelService.getRoom(event.room_id); + var room = modelService.getRoom(event.room_id); // add membership changes as if they were a room message if something interesting changed @@ -216,14 +216,14 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // If there was a change we want to display, dump it in the message // list. if (memberChanges) { - __room.addMessageEvent(event, !isLiveEvent); + room.addMessageEvent(event, !isLiveEvent); } } // Use data from state event or the latest data from the stream. // Do not care of events that come when paginating back if (isStateEvent || isLiveEvent) { - __room.current_room_state.members[event.state_key] = event; + room.current_room_state.members[event.state_key] = event; } $rootScope.$broadcast(MEMBER_EVENT, event, isLiveEvent, isStateEvent); @@ -255,8 +255,8 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var handleCallEvent = function(event, isLiveEvent) { $rootScope.$broadcast(CALL_EVENT, event, isLiveEvent); if (event.type === 'm.call.invite') { - var __room = modelService.getRoom(event.room_id); - __room.addMessageEvent(event, !isLiveEvent); + var room = modelService.getRoom(event.room_id); + room.addMessageEvent(event, !isLiveEvent); } }; @@ -270,9 +270,9 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // we need to remove something possibly: do we know the redacted // event ID? if (eventMap[event.redacts]) { - var __room = modelService.getRoom(event.room_id); + var room = modelService.getRoom(event.room_id); // remove event from list of messages in this room. - var eventList = __room.events; + var eventList = room.events; for (var i=0; i> "+room.room_id); } - // ========================================= - var __room = modelService.getRoom(room.room_id); - __room.current_room_state.storeStateEvents(room.state); - __room.old_room_state.storeStateEvents(room.state); + var newRoom = modelService.getRoom(room.room_id); + newRoom.current_room_state.storeStateEvents(room.state); + newRoom.old_room_state.storeStateEvents(room.state); if ("messages" in room) { this.handleRoomMessages(room.room_id, room.messages, false); - __room.current_room_state.pagination_token = room.messages.end; - __room.old_room_state.pagination_token = room.messages.start; - } - - if ("state" in room) { // TODO FIXME remove this. - this.handleEvents(room.state, false, true); + newRoom.current_room_state.pagination_token = room.messages.end; + newRoom.old_room_state.pagination_token = room.messages.start; } } var presence = response.data.presence; diff --git a/webclient/recents/recents-filter.js b/webclient/recents/recents-filter.js index db7fc5e4e..39c235996 100644 --- a/webclient/recents/recents-filter.js +++ b/webclient/recents/recents-filter.js @@ -45,14 +45,10 @@ angular.module('RecentsController') // The only information we have about the room is that the user has been invited filtered.push(room); } - else { - console.error("Dropping "+room.room_id); - } - }); // And time sort them - // The room with the lastest message at first + // The room with the latest message at first filtered.sort(function (roomA, roomB) { var lastMsgRoomA = eventHandlerService.getLastMessage(roomA.room_id, true); From 2221a13a4da2a417a8b8dcc09350f884cdb10197 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 3 Nov 2014 15:58:00 +0000 Subject: [PATCH 083/265] script for checking signatures on signed json --- scripts/check_signature.py | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 scripts/check_signature.py diff --git a/scripts/check_signature.py b/scripts/check_signature.py new file mode 100644 index 000000000..6309c32cc --- /dev/null +++ b/scripts/check_signature.py @@ -0,0 +1,70 @@ + +from synapse.crypto.event_signing import verify_signed_event_pdu +from syutil.crypto.jsonsign import verify_signed_json +from syutil.crypto.signing_key import ( + decode_verify_key_bytes, write_signing_keys +) +from syutil.base64util import decode_base64 + +import urllib2 +import json +import sys +import dns.resolver +import pprint +import argparse +import logging + +def get_targets(server_name): + try: + answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") + for srv in answers: + yield (srv.target, srv.port) + except dns.resolver.NXDOMAIN: + yield (server_name, 8480) + +def get_server_keys(server_name, target, port): + url = "https://%s:%i/_matrix/key/v1" % (target, port) + keys = json.load(urllib2.urlopen(url)) + verify_keys = {} + for key_id, key_base64 in keys["verify_keys"].items(): + verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64)) + verify_signed_json(keys, server_name, verify_key) + verify_keys[key_id] = verify_key + return verify_keys + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument("signature_name") + parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), + default=sys.stdin) + + args = parser.parse_args() + logging.basicConfig() + + server_name = args.signature_name + keys = {} + for target, port in get_targets(server_name): + try: + keys = get_server_keys(server_name, target, port) + print "Using keys from https://%s:%s/_matrix/key/v1" % (target, port) + write_signing_keys(sys.stdout, keys.values()) + break + except: + logging.exception("Error talking to %s:%s", target, port) + + json_to_check = json.load(args.input_json) + print "Checking JSON:" + for key_id in json_to_check["signatures"][args.signature_name]: + try: + key = keys[key_id] + verify_signed_json(json_to_check, args.signature_name, key) + print "PASS %s" % (key_id,) + except: + logging.exception("Check for key %s failed" % (key_id,)) + print "FAIL %s" % (key_id,) + + +if __name__ == '__main__': + main() + From fe6832fae85d56b0b4cdc906a73a2c64a26de15d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 3 Nov 2014 16:08:22 +0000 Subject: [PATCH 084/265] handle server names with embeded ports --- scripts/check_signature.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/check_signature.py b/scripts/check_signature.py index 6309c32cc..e7964e7e7 100644 --- a/scripts/check_signature.py +++ b/scripts/check_signature.py @@ -15,6 +15,10 @@ import argparse import logging def get_targets(server_name): + if ":" in server_name: + target, port = server_name.split(":") + yield (target, int(port)) + return try: answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") for srv in answers: From af83bf6712629bd9121d1b28b18dda936f0c28f2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 3 Nov 2014 16:35:24 +0000 Subject: [PATCH 085/265] Script for checking event hashes --- scripts/check_event_hash.py | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 scripts/check_event_hash.py diff --git a/scripts/check_event_hash.py b/scripts/check_event_hash.py new file mode 100644 index 000000000..9fa4452ee --- /dev/null +++ b/scripts/check_event_hash.py @@ -0,0 +1,43 @@ +from synapse.crypto.event_signing import * +from syutil.base64util import encode_base64 + +import argparse +import hashlib +import sys +import json + +class dictobj(dict): + def __init__(self, *args, **kargs): + dict.__init__(self, *args, **kargs) + self.__dict__ = self + + def get_dict(self): + return dict(self) + + +def main(): + parser = parser = argparse.ArgumentParser() + parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), + default=sys.stdin) + args = parser.parse_args() + logging.basicConfig() + + event_json = dictobj(json.load(args.input_json)) + + algorithms = { + "sha256": hashlib.sha256, + } + + for alg_name in event_json.hashes: + if check_event_pdu_content_hash(event_json, algorithms[alg_name]): + print "PASS content hash %s" % (alg_name,) + else: + print "FAIL content hash %s" % (alg_name,) + + for algorithm in algorithms.values(): + name, h_bytes = compute_pdu_event_reference_hash(event_json, algorithm) + print "Reference hash %s: %s" % (name, encode_base64(bytes)) + +if __name__=="__main__": + main() + From 4b3f74388570f104ebd3cae8543f1aaca1572127 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 17:44:14 +0000 Subject: [PATCH 086/265] Reference the room_member key on messages which adjusts for current vs old_room_state. This displays names for historical users correctly, but is off by one (referencing content not prev_content). --- .../matrix/event-handler-service.js | 22 ++++++++++++++----- webclient/room/room.html | 4 ++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 7f5342975..eff8674e0 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -193,6 +193,17 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var handleRoomMember = function(event, isLiveEvent, isStateEvent) { var room = modelService.getRoom(event.room_id); + // modify state before adding the message so it points to the right thing. + // The events are copied to avoid referencing the same event when adding + // the message (circular json structures) + if (isStateEvent || isLiveEvent) { + room.current_room_state.storeStateEvent(angular.copy(event)); + } + else if (!isLiveEvent) { + // mutate the old room state + room.old_room_state.storeStateEvent(angular.copy(event)); + } + // add membership changes as if they were a room message if something interesting changed // Exception: Do not do this if the event is a room state event because such events already come @@ -220,11 +231,7 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } } - // Use data from state event or the latest data from the stream. - // Do not care of events that come when paginating back - if (isStateEvent || isLiveEvent) { - room.current_room_state.members[event.state_key] = event; - } + $rootScope.$broadcast(MEMBER_EVENT, event, isLiveEvent, isStateEvent); }; @@ -439,7 +446,8 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } else { - // InitialSync returns messages in chronological order + // InitialSync returns messages in chronological order, so invert + // it to get most recent > oldest for (var i=events.length - 1; i>=0; i--) { this.handleEvent(events[i], isLiveEvents, isLiveEvents); } @@ -482,6 +490,8 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService newRoom.current_room_state.storeStateEvents(room.state); newRoom.old_room_state.storeStateEvents(room.state); + // this should be done AFTER storing state events since these + // messages may make the old_room_state diverge. if ("messages" in room) { this.handleRoomMessages(room.room_id, room.messages, false); newRoom.current_room_state.pagination_token = room.messages.end; diff --git a/webclient/room/room.html b/webclient/room/room.html index 3f4c4c57b..cd2a5e9c2 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -127,14 +127,14 @@
-
{{ msg.user_id | mUserDisplayName: room_id }}
+
{{ msg.room_member.content.displayname || msg.user_id | mUserDisplayName: room_id }}
{{ (msg.origin_server_ts) | date:'MMM d HH:mm' }}
- From efb0f6e23b3dfc8e6f239f9dddc01b8bdd221e64 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 17:49:09 +0000 Subject: [PATCH 087/265] Backwards compat for old-style avatar_urls --- webclient/room/room.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index cd2a5e9c2..60b875f17 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -134,7 +134,8 @@ - + From 68698e0ac8c39083f6ab7d377a48b5bead3d3598 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Nov 2014 17:51:42 +0000 Subject: [PATCH 088/265] Fix bugs in generating event signatures and hashing --- scripts/check_event_hash.py | 12 ++-- scripts/check_signature.py | 1 - synapse/api/events/__init__.py | 1 + synapse/crypto/event_signing.py | 116 +++++++++++++------------------- synapse/federation/pdu_codec.py | 13 +--- synapse/federation/units.py | 11 +-- 6 files changed, 58 insertions(+), 96 deletions(-) diff --git a/scripts/check_event_hash.py b/scripts/check_event_hash.py index 9fa4452ee..7c32f8102 100644 --- a/scripts/check_event_hash.py +++ b/scripts/check_event_hash.py @@ -6,6 +6,7 @@ import hashlib import sys import json + class dictobj(dict): def __init__(self, *args, **kargs): dict.__init__(self, *args, **kargs) @@ -14,9 +15,12 @@ class dictobj(dict): def get_dict(self): return dict(self) + def get_full_dict(self): + return dict(self) + def main(): - parser = parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser() parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), default=sys.stdin) args = parser.parse_args() @@ -29,14 +33,14 @@ def main(): } for alg_name in event_json.hashes: - if check_event_pdu_content_hash(event_json, algorithms[alg_name]): + if check_event_content_hash(event_json, algorithms[alg_name]): print "PASS content hash %s" % (alg_name,) else: print "FAIL content hash %s" % (alg_name,) for algorithm in algorithms.values(): - name, h_bytes = compute_pdu_event_reference_hash(event_json, algorithm) - print "Reference hash %s: %s" % (name, encode_base64(bytes)) + name, h_bytes = compute_event_reference_hash(event_json, algorithm) + print "Reference hash %s: %s" % (name, encode_base64(h_bytes)) if __name__=="__main__": main() diff --git a/scripts/check_signature.py b/scripts/check_signature.py index e7964e7e7..e146e18e2 100644 --- a/scripts/check_signature.py +++ b/scripts/check_signature.py @@ -1,5 +1,4 @@ -from synapse.crypto.event_signing import verify_signed_event_pdu from syutil.crypto.jsonsign import verify_signed_json from syutil.crypto.signing_key import ( decode_verify_key_bytes, write_signing_keys diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index b855811b9..168b81231 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -61,6 +61,7 @@ class SynapseEvent(JsonEncodedObject): "prev_content", "prev_state", "redacted_because", + "origin_server_ts", ] internal_keys = [ diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 0e8bc7eb6..de5d2e746 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -15,11 +15,11 @@ # limitations under the License. -from synapse.federation.units import Pdu -from synapse.api.events.utils import prune_pdu, prune_event +from synapse.api.events.utils import prune_event from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 -from syutil.crypto.jsonsign import sign_json, verify_signed_json +from syutil.crypto.jsonsign import sign_json +from synapse.api.events.room import GenericEvent import copy import hashlib @@ -28,20 +28,14 @@ import logging logger = logging.getLogger(__name__) -def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): - hashed = _compute_content_hash(pdu, hash_algorithm) - pdu.hashes[hashed.name] = encode_base64(hashed.digest()) - return pdu - - -def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): +def check_event_content_hash(event, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" - computed_hash = _compute_content_hash(pdu, hash_algorithm) - if computed_hash.name not in pdu.hashes: + computed_hash = _compute_content_hash(event, hash_algorithm) + if computed_hash.name not in event.hashes: raise Exception("Algorithm %s not in hashes %s" % ( - computed_hash.name, list(pdu.hashes) + computed_hash.name, list(event.hashes) )) - message_hash_base64 = pdu.hashes[computed_hash.name] + message_hash_base64 = event.hashes[computed_hash.name] try: message_hash_bytes = decode_base64(message_hash_base64) except: @@ -49,62 +43,7 @@ def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256): return message_hash_bytes == computed_hash.digest() -def _compute_content_hash(pdu, hash_algorithm): - pdu_json = pdu.get_dict() - #TODO: Make "age_ts" key internal - pdu_json.pop("age_ts", None) - pdu_json.pop("unsigned", None) - pdu_json.pop("signatures", None) - pdu_json.pop("hashes", None) - pdu_json_bytes = encode_canonical_json(pdu_json) - return hash_algorithm(pdu_json_bytes) - - -def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256): - tmp_pdu = Pdu(**pdu.get_dict()) - tmp_pdu = prune_pdu(tmp_pdu) - pdu_json = tmp_pdu.get_dict() - pdu_json.pop("signatures", None) - pdu_json_bytes = encode_canonical_json(pdu_json) - hashed = hash_algorithm(pdu_json_bytes) - return (hashed.name, hashed.digest()) - - -def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): - tmp_event = copy.deepcopy(event) - tmp_event = prune_event(tmp_event) - event_json = tmp_event.get_dict() - event_json.pop("signatures", None) - event_json_bytes = encode_canonical_json(event_json) - hashed = hash_algorithm(event_json_bytes) - return (hashed.name, hashed.digest()) - - -def sign_event_pdu(pdu, signature_name, signing_key): - tmp_pdu = Pdu(**pdu.get_dict()) - tmp_pdu = prune_pdu(tmp_pdu) - pdu_json = tmp_pdu.get_dict() - pdu_json = sign_json(pdu_json, signature_name, signing_key) - pdu.signatures = pdu_json["signatures"] - return pdu - - -def verify_signed_event_pdu(pdu, signature_name, verify_key): - tmp_pdu = Pdu(**pdu.get_dict()) - tmp_pdu = prune_pdu(tmp_pdu) - pdu_json = tmp_pdu.get_dict() - verify_signed_json(pdu_json, signature_name, verify_key) - - -def add_hashes_and_signatures(event, signature_name, signing_key, - hash_algorithm=hashlib.sha256): - tmp_event = copy.deepcopy(event) - tmp_event = prune_event(tmp_event) - redact_json = tmp_event.get_dict() - redact_json.pop("signatures", None) - redact_json = sign_json(redact_json, signature_name, signing_key) - event.signatures = redact_json["signatures"] - +def _compute_content_hash(event, hash_algorithm): event_json = event.get_full_dict() #TODO: We need to sign the JSON that is going out via fedaration. event_json.pop("age_ts", None) @@ -112,7 +51,44 @@ def add_hashes_and_signatures(event, signature_name, signing_key, event_json.pop("signatures", None) event_json.pop("hashes", None) event_json_bytes = encode_canonical_json(event_json) + return hash_algorithm(event_json_bytes) + + +def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): + # FIXME(erikj): GenericEvent! + tmp_event = GenericEvent(**event.get_full_dict()) + tmp_event = prune_event(tmp_event) + event_json = tmp_event.get_dict() + event_json.pop("signatures", None) + event_json.pop("age_ts", None) + event_json.pop("unsigned", None) + event_json_bytes = encode_canonical_json(event_json) hashed = hash_algorithm(event_json_bytes) + return (hashed.name, hashed.digest()) + + +def compute_event_signature(event, signature_name, signing_key): + tmp_event = copy.deepcopy(event) + tmp_event = prune_event(tmp_event) + redact_json = tmp_event.get_full_dict() + redact_json.pop("signatures", None) + redact_json.pop("age_ts", None) + redact_json.pop("unsigned", None) + logger.debug("Signing event: %s", redact_json) + redact_json = sign_json(redact_json, signature_name, signing_key) + return redact_json["signatures"] + + +def add_hashes_and_signatures(event, signature_name, signing_key, + hash_algorithm=hashlib.sha256): + hashed = _compute_content_hash(event, hash_algorithm=hash_algorithm) + if not hasattr(event, "hashes"): event.hashes = {} event.hashes[hashed.name] = encode_base64(hashed.digest()) + + event.signatures = compute_event_signature( + event, + signature_name=signature_name, + signing_key=signing_key, + ) diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index 5ec97a698..52c84efb5 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -14,10 +14,6 @@ # limitations under the License. from .units import Pdu -from synapse.crypto.event_signing import ( - add_event_pdu_content_hash, sign_event_pdu -) -from synapse.types import EventID import copy @@ -49,17 +45,10 @@ class PduCodec(object): def pdu_from_event(self, event): d = event.get_full_dict() - if hasattr(event, "state_key"): - d["is_state"] = True - kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() }) - if "origin_server_ts" not in kwargs: - kwargs["origin_server_ts"] = int(self.clock.time_msec()) - pdu = Pdu(**kwargs) - pdu = add_event_pdu_content_hash(pdu) - return sign_event_pdu(pdu, self.server_name, self.signing_key) + return pdu diff --git a/synapse/federation/units.py b/synapse/federation/units.py index c94dcf64c..c2d8dca8f 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -65,8 +65,7 @@ class Pdu(JsonEncodedObject): "content", "outlier", "hashes", - "signatures", - "is_state", # Below this are keys valid only for State Pdus. + "signatures", # Below this are keys valid only for State Pdus. "state_key", "prev_state", "required_power_level", @@ -91,16 +90,10 @@ class Pdu(JsonEncodedObject): # TODO: We need to make this properly load content rather than # just leaving it as a dict. (OR DO WE?!) - def __init__(self, destinations=[], is_state=False, prev_events=[], + def __init__(self, destinations=[], prev_events=[], outlier=False, hashes={}, signatures={}, **kwargs): - if is_state: - for required_key in ["state_key"]: - if required_key not in kwargs: - raise RuntimeError("Key %s is required" % required_key) - super(Pdu, self).__init__( destinations=destinations, - is_state=bool(is_state), prev_events=prev_events, outlier=outlier, hashes=hashes, From 2438b8b66b8dc08d0fe77447b0a3cdb39e9a8f95 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 17:52:41 +0000 Subject: [PATCH 089/265] Fix off-by-one bug when displaying display names / avatar urls when paginating. --- webclient/components/matrix/event-handler-service.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index eff8674e0..1b9a99eaa 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -201,6 +201,13 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService } else if (!isLiveEvent) { // mutate the old room state + var oldEvent = angular.copy(event); + if (event.prev_content) { + // the m.room.member event we are handling is the NEW event. When + // we keep going back in time, we want the PREVIOUS value for displaying + // names/etc, hence the clobber here. + event.content = event.prev_content; + } room.old_room_state.storeStateEvent(angular.copy(event)); } From 43e7ad1b1c2ed2b23e928175d29dfcd8a2b00c76 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 3 Nov 2014 17:58:11 +0000 Subject: [PATCH 090/265] Rejig order of checks/state updates/message insertions for m.room.member. Mark known issue. --- .../matrix/event-handler-service.js | 49 +++++++++---------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 1b9a99eaa..bba24cfaa 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -193,6 +193,22 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService var handleRoomMember = function(event, isLiveEvent, isStateEvent) { var room = modelService.getRoom(event.room_id); + // did something change? + var memberChanges = undefined; + if (!isStateEvent) { + // could be a membership change, display name change, etc. + // Find out which one. + if ((event.prev_content === undefined && event.content.membership) || (event.prev_content && (event.prev_content.membership !== event.content.membership))) { + memberChanges = "membership"; + } + else if (event.prev_content && (event.prev_content.displayname !== event.content.displayname)) { + memberChanges = "displayname"; + } + // mark the key which changed + event.changedKey = memberChanges; + } + + // modify state before adding the message so it points to the right thing. // The events are copied to avoid referencing the same event when adding // the message (circular json structures) @@ -206,36 +222,17 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // the m.room.member event we are handling is the NEW event. When // we keep going back in time, we want the PREVIOUS value for displaying // names/etc, hence the clobber here. + // FIXME TODO: We can't be doing this, we should have separate keys for + // avatar_url and displayname. event.content = event.prev_content; } - room.old_room_state.storeStateEvent(angular.copy(event)); + room.old_room_state.storeStateEvent(oldEvent); } - - // add membership changes as if they were a room message if something interesting changed - // Exception: Do not do this if the event is a room state event because such events already come - // as room messages events. Moreover, when they come as room messages events, they are relatively ordered - // with other other room messages - // FIXME this isn't true anymore, work out what this is doing. - if (!isStateEvent) { - // could be a membership change, display name change, etc. - // Find out which one. - var memberChanges = undefined; - if ((event.prev_content === undefined && event.content.membership) || (event.prev_content && (event.prev_content.membership !== event.content.membership))) { - memberChanges = "membership"; - } - else if (event.prev_content && (event.prev_content.displayname !== event.content.displayname)) { - memberChanges = "displayname"; - } - - // mark the key which changed - event.changedKey = memberChanges; - - // If there was a change we want to display, dump it in the message - // list. - if (memberChanges) { - room.addMessageEvent(event, !isLiveEvent); - } + // If there was a change we want to display, dump it in the message + // list. This has to be done after room state is updated. + if (memberChanges) { + room.addMessageEvent(event, !isLiveEvent); } From 1c86ec5b8dcdfaf72259b63d2c588014707c70de Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 10:18:46 +0000 Subject: [PATCH 091/265] Rejig display names when paginating to lie less. --- .../components/matrix/event-handler-service.js | 11 ++++++++--- webclient/components/matrix/model-service.js | 9 +++++++-- webclient/room/room.html | 17 +++++++++-------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index bba24cfaa..226db0798 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -218,14 +218,19 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService else if (!isLiveEvent) { // mutate the old room state var oldEvent = angular.copy(event); + oldEvent.cnt = event.content; if (event.prev_content) { // the m.room.member event we are handling is the NEW event. When // we keep going back in time, we want the PREVIOUS value for displaying // names/etc, hence the clobber here. - // FIXME TODO: We can't be doing this, we should have separate keys for - // avatar_url and displayname. - event.content = event.prev_content; + oldEvent.cnt = event.prev_content; } + + if (event.changedKey === "membership" && event.content.membership === "join") { + // join has a prev_content but it doesn't contain all the info unlike the join, so use that. + oldEvent.cnt = event.content; + } + room.old_room_state.storeStateEvent(oldEvent); } diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index d5acd7eda..7277ae347 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -45,12 +45,17 @@ angular.module('modelService', []) addMessageEvent: function addMessageEvent(event, toFront) { // every message must reference the RoomMember which made it *at // that time* so things like display names display correctly. + var stateAtTheTime = toFront ? this.old_room_state : this.current_room_state; + event.room_member = stateAtTheTime.getStateEvent("m.room.member", event.user_id); + if (event.type === "m.room.member" && event.content.membership === "invite") { + // give information on both the inviter and invitee + event.__target_room_member = stateAtTheTime.getStateEvent("m.room.member", event.state_key); + } + if (toFront) { - event.room_member = this.old_room_state.getStateEvent("m.room.member", event.user_id); this.events.unshift(event); } else { - event.room_member = this.current_room_state.getStateEvent("m.room.member", event.user_id); this.events.push(event); } }, diff --git a/webclient/room/room.html b/webclient/room/room.html index 60b875f17..332a9928d 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -127,7 +127,7 @@
-
{{ msg.room_member.content.displayname || msg.user_id | mUserDisplayName: room_id }}
+
{{ msg.room_member.cnt.displayname || msg.user_id | mUserDisplayName: room_id }}
{{ (msg.origin_server_ts) | date:'MMM d HH:mm' }} @@ -135,22 +135,23 @@
-
- {{ members[msg.state_key].displayname || msg.state_key }} joined + {{ msg.content.displayname || members[msg.state_key].displayname || msg.state_key }} joined - {{ members[msg.state_key].displayname || msg.state_key }} left + + {{ msg.room_member.cnt.displayname || members[msg.state_key].displayname || msg.state_key }} left - {{ members[msg.user_id].displayname || msg.user_id }} + {{ msg.content.displayname || members[msg.user_id].displayname || msg.user_id }} {{ {"invite": "kicked", "join": "kicked", "ban": "unbanned"}[msg.prev_content.membership] }} - {{ members[msg.state_key].displayname || msg.state_key }} + {{ msg.__target_room_member.content.displayname || msg.state_key }} : {{ msg.content.reason }} @@ -158,9 +159,9 @@ - {{ members[msg.user_id].displayname || msg.user_id }} + {{ msg.content.displayname || members[msg.user_id].displayname || msg.user_id }} {{ {"invite": "invited", "ban": "banned"}[msg.content.membership] }} - {{ members[msg.state_key].displayname || msg.state_key }} + {{ msg.__target_room_member.displayname || msg.state_key }} : {{ msg.content.reason }} From 5949571fe75eecefc9ff2433d9de46dfa5b80902 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 10:30:34 +0000 Subject: [PATCH 092/265] SYWEB-116: Implement historical display name support. This works for both live and paginated events. Each 'message' event has an associated '__room_member' key which points to the state of the sender at that point in time. Invites have an additional key '__target_room_member' which point to the state of the invitee at that point in time. This obviates the need for mapping user_ids to *current* displaynames in the message list, though this is still required for the user/presence list. --- webclient/components/matrix/event-handler-service.js | 4 +++- webclient/components/matrix/model-service.js | 2 +- webclient/room/room-controller.js | 3 ++- webclient/room/room.html | 12 ++++++------ 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/webclient/components/matrix/event-handler-service.js b/webclient/components/matrix/event-handler-service.js index 226db0798..027c80a1b 100644 --- a/webclient/components/matrix/event-handler-service.js +++ b/webclient/components/matrix/event-handler-service.js @@ -213,7 +213,9 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService // The events are copied to avoid referencing the same event when adding // the message (circular json structures) if (isStateEvent || isLiveEvent) { - room.current_room_state.storeStateEvent(angular.copy(event)); + var newEvent = angular.copy(event); + newEvent.cnt = event.content; + room.current_room_state.storeStateEvent(newEvent); } else if (!isLiveEvent) { // mutate the old room state diff --git a/webclient/components/matrix/model-service.js b/webclient/components/matrix/model-service.js index 7277ae347..8b2ee877b 100644 --- a/webclient/components/matrix/model-service.js +++ b/webclient/components/matrix/model-service.js @@ -46,7 +46,7 @@ angular.module('modelService', []) // every message must reference the RoomMember which made it *at // that time* so things like display names display correctly. var stateAtTheTime = toFront ? this.old_room_state : this.current_room_state; - event.room_member = stateAtTheTime.getStateEvent("m.room.member", event.user_id); + event.__room_member = stateAtTheTime.getStateEvent("m.room.member", event.user_id); if (event.type === "m.room.member" && event.content.membership === "invite") { // give information on both the inviter and invitee event.__target_room_member = stateAtTheTime.getStateEvent("m.room.member", event.state_key); diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 0bcc33ab4..2e0f5faff 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -1007,7 +1007,8 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) $scope.event_selected = angular.copy(content); // FIXME: Pre-calculated event data should be stripped in a nicer way. - $scope.event_selected.room_member = undefined; + $scope.event_selected.__room_member = undefined; + $scope.event_selected.__target_room_member = undefined; // scope this so the template can check power levels and enable/disable // buttons diff --git a/webclient/room/room.html b/webclient/room/room.html index 332a9928d..407daf56f 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -127,15 +127,15 @@
-
{{ msg.room_member.cnt.displayname || msg.user_id | mUserDisplayName: room_id }}
+
{{ msg.__room_member.cnt.displayname || msg.user_id | mUserDisplayName: room_id }}
{{ (msg.origin_server_ts) | date:'MMM d HH:mm' }}
- - + @@ -146,7 +146,7 @@ - {{ msg.room_member.cnt.displayname || members[msg.state_key].displayname || msg.state_key }} left + {{ msg.__room_member.cnt.displayname || members[msg.state_key].displayname || msg.state_key }} left {{ msg.content.displayname || members[msg.user_id].displayname || msg.user_id }} @@ -159,9 +159,9 @@ - {{ msg.content.displayname || members[msg.user_id].displayname || msg.user_id }} + {{ msg.__room_member.cnt.displayname || msg.user_id }} {{ {"invite": "invited", "ban": "banned"}[msg.content.membership] }} - {{ msg.__target_room_member.displayname || msg.state_key }} + {{ msg.__target_room_member.cnt.displayname || msg.state_key }} : {{ msg.content.reason }} From 1ac1cd6c14275d535f7edcbeeba37b32b6cb056e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 11:06:31 +0000 Subject: [PATCH 093/265] SYWEB-133: JSON dialog now appears on dblclick to allow users to highlight text. --- webclient/room/room.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index 5265f42dd..b9147b1dc 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -137,7 +137,7 @@ ng-hide="events.rooms[room_id].messages[$index - 1].user_id === msg.user_id || msg.user_id === state.user_id"/> -
+
{{ members[msg.state_key].displayname || msg.state_key }} joined From f7cf978f68cddc66c2548e25ddeef468b7f25435 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 11:26:03 +0000 Subject: [PATCH 094/265] SYWEB-136: Send m.images according to the spec. --- webclient/components/fileUpload/file-upload-service.js | 7 ++++--- webclient/components/matrix/matrix-service.js | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/webclient/components/fileUpload/file-upload-service.js b/webclient/components/fileUpload/file-upload-service.js index e0f67b2c6..b544e2950 100644 --- a/webclient/components/fileUpload/file-upload-service.js +++ b/webclient/components/fileUpload/file-upload-service.js @@ -64,7 +64,8 @@ angular.module('mFileUpload', ['matrixService', 'mUtilities']) var imageMessage = { msgtype: "m.image", url: undefined, - body: { + body: "Image", + info: { size: undefined, w: undefined, h: undefined, @@ -90,7 +91,7 @@ angular.module('mFileUpload', ['matrixService', 'mUtilities']) function(url) { // Update message metadata imageMessage.url = url; - imageMessage.body = { + imageMessage.info = { size: imageFile.size, w: size.width, h: size.height, @@ -101,7 +102,7 @@ angular.module('mFileUpload', ['matrixService', 'mUtilities']) // reuse the original image info for thumbnail data if (!imageMessage.thumbnail_url) { imageMessage.thumbnail_url = imageMessage.url; - imageMessage.thumbnail_info = imageMessage.body; + imageMessage.thumbnail_info = imageMessage.info; } // We are done diff --git a/webclient/components/matrix/matrix-service.js b/webclient/components/matrix/matrix-service.js index 1840cf46c..06a1ad4cf 100644 --- a/webclient/components/matrix/matrix-service.js +++ b/webclient/components/matrix/matrix-service.js @@ -422,7 +422,8 @@ angular.module('matrixService', []) var content = { msgtype: "m.image", url: image_url, - body: image_body + info: image_body, + body: "Image" }; return this.sendMessage(room_id, msg_id, content); From 29b54d663803c73f5521a484ffba0182b24b3f89 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 14:01:25 +0000 Subject: [PATCH 095/265] Update karma.conf.js to A: actually run the tests, and B: generate JUnit XML. --- webclient/test/karma.conf.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/webclient/test/karma.conf.js b/webclient/test/karma.conf.js index 22c4eaaaf..df63ae826 100644 --- a/webclient/test/karma.conf.js +++ b/webclient/test/karma.conf.js @@ -23,6 +23,8 @@ module.exports = function(config) { '../js/angular-animate.js', '../js/angular-sanitize.js', '../js/ng-infinite-scroll-matrix.js', + '../js/ui-bootstrap*', + '../js/elastic.js', '../login/**/*.*', '../room/**/*.*', '../components/**/*.*', @@ -35,6 +37,11 @@ module.exports = function(config) { './unit/**/*.js' ], + plugins: [ + 'karma-*', + require('./node_modules/karma-junit-reporter') + ], + // list of files to exclude exclude: [ @@ -50,8 +57,11 @@ module.exports = function(config) { // test results reporter to use // possible values: 'dots', 'progress' // available reporters: https://npmjs.org/browse/keyword/karma-reporter - reporters: ['progress'], - + reporters: ['progress', 'junit'], + junitReporter: { + outputFile: 'test-results.xml', + suite: '' + }, // web server port port: 9876, @@ -77,6 +87,6 @@ module.exports = function(config) { // Continuous Integration mode // if true, Karma captures browsers, runs the tests and exits - singleRun: false + singleRun: true }); }; From aa76bf39aba2eadf506f57952a1dffce629f2637 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 14:14:02 +0000 Subject: [PATCH 096/265] Remove unused imports --- synapse/federation/persistence.py | 2 -- synapse/federation/units.py | 3 --- synapse/handlers/federation.py | 6 ++---- synapse/state.py | 2 -- synapse/storage/_base.py | 2 -- 5 files changed, 2 insertions(+), 13 deletions(-) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index b04fbb417..73dc844d5 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -21,8 +21,6 @@ These actions are mostly only used by the :py:mod:`.replication` module. from twisted.internet import defer -from .units import Pdu - from synapse.util.logutils import log_function import json diff --git a/synapse/federation/units.py b/synapse/federation/units.py index c2d8dca8f..9b2555670 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -18,11 +18,8 @@ server protocol. """ from synapse.util.jsonobject import JsonEncodedObject -from syutil.base64util import encode_base64 import logging -import json -import copy logger = logging.getLogger(__name__) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index bdd28f04b..49bfff88a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,15 +17,13 @@ from ._base import BaseHandler -from synapse.api.events.room import InviteJoinEvent, RoomMemberEvent +from synapse.api.events.room import RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function from synapse.federation.pdu_codec import PduCodec -from synapse.api.errors import SynapseError from synapse.util.async import run_on_reactor -from synapse.types import EventID -from twisted.internet import defer, reactor +from twisted.internet import defer import logging diff --git a/synapse/state.py b/synapse/state.py index f4efc287c..9771883bc 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -19,8 +19,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.types import EventID - from collections import namedtuple import copy diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 464b12f03..5e00c23fd 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -14,8 +14,6 @@ # limitations under the License. import logging -from twisted.internet import defer - from synapse.api.errors import StoreError from synapse.api.events.utils import prune_event from synapse.util.logutils import log_function From d7412c4df1cdfc1128b6d63a38c4ac1d6f08b76b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 14:16:19 +0000 Subject: [PATCH 097/265] Remove unused interface --- synapse/federation/replication.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 838e660a4..99dd390a6 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -533,14 +533,6 @@ class ReplicationLayer(object): return "" % self.server_name -class ReplicationHandler(object): - """This defines the methods that the :py:class:`.ReplicationLayer` will - use to communicate with the rest of the home server. - """ - def on_receive_pdu(self, pdu): - raise NotImplementedError("on_receive_pdu") - - class _TransactionQueue(object): """This class makes sure we only have one transaction in flight at a time for a given destination. From 440cbd5235e7e23dfe97d8e3d394cc0d35b35fd6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 14:17:55 +0000 Subject: [PATCH 098/265] Add support for sending failures --- synapse/federation/replication.py | 30 +++++++++++++++++++++++++-- synapse/federation/units.py | 1 + synapse/types.py | 34 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 99dd390a6..680e7322a 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -143,6 +143,11 @@ class ReplicationLayer(object): self._transaction_queue.enqueue_edu(edu) return defer.succeed(None) + @log_function + def send_failure(self, failure, destination): + self._transaction_queue.enqueue_failure(failure, destination) + return defer.succeed(None) + @log_function def make_query(self, destination, query_type, args, retry_on_dns_fail=True): @@ -558,6 +563,9 @@ class _TransactionQueue(object): # destination -> list of tuple(edu, deferred) self.pending_edus_by_dest = {} + # destination -> list of tuple(failure, deferred) + self.pending_failures_by_dest = {} + # HACK to get unique tx id self._next_txn_id = int(self._clock.time_msec()) @@ -610,6 +618,18 @@ class _TransactionQueue(object): return deferred + @defer.inlineCallbacks + def enqueue_failure(self, failure, destination): + deferred = defer.Deferred() + + self.pending_failures_by_dest.setdefault( + destination, [] + ).append( + (failure, deferred) + ) + + yield deferred + @defer.inlineCallbacks @log_function def _attempt_new_transaction(self, destination): @@ -619,8 +639,9 @@ class _TransactionQueue(object): # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) pending_edus = self.pending_edus_by_dest.pop(destination, []) + pending_failures = self.pending_failures_by_dest(destination, []) - if not pending_pdus and not pending_edus: + if not pending_pdus and not pending_edus and not pending_failures: return logger.debug("TX [%s] Attempting new transaction", destination) @@ -630,7 +651,11 @@ class _TransactionQueue(object): pdus = [x[0] for x in pending_pdus] edus = [x[0] for x in pending_edus] - deferreds = [x[1] for x in pending_pdus + pending_edus] + failures = [x[0].get_dict() for x in pending_failures] + deferreds = [ + x[1] + for x in pending_pdus + pending_edus + pending_failures + ] try: self.pending_transactions[destination] = 1 @@ -644,6 +669,7 @@ class _TransactionQueue(object): destination=destination, pdus=pdus, edus=edus, + pdu_failures=failures, ) self._next_txn_id += 1 diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 9b2555670..2070ffe1e 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -157,6 +157,7 @@ class Transaction(JsonEncodedObject): "edus", "transaction_id", "destination", + "pdu_failures", ] internal_keys = [ diff --git a/synapse/types.py b/synapse/types.py index 649ff2f7d..8fac20fd2 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -128,3 +128,37 @@ class StreamToken( d = self._asdict() d[key] = new_value return StreamToken(**d) + + +class FederationError(RuntimeError): + """ This class is used to inform remote home servers about erroneous + PDUs they sent us. + + FATAL: The remote server could not interpret the source event. + (e.g., it was missing a required field) + ERROR: The remote server interpreted the event, but it failed some other + check (e.g. auth) + WARN: The remote server accepted the event, but believes some part of it + is wrong (e.g., it referred to an invalid event) + """ + + def __init__(self, level, code, reason, affected, source=None): + if level not in ["FATAL", "ERROR", "WARN"]: + raise ValueError("Level is not valid: %s" % (level,)) + self.level = level + self.code = code + self.reason = reason + self.affected = affected + self.source = source + + msg = "%s %s: %s" % (level, code, reason,) + super(FederationError, self).__init__(msg) + + def get_dict(self): + return { + "level": self.level, + "code": self.code, + "reason": self.reason, + "affected": self.affected, + "source": self.source if self.source else self.affected, + } From ae48e75ad70cfc45d01647a4172a6c7f7659a6c0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 14:38:45 +0000 Subject: [PATCH 099/265] Use phantomjs as the default browser and not chrome. --- webclient/test/README | 32 +++++++++++++++++++++++++------- webclient/test/karma.conf.js | 2 +- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/webclient/test/README b/webclient/test/README index 1a7bc832c..e7ed4eaa8 100644 --- a/webclient/test/README +++ b/webclient/test/README @@ -1,13 +1,31 @@ -Requires: - - nodejs/npm - - npm install karma - - npm install jasmine - - npm install protractor (e2e testing) +Testing is done using Karma. -Setting up continuous integration / run the unit tests (make sure you're in -this directory so it can find the config file): + +UNIT TESTING +============ + +Requires the following: + - npm/nodejs + - phantomjs + +Requires the following node packages: + - npm install jasmine + - npm install karma + - npm install karma-jasmine + - npm install karma-phantomjs-launcher + - npm install karma-junit-reporter + +Make sure you're in this directory so it can find the config file and run: karma start +You should see all the tests pass. + + +E2E TESTING +=========== + +npm install protractor + Setting up e2e tests (only if you don't have a selenium server to run the tests on. If you do, edit the config to point to that url): diff --git a/webclient/test/karma.conf.js b/webclient/test/karma.conf.js index df63ae826..83c9dca57 100644 --- a/webclient/test/karma.conf.js +++ b/webclient/test/karma.conf.js @@ -82,7 +82,7 @@ module.exports = function(config) { // start these browsers // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher - browsers: ['Chrome'], + browsers: ['PhantomJS'], // Continuous Integration mode From 402d08099002f19a882f76ca7d2baeb1a29868e7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Nov 2014 15:08:00 +0000 Subject: [PATCH 100/265] Fix installation instructions in README --- MANIFEST.in | 3 ++- README.rst | 57 +++++++++++++++++++++++++++++------------------------ 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 73e0eff6e..e1043fb57 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ recursive-include docs * recursive-include tests *.py -recursive-include synapse/persistence/schema *.sql +recursive-include synapse/storage/schema *.sql +recursive-include webclient * diff --git a/README.rst b/README.rst index f40492b8a..b3b2a94db 100644 --- a/README.rst +++ b/README.rst @@ -122,12 +122,12 @@ Thanks for trying Matrix! [2] End-to-end encryption is currently in development - Homeserver Installation ======================= -First, the dependencies need to be installed. Start by installing -'python2.7-dev' and the various tools of the compiler toolchain. +Synapse is written in python but some of the libraries is uses are written in +C. So before we can install synapse itself we need a working C compiler and the +header files for python C extensions. Installing prerequisites on Ubuntu:: @@ -137,30 +137,35 @@ Installing prerequisites on Mac OS X:: $ xcode-select --install +Synapse uses NaCl (http://nacl.cr.yp.to/) for encryption and digital +signatures. Unfortunately PyNACL currently has a few issues +(https://github.com/pyca/pynacl/issues/53) and +(https://github.com/pyca/pynacl/issues/79) that mean it may not install +correctly. To fix try re-installing from PyPI or directly from (https://github.com/pyca/pynacl):: + + $ # Install from PyPI + $ pip install --user --upgrade --force pynacl + $ # Install from github + $ pip install --user https://github.com/pyca/pynacl/tarball/master + +On OSX, if you encounter ``clang: error: unknown argument: '-mno-fused-madd'`` +you will need to ``export CFLAGS=-Qunused-arguments``. + +To install the synapse homeserver run:: + + $ pip install --user --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + +This installs synapse, along with the libraries it uses, into +``$HOME/.local/lib/``. + +Homeserver Development +====================== + The homeserver has a number of external dependencies, that are easiest to install by making setup.py do so, in --user mode:: $ python setup.py develop --user -You'll need a version of setuptools new enough to know about git, so you -may need to also run:: - - $ sudo apt-get install python-pip - $ sudo pip install --upgrade setuptools - -If you don't have access to github, then you may need to install ``syutil`` -manually by checking it out and running ``python setup.py develop --user`` on -it too. - -If you get errors about ``sodium.h`` being missing, you may also need to -manually install a newer PyNaCl via pip as setuptools installs an old one. Or -you can check PyNaCl out of git directly (https://github.com/pyca/pynacl) and -installing it. Installing PyNaCl using pip may also work (remember to remove -any other versions installed by setuputils in, for example, ~/.local/lib). - -On OSX, if you encounter ``clang: error: unknown argument: '-mno-fused-madd'`` -you will need to ``export CFLAGS=-Qunused-arguments``. - This will run a process of downloading and installing into your user's .local/lib directory all of the required dependencies that are missing. @@ -204,11 +209,11 @@ IDs: For the first form, simply pass the required hostname (of the machine) as the --host parameter:: - $ python synapse/app/homeserver.py \ + $ python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ --config-path homeserver.config \ --generate-config - $ python synapse/app/homeserver.py --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.config Alternatively, you can run synapse via synctl - running ``synctl start`` to generate a homeserver.yaml config file, where you can then edit server-name to @@ -226,12 +231,12 @@ record would then look something like:: At this point, you should then run the homeserver with the hostname of this SRV record, as that is the name other machines will expect it to have:: - $ python synapse/app/homeserver.py \ + $ python -m synapse.app.homeserver \ --server-name YOURDOMAIN \ --bind-port 8448 \ --config-path homeserver.config \ --generate-config - $ python synapse/app/homeserver.py --config-path homeserver.config + $ python -m synapse.app.homeserver --config-path homeserver.config You may additionally want to pass one or more "-v" options, in order to From fc7b2b11a28b49a4d5b50fef66bae0faee47503f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 15:09:34 +0000 Subject: [PATCH 101/265] PEP8 --- synapse/federation/replication.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 680e7322a..98997998f 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -81,7 +81,7 @@ class ReplicationLayer(object): def register_edu_handler(self, edu_type, handler): if edu_type in self.edu_handlers: - raise KeyError("Already have an EDU handler for %s" % (edu_type)) + raise KeyError("Already have an EDU handler for %s" % (edu_type,)) self.edu_handlers[edu_type] = handler @@ -102,7 +102,9 @@ class ReplicationLayer(object): object to encode as JSON. """ if query_type in self.query_handlers: - raise KeyError("Already have a Query handler for %s" % (query_type)) + raise KeyError( + "Already have a Query handler for %s" % (query_type,) + ) self.query_handlers[query_type] = handler @@ -128,7 +130,10 @@ class ReplicationLayer(object): # TODO, add errback, etc. self._transaction_queue.enqueue_pdu(pdu, order) - logger.debug("[%s] transaction_layer.enqueue_pdu... done", pdu.event_id) + logger.debug( + "[%s] transaction_layer.enqueue_pdu... done", + pdu.event_id + ) @log_function def send_edu(self, destination, edu_type, content): @@ -317,7 +322,11 @@ class ReplicationLayer(object): if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu(transaction.origin, edu.edu_type, edu.content) + self.received_edu( + transaction.origin, + edu.edu_type, + edu.content + ) results = yield defer.DeferredList(dl) From 8918422156a3555a61b7e5ba2ee0c04eb3e52cb5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 15:10:27 +0000 Subject: [PATCH 102/265] Move FederationError to synapse.api.errors --- synapse/api/errors.py | 34 ++++++++++++++++++++++++++++++++++ synapse/types.py | 34 ---------------------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 38ccb4f9d..33d15072a 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -158,3 +158,37 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs): for key, value in kwargs.iteritems(): err[key] = value return err + + +class FederationError(RuntimeError): + """ This class is used to inform remote home servers about erroneous + PDUs they sent us. + + FATAL: The remote server could not interpret the source event. + (e.g., it was missing a required field) + ERROR: The remote server interpreted the event, but it failed some other + check (e.g. auth) + WARN: The remote server accepted the event, but believes some part of it + is wrong (e.g., it referred to an invalid event) + """ + + def __init__(self, level, code, reason, affected, source=None): + if level not in ["FATAL", "ERROR", "WARN"]: + raise ValueError("Level is not valid: %s" % (level,)) + self.level = level + self.code = code + self.reason = reason + self.affected = affected + self.source = source + + msg = "%s %s: %s" % (level, code, reason,) + super(FederationError, self).__init__(msg) + + def get_dict(self): + return { + "level": self.level, + "code": self.code, + "reason": self.reason, + "affected": self.affected, + "source": self.source if self.source else self.affected, + } diff --git a/synapse/types.py b/synapse/types.py index 8fac20fd2..649ff2f7d 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -128,37 +128,3 @@ class StreamToken( d = self._asdict() d[key] = new_value return StreamToken(**d) - - -class FederationError(RuntimeError): - """ This class is used to inform remote home servers about erroneous - PDUs they sent us. - - FATAL: The remote server could not interpret the source event. - (e.g., it was missing a required field) - ERROR: The remote server interpreted the event, but it failed some other - check (e.g. auth) - WARN: The remote server accepted the event, but believes some part of it - is wrong (e.g., it referred to an invalid event) - """ - - def __init__(self, level, code, reason, affected, source=None): - if level not in ["FATAL", "ERROR", "WARN"]: - raise ValueError("Level is not valid: %s" % (level,)) - self.level = level - self.code = code - self.reason = reason - self.affected = affected - self.source = source - - msg = "%s %s: %s" % (level, code, reason,) - super(FederationError, self).__init__(msg) - - def get_dict(self): - return { - "level": self.level, - "code": self.code, - "reason": self.reason, - "affected": self.affected, - "source": self.source if self.source else self.affected, - } From 2a49f177fe39ee15c7fc2915db9aadb6152bc547 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 15:10:43 +0000 Subject: [PATCH 103/265] On AuthError, raise a FederationError --- synapse/handlers/federation.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 49bfff88a..8848656b1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,6 +17,7 @@ from ._base import BaseHandler +from synapse.api.errors import AuthError, FederationError from synapse.api.events.room import RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function @@ -116,8 +117,15 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) - if not backfilled: + try: yield self.auth.check(event, None, raises=True) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) is_new_state = is_new_state and not backfilled From a5a4ef3fd7b632d838940fb3bd7b079a897fa8a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 15:16:43 +0000 Subject: [PATCH 104/265] Fix bug in replication --- synapse/federation/replication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 98997998f..92a9678e2 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -648,7 +648,7 @@ class _TransactionQueue(object): # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_failures = self.pending_failures_by_dest(destination, []) + pending_failures = self.pending_failures_by_dest.pop(destination, []) if not pending_pdus and not pending_edus and not pending_failures: return From 4a73c366faad6c64c062545bef6734414bf3dae4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Nov 2014 15:18:43 +0000 Subject: [PATCH 105/265] exclude tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 660efd5b8..0cb56c400 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ def read(fname): setup( name="SynapseHomeServer", version="0.0.1", - packages=find_packages(exclude=["tests"]), + packages=find_packages(exclude=["tests", "tests.*"]), description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", From 1273023ac350328f15569702255caa4345e1368a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 15:35:25 +0000 Subject: [PATCH 106/265] Don't need this; should be on -g path. --- webclient/test/karma.conf.js | 1 - 1 file changed, 1 deletion(-) diff --git a/webclient/test/karma.conf.js b/webclient/test/karma.conf.js index 83c9dca57..083c7c720 100644 --- a/webclient/test/karma.conf.js +++ b/webclient/test/karma.conf.js @@ -39,7 +39,6 @@ module.exports = function(config) { plugins: [ 'karma-*', - require('./node_modules/karma-junit-reporter') ], From 89ba802b23bf1fd22afbc5e9a4b3b732264e3c18 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Nov 2014 15:57:23 +0000 Subject: [PATCH 107/265] Move webclient to a python module so that it can be installed --- MANIFEST.in | 2 +- syweb/__init__.py | 0 {webclient => syweb/webclient}/CAPTCHA_SETUP | 0 {webclient => syweb/webclient}/README | 0 {webclient => syweb/webclient}/app-controller.js | 0 {webclient => syweb/webclient}/app-directive.js | 0 {webclient => syweb/webclient}/app-filter.js | 0 {webclient => syweb/webclient}/app.css | 0 {webclient => syweb/webclient}/app.js | 0 {webclient => syweb/webclient}/bootstrap.css | 0 .../components/fileInput/file-input-directive.js | 0 .../components/fileUpload/file-upload-service.js | 0 .../components/matrix/event-handler-service.js | 0 .../components/matrix/event-stream-service.js | 0 .../webclient}/components/matrix/matrix-call.js | 0 .../webclient}/components/matrix/matrix-filter.js | 0 .../components/matrix/matrix-phone-service.js | 0 .../webclient}/components/matrix/matrix-service.js | 0 .../webclient}/components/matrix/model-service.js | 0 .../components/matrix/notification-service.js | 0 .../components/matrix/presence-service.js | 0 .../components/utilities/utilities-service.js | 0 {webclient => syweb/webclient}/favicon.ico | Bin .../webclient}/home/home-controller.js | 0 {webclient => syweb/webclient}/home/home.html | 0 {webclient => syweb/webclient}/img/close.png | Bin .../webclient}/img/default-profile.png | Bin {webclient => syweb/webclient}/img/gradient.png | Bin {webclient => syweb/webclient}/img/green_phone.png | Bin {webclient => syweb/webclient}/img/logo-small.png | Bin {webclient => syweb/webclient}/img/logo.png | Bin {webclient => syweb/webclient}/img/red_phone.png | Bin {webclient => syweb/webclient}/index.html | 0 .../webclient}/js/angular-animate.js | 0 .../webclient}/js/angular-animate.min.js | 0 {webclient => syweb/webclient}/js/angular-mocks.js | 0 {webclient => syweb/webclient}/js/angular-route.js | 0 .../webclient}/js/angular-route.min.js | 0 .../webclient}/js/angular-sanitize.js | 0 .../webclient}/js/angular-sanitize.min.js | 0 {webclient => syweb/webclient}/js/angular.js | 0 {webclient => syweb/webclient}/js/angular.min.js | 0 {webclient => syweb/webclient}/js/autofill-event.js | 0 {webclient => syweb/webclient}/js/elastic.js | 0 .../webclient}/js/jquery-1.8.3.min.js | 0 .../webclient}/js/ng-infinite-scroll-matrix.js | 0 .../webclient}/js/ui-bootstrap-tpls-0.11.2.js | 0 .../webclient}/login/login-controller.js | 0 {webclient => syweb/webclient}/login/login.html | 0 .../webclient}/login/register-controller.js | 0 {webclient => syweb/webclient}/login/register.html | 0 {webclient => syweb/webclient}/media/busy.mp3 | Bin {webclient => syweb/webclient}/media/busy.ogg | Bin {webclient => syweb/webclient}/media/callend.mp3 | Bin {webclient => syweb/webclient}/media/callend.ogg | Bin {webclient => syweb/webclient}/media/ring.mp3 | Bin {webclient => syweb/webclient}/media/ring.ogg | Bin {webclient => syweb/webclient}/media/ringback.mp3 | Bin {webclient => syweb/webclient}/media/ringback.ogg | Bin {webclient => syweb/webclient}/mobile.css | 0 .../webclient}/recents/recents-controller.js | 0 .../webclient}/recents/recents-filter.js | 0 {webclient => syweb/webclient}/recents/recents.html | 0 .../webclient}/room/room-controller.js | 0 .../webclient}/room/room-directive.js | 0 {webclient => syweb/webclient}/room/room.html | 0 .../webclient}/settings/settings-controller.js | 0 .../webclient}/settings/settings.html | 0 {webclient => syweb/webclient}/test/README | 0 .../webclient}/test/e2e/home.spec.js | 0 {webclient => syweb/webclient}/test/karma.conf.js | 0 .../webclient}/test/protractor.conf.js | 0 .../webclient}/test/unit/user-controller.spec.js | 0 .../webclient}/user/user-controller.js | 0 {webclient => syweb/webclient}/user/user.html | 0 75 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 syweb/__init__.py rename {webclient => syweb/webclient}/CAPTCHA_SETUP (100%) rename {webclient => syweb/webclient}/README (100%) rename {webclient => syweb/webclient}/app-controller.js (100%) rename {webclient => syweb/webclient}/app-directive.js (100%) rename {webclient => syweb/webclient}/app-filter.js (100%) rename {webclient => syweb/webclient}/app.css (100%) rename {webclient => syweb/webclient}/app.js (100%) rename {webclient => syweb/webclient}/bootstrap.css (100%) rename {webclient => syweb/webclient}/components/fileInput/file-input-directive.js (100%) rename {webclient => syweb/webclient}/components/fileUpload/file-upload-service.js (100%) rename {webclient => syweb/webclient}/components/matrix/event-handler-service.js (100%) rename {webclient => syweb/webclient}/components/matrix/event-stream-service.js (100%) rename {webclient => syweb/webclient}/components/matrix/matrix-call.js (100%) rename {webclient => syweb/webclient}/components/matrix/matrix-filter.js (100%) rename {webclient => syweb/webclient}/components/matrix/matrix-phone-service.js (100%) rename {webclient => syweb/webclient}/components/matrix/matrix-service.js (100%) rename {webclient => syweb/webclient}/components/matrix/model-service.js (100%) rename {webclient => syweb/webclient}/components/matrix/notification-service.js (100%) rename {webclient => syweb/webclient}/components/matrix/presence-service.js (100%) rename {webclient => syweb/webclient}/components/utilities/utilities-service.js (100%) rename {webclient => syweb/webclient}/favicon.ico (100%) rename {webclient => syweb/webclient}/home/home-controller.js (100%) rename {webclient => syweb/webclient}/home/home.html (100%) rename {webclient => syweb/webclient}/img/close.png (100%) rename {webclient => syweb/webclient}/img/default-profile.png (100%) rename {webclient => syweb/webclient}/img/gradient.png (100%) rename {webclient => syweb/webclient}/img/green_phone.png (100%) rename {webclient => syweb/webclient}/img/logo-small.png (100%) rename {webclient => syweb/webclient}/img/logo.png (100%) rename {webclient => syweb/webclient}/img/red_phone.png (100%) rename {webclient => syweb/webclient}/index.html (100%) rename {webclient => syweb/webclient}/js/angular-animate.js (100%) rename {webclient => syweb/webclient}/js/angular-animate.min.js (100%) rename {webclient => syweb/webclient}/js/angular-mocks.js (100%) rename {webclient => syweb/webclient}/js/angular-route.js (100%) rename {webclient => syweb/webclient}/js/angular-route.min.js (100%) rename {webclient => syweb/webclient}/js/angular-sanitize.js (100%) rename {webclient => syweb/webclient}/js/angular-sanitize.min.js (100%) rename {webclient => syweb/webclient}/js/angular.js (100%) rename {webclient => syweb/webclient}/js/angular.min.js (100%) rename {webclient => syweb/webclient}/js/autofill-event.js (100%) rename {webclient => syweb/webclient}/js/elastic.js (100%) rename {webclient => syweb/webclient}/js/jquery-1.8.3.min.js (100%) rename {webclient => syweb/webclient}/js/ng-infinite-scroll-matrix.js (100%) rename {webclient => syweb/webclient}/js/ui-bootstrap-tpls-0.11.2.js (100%) rename {webclient => syweb/webclient}/login/login-controller.js (100%) rename {webclient => syweb/webclient}/login/login.html (100%) rename {webclient => syweb/webclient}/login/register-controller.js (100%) rename {webclient => syweb/webclient}/login/register.html (100%) rename {webclient => syweb/webclient}/media/busy.mp3 (100%) rename {webclient => syweb/webclient}/media/busy.ogg (100%) rename {webclient => syweb/webclient}/media/callend.mp3 (100%) rename {webclient => syweb/webclient}/media/callend.ogg (100%) rename {webclient => syweb/webclient}/media/ring.mp3 (100%) rename {webclient => syweb/webclient}/media/ring.ogg (100%) rename {webclient => syweb/webclient}/media/ringback.mp3 (100%) rename {webclient => syweb/webclient}/media/ringback.ogg (100%) rename {webclient => syweb/webclient}/mobile.css (100%) rename {webclient => syweb/webclient}/recents/recents-controller.js (100%) rename {webclient => syweb/webclient}/recents/recents-filter.js (100%) rename {webclient => syweb/webclient}/recents/recents.html (100%) rename {webclient => syweb/webclient}/room/room-controller.js (100%) rename {webclient => syweb/webclient}/room/room-directive.js (100%) rename {webclient => syweb/webclient}/room/room.html (100%) rename {webclient => syweb/webclient}/settings/settings-controller.js (100%) rename {webclient => syweb/webclient}/settings/settings.html (100%) rename {webclient => syweb/webclient}/test/README (100%) rename {webclient => syweb/webclient}/test/e2e/home.spec.js (100%) rename {webclient => syweb/webclient}/test/karma.conf.js (100%) rename {webclient => syweb/webclient}/test/protractor.conf.js (100%) rename {webclient => syweb/webclient}/test/unit/user-controller.spec.js (100%) rename {webclient => syweb/webclient}/user/user-controller.js (100%) rename {webclient => syweb/webclient}/user/user.html (100%) diff --git a/MANIFEST.in b/MANIFEST.in index e1043fb57..a1a77ff54 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ recursive-include docs * recursive-include tests *.py recursive-include synapse/storage/schema *.sql -recursive-include webclient * +recursive-include syweb/webclient * diff --git a/syweb/__init__.py b/syweb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/webclient/CAPTCHA_SETUP b/syweb/webclient/CAPTCHA_SETUP similarity index 100% rename from webclient/CAPTCHA_SETUP rename to syweb/webclient/CAPTCHA_SETUP diff --git a/webclient/README b/syweb/webclient/README similarity index 100% rename from webclient/README rename to syweb/webclient/README diff --git a/webclient/app-controller.js b/syweb/webclient/app-controller.js similarity index 100% rename from webclient/app-controller.js rename to syweb/webclient/app-controller.js diff --git a/webclient/app-directive.js b/syweb/webclient/app-directive.js similarity index 100% rename from webclient/app-directive.js rename to syweb/webclient/app-directive.js diff --git a/webclient/app-filter.js b/syweb/webclient/app-filter.js similarity index 100% rename from webclient/app-filter.js rename to syweb/webclient/app-filter.js diff --git a/webclient/app.css b/syweb/webclient/app.css similarity index 100% rename from webclient/app.css rename to syweb/webclient/app.css diff --git a/webclient/app.js b/syweb/webclient/app.js similarity index 100% rename from webclient/app.js rename to syweb/webclient/app.js diff --git a/webclient/bootstrap.css b/syweb/webclient/bootstrap.css similarity index 100% rename from webclient/bootstrap.css rename to syweb/webclient/bootstrap.css diff --git a/webclient/components/fileInput/file-input-directive.js b/syweb/webclient/components/fileInput/file-input-directive.js similarity index 100% rename from webclient/components/fileInput/file-input-directive.js rename to syweb/webclient/components/fileInput/file-input-directive.js diff --git a/webclient/components/fileUpload/file-upload-service.js b/syweb/webclient/components/fileUpload/file-upload-service.js similarity index 100% rename from webclient/components/fileUpload/file-upload-service.js rename to syweb/webclient/components/fileUpload/file-upload-service.js diff --git a/webclient/components/matrix/event-handler-service.js b/syweb/webclient/components/matrix/event-handler-service.js similarity index 100% rename from webclient/components/matrix/event-handler-service.js rename to syweb/webclient/components/matrix/event-handler-service.js diff --git a/webclient/components/matrix/event-stream-service.js b/syweb/webclient/components/matrix/event-stream-service.js similarity index 100% rename from webclient/components/matrix/event-stream-service.js rename to syweb/webclient/components/matrix/event-stream-service.js diff --git a/webclient/components/matrix/matrix-call.js b/syweb/webclient/components/matrix/matrix-call.js similarity index 100% rename from webclient/components/matrix/matrix-call.js rename to syweb/webclient/components/matrix/matrix-call.js diff --git a/webclient/components/matrix/matrix-filter.js b/syweb/webclient/components/matrix/matrix-filter.js similarity index 100% rename from webclient/components/matrix/matrix-filter.js rename to syweb/webclient/components/matrix/matrix-filter.js diff --git a/webclient/components/matrix/matrix-phone-service.js b/syweb/webclient/components/matrix/matrix-phone-service.js similarity index 100% rename from webclient/components/matrix/matrix-phone-service.js rename to syweb/webclient/components/matrix/matrix-phone-service.js diff --git a/webclient/components/matrix/matrix-service.js b/syweb/webclient/components/matrix/matrix-service.js similarity index 100% rename from webclient/components/matrix/matrix-service.js rename to syweb/webclient/components/matrix/matrix-service.js diff --git a/webclient/components/matrix/model-service.js b/syweb/webclient/components/matrix/model-service.js similarity index 100% rename from webclient/components/matrix/model-service.js rename to syweb/webclient/components/matrix/model-service.js diff --git a/webclient/components/matrix/notification-service.js b/syweb/webclient/components/matrix/notification-service.js similarity index 100% rename from webclient/components/matrix/notification-service.js rename to syweb/webclient/components/matrix/notification-service.js diff --git a/webclient/components/matrix/presence-service.js b/syweb/webclient/components/matrix/presence-service.js similarity index 100% rename from webclient/components/matrix/presence-service.js rename to syweb/webclient/components/matrix/presence-service.js diff --git a/webclient/components/utilities/utilities-service.js b/syweb/webclient/components/utilities/utilities-service.js similarity index 100% rename from webclient/components/utilities/utilities-service.js rename to syweb/webclient/components/utilities/utilities-service.js diff --git a/webclient/favicon.ico b/syweb/webclient/favicon.ico similarity index 100% rename from webclient/favicon.ico rename to syweb/webclient/favicon.ico diff --git a/webclient/home/home-controller.js b/syweb/webclient/home/home-controller.js similarity index 100% rename from webclient/home/home-controller.js rename to syweb/webclient/home/home-controller.js diff --git a/webclient/home/home.html b/syweb/webclient/home/home.html similarity index 100% rename from webclient/home/home.html rename to syweb/webclient/home/home.html diff --git a/webclient/img/close.png b/syweb/webclient/img/close.png similarity index 100% rename from webclient/img/close.png rename to syweb/webclient/img/close.png diff --git a/webclient/img/default-profile.png b/syweb/webclient/img/default-profile.png similarity index 100% rename from webclient/img/default-profile.png rename to syweb/webclient/img/default-profile.png diff --git a/webclient/img/gradient.png b/syweb/webclient/img/gradient.png similarity index 100% rename from webclient/img/gradient.png rename to syweb/webclient/img/gradient.png diff --git a/webclient/img/green_phone.png b/syweb/webclient/img/green_phone.png similarity index 100% rename from webclient/img/green_phone.png rename to syweb/webclient/img/green_phone.png diff --git a/webclient/img/logo-small.png b/syweb/webclient/img/logo-small.png similarity index 100% rename from webclient/img/logo-small.png rename to syweb/webclient/img/logo-small.png diff --git a/webclient/img/logo.png b/syweb/webclient/img/logo.png similarity index 100% rename from webclient/img/logo.png rename to syweb/webclient/img/logo.png diff --git a/webclient/img/red_phone.png b/syweb/webclient/img/red_phone.png similarity index 100% rename from webclient/img/red_phone.png rename to syweb/webclient/img/red_phone.png diff --git a/webclient/index.html b/syweb/webclient/index.html similarity index 100% rename from webclient/index.html rename to syweb/webclient/index.html diff --git a/webclient/js/angular-animate.js b/syweb/webclient/js/angular-animate.js similarity index 100% rename from webclient/js/angular-animate.js rename to syweb/webclient/js/angular-animate.js diff --git a/webclient/js/angular-animate.min.js b/syweb/webclient/js/angular-animate.min.js similarity index 100% rename from webclient/js/angular-animate.min.js rename to syweb/webclient/js/angular-animate.min.js diff --git a/webclient/js/angular-mocks.js b/syweb/webclient/js/angular-mocks.js similarity index 100% rename from webclient/js/angular-mocks.js rename to syweb/webclient/js/angular-mocks.js diff --git a/webclient/js/angular-route.js b/syweb/webclient/js/angular-route.js similarity index 100% rename from webclient/js/angular-route.js rename to syweb/webclient/js/angular-route.js diff --git a/webclient/js/angular-route.min.js b/syweb/webclient/js/angular-route.min.js similarity index 100% rename from webclient/js/angular-route.min.js rename to syweb/webclient/js/angular-route.min.js diff --git a/webclient/js/angular-sanitize.js b/syweb/webclient/js/angular-sanitize.js similarity index 100% rename from webclient/js/angular-sanitize.js rename to syweb/webclient/js/angular-sanitize.js diff --git a/webclient/js/angular-sanitize.min.js b/syweb/webclient/js/angular-sanitize.min.js similarity index 100% rename from webclient/js/angular-sanitize.min.js rename to syweb/webclient/js/angular-sanitize.min.js diff --git a/webclient/js/angular.js b/syweb/webclient/js/angular.js similarity index 100% rename from webclient/js/angular.js rename to syweb/webclient/js/angular.js diff --git a/webclient/js/angular.min.js b/syweb/webclient/js/angular.min.js similarity index 100% rename from webclient/js/angular.min.js rename to syweb/webclient/js/angular.min.js diff --git a/webclient/js/autofill-event.js b/syweb/webclient/js/autofill-event.js similarity index 100% rename from webclient/js/autofill-event.js rename to syweb/webclient/js/autofill-event.js diff --git a/webclient/js/elastic.js b/syweb/webclient/js/elastic.js similarity index 100% rename from webclient/js/elastic.js rename to syweb/webclient/js/elastic.js diff --git a/webclient/js/jquery-1.8.3.min.js b/syweb/webclient/js/jquery-1.8.3.min.js similarity index 100% rename from webclient/js/jquery-1.8.3.min.js rename to syweb/webclient/js/jquery-1.8.3.min.js diff --git a/webclient/js/ng-infinite-scroll-matrix.js b/syweb/webclient/js/ng-infinite-scroll-matrix.js similarity index 100% rename from webclient/js/ng-infinite-scroll-matrix.js rename to syweb/webclient/js/ng-infinite-scroll-matrix.js diff --git a/webclient/js/ui-bootstrap-tpls-0.11.2.js b/syweb/webclient/js/ui-bootstrap-tpls-0.11.2.js similarity index 100% rename from webclient/js/ui-bootstrap-tpls-0.11.2.js rename to syweb/webclient/js/ui-bootstrap-tpls-0.11.2.js diff --git a/webclient/login/login-controller.js b/syweb/webclient/login/login-controller.js similarity index 100% rename from webclient/login/login-controller.js rename to syweb/webclient/login/login-controller.js diff --git a/webclient/login/login.html b/syweb/webclient/login/login.html similarity index 100% rename from webclient/login/login.html rename to syweb/webclient/login/login.html diff --git a/webclient/login/register-controller.js b/syweb/webclient/login/register-controller.js similarity index 100% rename from webclient/login/register-controller.js rename to syweb/webclient/login/register-controller.js diff --git a/webclient/login/register.html b/syweb/webclient/login/register.html similarity index 100% rename from webclient/login/register.html rename to syweb/webclient/login/register.html diff --git a/webclient/media/busy.mp3 b/syweb/webclient/media/busy.mp3 similarity index 100% rename from webclient/media/busy.mp3 rename to syweb/webclient/media/busy.mp3 diff --git a/webclient/media/busy.ogg b/syweb/webclient/media/busy.ogg similarity index 100% rename from webclient/media/busy.ogg rename to syweb/webclient/media/busy.ogg diff --git a/webclient/media/callend.mp3 b/syweb/webclient/media/callend.mp3 similarity index 100% rename from webclient/media/callend.mp3 rename to syweb/webclient/media/callend.mp3 diff --git a/webclient/media/callend.ogg b/syweb/webclient/media/callend.ogg similarity index 100% rename from webclient/media/callend.ogg rename to syweb/webclient/media/callend.ogg diff --git a/webclient/media/ring.mp3 b/syweb/webclient/media/ring.mp3 similarity index 100% rename from webclient/media/ring.mp3 rename to syweb/webclient/media/ring.mp3 diff --git a/webclient/media/ring.ogg b/syweb/webclient/media/ring.ogg similarity index 100% rename from webclient/media/ring.ogg rename to syweb/webclient/media/ring.ogg diff --git a/webclient/media/ringback.mp3 b/syweb/webclient/media/ringback.mp3 similarity index 100% rename from webclient/media/ringback.mp3 rename to syweb/webclient/media/ringback.mp3 diff --git a/webclient/media/ringback.ogg b/syweb/webclient/media/ringback.ogg similarity index 100% rename from webclient/media/ringback.ogg rename to syweb/webclient/media/ringback.ogg diff --git a/webclient/mobile.css b/syweb/webclient/mobile.css similarity index 100% rename from webclient/mobile.css rename to syweb/webclient/mobile.css diff --git a/webclient/recents/recents-controller.js b/syweb/webclient/recents/recents-controller.js similarity index 100% rename from webclient/recents/recents-controller.js rename to syweb/webclient/recents/recents-controller.js diff --git a/webclient/recents/recents-filter.js b/syweb/webclient/recents/recents-filter.js similarity index 100% rename from webclient/recents/recents-filter.js rename to syweb/webclient/recents/recents-filter.js diff --git a/webclient/recents/recents.html b/syweb/webclient/recents/recents.html similarity index 100% rename from webclient/recents/recents.html rename to syweb/webclient/recents/recents.html diff --git a/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js similarity index 100% rename from webclient/room/room-controller.js rename to syweb/webclient/room/room-controller.js diff --git a/webclient/room/room-directive.js b/syweb/webclient/room/room-directive.js similarity index 100% rename from webclient/room/room-directive.js rename to syweb/webclient/room/room-directive.js diff --git a/webclient/room/room.html b/syweb/webclient/room/room.html similarity index 100% rename from webclient/room/room.html rename to syweb/webclient/room/room.html diff --git a/webclient/settings/settings-controller.js b/syweb/webclient/settings/settings-controller.js similarity index 100% rename from webclient/settings/settings-controller.js rename to syweb/webclient/settings/settings-controller.js diff --git a/webclient/settings/settings.html b/syweb/webclient/settings/settings.html similarity index 100% rename from webclient/settings/settings.html rename to syweb/webclient/settings/settings.html diff --git a/webclient/test/README b/syweb/webclient/test/README similarity index 100% rename from webclient/test/README rename to syweb/webclient/test/README diff --git a/webclient/test/e2e/home.spec.js b/syweb/webclient/test/e2e/home.spec.js similarity index 100% rename from webclient/test/e2e/home.spec.js rename to syweb/webclient/test/e2e/home.spec.js diff --git a/webclient/test/karma.conf.js b/syweb/webclient/test/karma.conf.js similarity index 100% rename from webclient/test/karma.conf.js rename to syweb/webclient/test/karma.conf.js diff --git a/webclient/test/protractor.conf.js b/syweb/webclient/test/protractor.conf.js similarity index 100% rename from webclient/test/protractor.conf.js rename to syweb/webclient/test/protractor.conf.js diff --git a/webclient/test/unit/user-controller.spec.js b/syweb/webclient/test/unit/user-controller.spec.js similarity index 100% rename from webclient/test/unit/user-controller.spec.js rename to syweb/webclient/test/unit/user-controller.spec.js diff --git a/webclient/user/user-controller.js b/syweb/webclient/user/user-controller.js similarity index 100% rename from webclient/user/user-controller.js rename to syweb/webclient/user/user-controller.js diff --git a/webclient/user/user.html b/syweb/webclient/user/user.html similarity index 100% rename from webclient/user/user.html rename to syweb/webclient/user/user.html From 4e52f9699b941f7845c2d88fb11428a2c30de465 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 16:15:13 +0000 Subject: [PATCH 108/265] Update .gitignore to ignore config.js files in syweb/webclient. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b91b52b61..bd37f92dc 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,7 @@ graph/*.svg graph/*.png graph/*.dot -webclient/config.js +**/webclient/config.js webclient/test/environment-protractor.js uploads From 24305ba5bf39538f692e108a4f9dcc13c0e4ab85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:15:30 +0000 Subject: [PATCH 109/265] Fix up federation tests --- tests/federation/test_federation.py | 238 +++++++++++++++------------- 1 file changed, 124 insertions(+), 114 deletions(-) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 3a14c7c3b..eb329eec5 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -24,7 +24,6 @@ from ..utils import MockHttpResource, MockClock, MockKey from synapse.server import HomeServer from synapse.federation import initialize_http_replication from synapse.federation.units import Pdu -from synapse.storage.pdu import PduTuple, PduEntry def make_pdu(prev_pdus=[], **kwargs): @@ -41,7 +40,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}, {}) + return Pdu(prev_pdus=prev_pdus, **pdu_fields) class FederationTestCase(unittest.TestCase): @@ -52,179 +51,185 @@ class FederationTestCase(unittest.TestCase): "put_json", ]) self.mock_persistence = Mock(spec=[ - "get_current_state_for_context", - "get_pdu", - "persist_event", - "update_min_depth_for_context", "prep_send_transaction", "delivered_txn", "get_received_txn_response", "set_received_txn_response", ]) self.mock_persistence.get_received_txn_response.return_value = ( - defer.succeed(None) + defer.succeed(None) ) self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] self.clock = MockClock() - hs = HomeServer("test", - resource_for_federation=self.mock_resource, - http_client=self.mock_http_client, - db_pool=None, - datastore=self.mock_persistence, - clock=self.clock, - config=self.mock_config, - keyring=Mock(), + hs = HomeServer( + "test", + resource_for_federation=self.mock_resource, + http_client=self.mock_http_client, + db_pool=None, + datastore=self.mock_persistence, + clock=self.clock, + config=self.mock_config, + keyring=Mock(), ) self.federation = initialize_http_replication(hs) self.distributor = hs.get_distributor() @defer.inlineCallbacks def test_get_state(self): - self.mock_persistence.get_current_state_for_context.return_value = ( - defer.succeed([]) - ) + mock_handler = Mock(spec=[ + "get_state_for_pdu", + ]) + + self.federation.set_handler(mock_handler) + + mock_handler.get_state_for_pdu.return_value = defer.succeed([]) # Empty context initially - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/state/my-context/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/state/my-context/", + None + ) self.assertEquals(200, code) self.assertFalse(response["pdus"]) # Now lets give the context some state - self.mock_persistence.get_current_state_for_context.return_value = ( + mock_handler.get_state_for_pdu.return_value = ( defer.succeed([ make_pdu( - pdu_id="the-pdu-id", + event_id="the-pdu-id", origin="red", - context="my-context", - pdu_type="m.topic", - ts=123456789000, + room_id="my-context", + type="m.topic", + origin_server_ts=123456789000, depth=1, - is_state=True, - content_json='{"topic":"The topic"}', + content={"topic": "The topic"}, state_key="", power_level=1000, - prev_state_id="last-pdu-id", - prev_state_origin="blue", + prev_state="last-pdu-id", ), ]) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/state/my-context/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/state/my-context/", + None + ) self.assertEquals(200, code) self.assertEquals(1, len(response["pdus"])) @defer.inlineCallbacks def test_get_pdu(self): - self.mock_persistence.get_pdu.return_value = ( + mock_handler = Mock(spec=[ + "get_persisted_pdu", + ]) + + self.federation.set_handler(mock_handler) + + mock_handler.get_persisted_pdu.return_value = ( defer.succeed(None) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/pdu/red/abc123def456/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/event/abc123def456/", + None + ) self.assertEquals(404, code) # Now insert such a PDU - self.mock_persistence.get_pdu.return_value = ( + mock_handler.get_persisted_pdu.return_value = ( defer.succeed( make_pdu( - pdu_id="abc123def456", + event_id="abc123def456", origin="red", - context="my-context", - pdu_type="m.text", - ts=123456789001, + room_id="my-context", + type="m.text", + origin_server_ts=123456789001, depth=1, - content_json='{"text":"Here is the message"}', + content={"text": "Here is the message"}, ) ) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/pdu/red/abc123def456/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/event/abc123def456/", + None + ) self.assertEquals(200, code) self.assertEquals(1, len(response["pdus"])) - self.assertEquals("m.text", response["pdus"][0]["pdu_type"]) + self.assertEquals("m.text", response["pdus"][0]["type"]) @defer.inlineCallbacks def test_send_pdu(self): self.mock_http_client.put_json.return_value = defer.succeed( - (200, "OK") + (200, "OK") ) pdu = Pdu( - pdu_id="abc123def456", - origin="red", - destinations=["remote"], - context="my-context", - origin_server_ts=123456789002, - pdu_type="m.test", - content={"testing": "content here"}, - depth=1, + event_id="abc123def456", + origin="red", + room_id="my-context", + type="m.text", + origin_server_ts=123456789001, + depth=1, + content={"text": "Here is the message"}, + destinations=["remote"], ) yield self.federation.send_pdu(pdu) self.mock_http_client.put_json.assert_called_with( - "remote", - path="/_matrix/federation/v1/send/1000000/", - data={ - "origin_server_ts": 1000000, - "origin": "test", - "pdus": [ - { - "origin": "red", - "pdu_id": "abc123def456", - "prev_pdus": [], - "origin_server_ts": 123456789002, - "context": "my-context", - "pdu_type": "m.test", - "is_state": False, - "content": {"testing": "content here"}, - "depth": 1, - "hashes": {}, - "signatures": {}, - }, - ] - }, - json_data_callback=ANY, + "remote", + path="/_matrix/federation/v1/send/1000000/", + data={ + "origin_server_ts": 1000000, + "origin": "test", + "pdus": [ + pdu.get_dict(), + ], + 'pdu_failures': [], + }, + json_data_callback=ANY, ) @defer.inlineCallbacks def test_send_edu(self): self.mock_http_client.put_json.return_value = defer.succeed( - (200, "OK") + (200, "OK") ) yield self.federation.send_edu( - destination="remote", - edu_type="m.test", - content={"testing": "content here"}, + destination="remote", + edu_type="m.test", + content={"testing": "content here"}, ) # MockClock ensures we can guess these timestamps self.mock_http_client.put_json.assert_called_with( - "remote", - path="/_matrix/federation/v1/send/1000000/", - data={ - "origin": "test", - "origin_server_ts": 1000000, - "pdus": [], - "edus": [ - { - # TODO: SYN-103: Remove "origin" and "destination" - "origin": "test", - "destination": "remote", - "edu_type": "m.test", - "content": {"testing": "content here"}, - } - ], - }, - json_data_callback=ANY, + "remote", + path="/_matrix/federation/v1/send/1000000/", + data={ + "origin": "test", + "origin_server_ts": 1000000, + "pdus": [], + "edus": [ + { + # TODO: SYN-103: Remove "origin" and "destination" + "origin": "test", + "destination": "remote", + "edu_type": "m.test", + "content": {"testing": "content here"}, + } + ], + 'pdu_failures': [], + }, + json_data_callback=ANY, ) - @defer.inlineCallbacks def test_recv_edu(self): recv_observer = Mock() @@ -232,24 +237,26 @@ class FederationTestCase(unittest.TestCase): self.federation.register_edu_handler("m.test", recv_observer) - yield self.mock_resource.trigger("PUT", - "/_matrix/federation/v1/send/1001000/", - """{ - "origin": "remote", - "origin_server_ts": 1001000, - "pdus": [], - "edus": [ - { - "origin": "remote", - "destination": "test", - "edu_type": "m.test", - "content": {"testing": "reply here"} - } - ] - }""") + yield self.mock_resource.trigger( + "PUT", + "/_matrix/federation/v1/send/1001000/", + """{ + "origin": "remote", + "origin_server_ts": 1001000, + "pdus": [], + "edus": [ + { + "origin": "remote", + "destination": "test", + "edu_type": "m.test", + "content": {"testing": "reply here"} + } + ] + }""" + ) recv_observer.assert_called_with( - "remote", {"testing": "reply here"} + "remote", {"testing": "reply here"} ) @defer.inlineCallbacks @@ -280,8 +287,11 @@ class FederationTestCase(unittest.TestCase): self.federation.register_query_handler("a-question", recv_handler) - code, response = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/query/a-question?three=3&four=4", None) + code, response = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/query/a-question?three=3&four=4", + None + ) self.assertEquals(200, code) self.assertEquals({"another": "response"}, response) From dca3ba2f77a3530d78a951556ea8c6cfdc0463df Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Nov 2014 16:19:03 +0000 Subject: [PATCH 110/265] Determine webclient path by the python module it is contained in --- demo/start.sh | 2 +- synapse/app/homeserver.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index fc6cd6303..8b0cc84fe 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -41,6 +41,6 @@ for port in 8080 8081 8082; do done echo "Starting webclient on port 8000..." -python "demo/webserver.py" -p 8000 -P "$DIR/webserver.pid" "webclient" +python "demo/webserver.py" -p 8000 -P "$DIR/webserver.pid" "syweb/webclient" cd "$CWD" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6394bc27d..b3dae5da6 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -42,6 +42,7 @@ import os import re import sys import sqlite3 +import syweb logger = logging.getLogger(__name__) @@ -58,7 +59,9 @@ class SynapseHomeServer(HomeServer): return JsonResource() def build_resource_for_web_client(self): - return File("webclient") # TODO configurable? + syweb_path = os.path.dirname(syweb.__file__) + webclient_path = os.path.join(syweb_path, "webclient") + return File(webclient_path) # TODO configurable? def build_resource_for_content_repo(self): return ContentRepoResource( From b15e8d5bbc58ccce49b852a07c5f6364e18c39cd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:20:02 +0000 Subject: [PATCH 111/265] event <-> pdu mappings are now trivial and will soon be scrapped --- tests/federation/test_pdu_codec.py | 167 ----------------------------- 1 file changed, 167 deletions(-) delete mode 100644 tests/federation/test_pdu_codec.py diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py deleted file mode 100644 index fd0f72c73..000000000 --- a/tests/federation/test_pdu_codec.py +++ /dev/null @@ -1,167 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from tests import unittest - -from synapse.federation.pdu_codec import ( - PduCodec, encode_event_id, decode_event_id -) -from synapse.federation.units import Pdu -#from synapse.api.events.room import MessageEvent - -from synapse.server import HomeServer - -from mock import Mock, NonCallableMock - -from ..utils import MockKey - - -class PduCodecTestCase(unittest.TestCase): - def setUp(self): - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - self.hs = HomeServer( - "blargle.net", - config=self.mock_config, - ) - self.event_factory = self.hs.get_event_factory() - self.codec = PduCodec(self.hs) - - def test_decode_event_id(self): - self.assertEquals( - ("foo", "bar.com"), - decode_event_id("foo@bar.com", "A") - ) - - self.assertEquals( - ("foo", "bar.com"), - decode_event_id("foo", "bar.com") - ) - - def test_encode_event_id(self): - self.assertEquals("A@B", encode_event_id("A", "B")) - - def test_codec_event_id(self): - event_id = "aa@bb.com" - - self.assertEquals( - event_id, - encode_event_id(*decode_event_id(event_id, None)) - ) - - pdu_id = ("aa", "bb.com") - - self.assertEquals( - pdu_id, - decode_event_id(encode_event_id(*pdu_id), None) - ) - - def test_event_from_pdu(self): - pdu = Pdu( - pdu_id="foo", - context="rooooom", - pdu_type="m.room.message", - origin="bar.com", - origin_server_ts=12345, - depth=5, - prev_pdus=[("alice", "bob.com")], - is_state=False, - content={"msgtype": u"test"}, - ) - - event = self.codec.event_from_pdu(pdu) - - self.assertEquals("foo@bar.com", event.event_id) - self.assertEquals(pdu.context, event.room_id) - self.assertEquals(pdu.is_state, event.is_state) - self.assertEquals(pdu.depth, event.depth) - self.assertEquals(pdu.prev_pdus, event.prev_pdus) - self.assertEquals(pdu.content, event.content) - - def test_pdu_from_event(self): - event = self.event_factory.create_event( - etype="m.room.message", - event_id="gargh_id", - room_id="rooom", - user_id="sender", - content={"msgtype": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals(event.event_id, pdu.pdu_id) - self.assertEquals(self.hs.hostname, pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - - event = self.event_factory.create_event( - etype="m.room.message", - event_id="gargh_id@bob.com", - room_id="rooom", - user_id="sender", - content={"msgtype": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals("gargh_id", pdu.pdu_id) - self.assertEquals("bob.com", pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - - def test_event_from_state_pdu(self): - pdu = Pdu( - pdu_id="foo", - context="rooooom", - pdu_type="m.room.topic", - origin="bar.com", - origin_server_ts=12345, - depth=5, - prev_pdus=[("alice", "bob.com")], - is_state=True, - content={"topic": u"test"}, - state_key="", - ) - - event = self.codec.event_from_pdu(pdu) - - self.assertEquals("foo@bar.com", event.event_id) - self.assertEquals(pdu.context, event.room_id) - self.assertEquals(pdu.is_state, event.is_state) - self.assertEquals(pdu.depth, event.depth) - self.assertEquals(pdu.prev_pdus, event.prev_pdus) - self.assertEquals(pdu.content, event.content) - self.assertEquals(pdu.state_key, event.state_key) - - def test_pdu_from_state_event(self): - event = self.event_factory.create_event( - etype="m.room.topic", - event_id="gargh_id", - room_id="rooom", - user_id="sender", - content={"topic": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals(event.event_id, pdu.pdu_id) - self.assertEquals(self.hs.hostname, pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - self.assertEquals(event.state_key, pdu.state_key) From dd3711bdbd2a439208a9e22b7b15f0c3676e611f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:33:52 +0000 Subject: [PATCH 112/265] Fix tests.handlers.test_directory --- tests/handlers/test_directory.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index e10a49a8a..16c446498 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -21,9 +21,8 @@ from mock import Mock from synapse.server import HomeServer from synapse.handlers.directory import DirectoryHandler -from synapse.storage.directory import RoomAliasMapping -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey class DirectoryHandlers(object): @@ -48,11 +47,15 @@ class DirectoryTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=db_pool, http_client=None, resource_for_federation=Mock(), replication_layer=self.mock_federation, + config=self.mock_config, ) hs.handlers = DirectoryHandlers(hs) From 7f4c7fe4e823eeacaa5b8c83201d504de53beefe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:35:38 +0000 Subject: [PATCH 113/265] PEP8 --- tests/handlers/test_directory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 16c446498..8e164e4be 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -40,6 +40,7 @@ class DirectoryTestCase(unittest.TestCase): ]) self.query_handlers = {} + def register_query_handler(query_type, handler): self.query_handlers[query_type] = handler self.mock_federation.register_query_handler = register_query_handler @@ -50,7 +51,8 @@ class DirectoryTestCase(unittest.TestCase): self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] - hs = HomeServer("test", + hs = HomeServer( + "test", db_pool=db_pool, http_client=None, resource_for_federation=Mock(), From 3068210a93e6240dd35445f272ec36edd188de83 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 4 Nov 2014 16:45:33 +0000 Subject: [PATCH 114/265] SYN-112: Get pynacl from github instead of PyPI --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0cb56c400..f5976cd76 100755 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ setup( ], dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", + "https://github.com/pyca/pynacl/tarball/52dbe2dc33f1#egg=pynacl-0.3.0", ], setup_requires=[ "setuptools_trial", From da4a09f977181ae222438dc75f5717cbebc6fe86 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:51:23 +0000 Subject: [PATCH 115/265] Don't bother locking --- synapse/handlers/federation.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 8848656b1..06a2dabae 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -132,12 +132,11 @@ class FederationHandler(BaseHandler): # TODO: Implement something in federation that allows us to # respond to PDU. - with (yield self.room_lock.lock(event.room_id)): - yield self.store.persist_event( - event, - backfilled, - is_new_state=is_new_state - ) + yield self.store.persist_event( + event, + backfilled, + is_new_state=is_new_state + ) room = yield self.store.get_room(event.room_id) From b0554682ed0c037f8726f3e37dba2113b5e8764f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:51:59 +0000 Subject: [PATCH 116/265] Fix federation handler tests. I've removed the invite/join dance ones as they are completely out of date. --- tests/handlers/test_federation.py | 84 ++++++++++--------------------- 1 file changed, 27 insertions(+), 57 deletions(-) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 219b2c4c5..63a9990f7 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -17,16 +17,15 @@ from twisted.internet import defer from tests import unittest from synapse.api.events.room import ( - InviteJoinEvent, MessageEvent, RoomMemberEvent + MessageEvent, ) -from synapse.api.constants import Membership from synapse.handlers.federation import FederationHandler from synapse.server import HomeServer from synapse.federation.units import Pdu from mock import NonCallableMock, ANY -from ..utils import get_mock_call_args, MockKey +from ..utils import MockKey class FederationTestCase(unittest.TestCase): @@ -36,6 +35,14 @@ class FederationTestCase(unittest.TestCase): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + self.state_handler = NonCallableMock(spec_set=[ + "annotate_state_groups", + ]) + + self.auth = NonCallableMock(spec_set=[ + "check", + ]) + self.hostname = "test" hs = HomeServer( self.hostname, @@ -53,6 +60,8 @@ class FederationTestCase(unittest.TestCase): "federation_handler", ]), config=self.mock_config, + auth=self.auth, + state_handler=self.state_handler, ) self.datastore = hs.get_datastore() @@ -65,74 +74,35 @@ class FederationTestCase(unittest.TestCase): @defer.inlineCallbacks def test_msg(self): pdu = Pdu( - pdu_type=MessageEvent.TYPE, - context="foo", + type=MessageEvent.TYPE, + room_id="foo", content={"msgtype": u"fooo"}, origin_server_ts=0, - pdu_id="a", + event_id="$a:b", origin="b", ) - store_id = "ASD" - self.datastore.persist_event.return_value = defer.succeed(store_id) + self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) + self.state_handler.annotate_state_groups.return_value = ( + defer.succeed(False) + ) + yield self.handlers.federation_handler.on_receive_pdu(pdu, False) self.datastore.persist_event.assert_called_once_with( ANY, False, is_new_state=False ) - self.notifier.on_new_room_event.assert_called_once_with(ANY, extra_users=[]) - @defer.inlineCallbacks - def test_invite_join_target_this(self): - room_id = "foo" - user_id = "@bob:red" - - pdu = Pdu( - pdu_type=InviteJoinEvent.TYPE, - user_id=user_id, - target_host=self.hostname, - context=room_id, - content={}, - origin_server_ts=0, - pdu_id="a", - origin="b", + self.state_handler.annotate_state_groups.assert_called_once_with( + ANY, + old_state=None, ) - yield self.handlers.federation_handler.on_receive_pdu(pdu, False) + self.auth.check.assert_called_once_with(ANY, None, raises=True) - mem_handler = self.handlers.room_member_handler - self.assertEquals(1, mem_handler.change_membership.call_count) - call_args = get_mock_call_args( - lambda event, do_auth: None, - mem_handler.change_membership + self.notifier.on_new_room_event.assert_called_once_with( + ANY, + extra_users=[] ) - self.assertEquals(False, call_args["do_auth"]) - - new_event = call_args["event"] - self.assertEquals(RoomMemberEvent.TYPE, new_event.type) - self.assertEquals(room_id, new_event.room_id) - self.assertEquals(user_id, new_event.state_key) - self.assertEquals(Membership.JOIN, new_event.membership) - - @defer.inlineCallbacks - def test_invite_join_target_other(self): - room_id = "foo" - user_id = "@bob:red" - - pdu = Pdu( - pdu_type=InviteJoinEvent.TYPE, - user_id=user_id, - state_key="@red:not%s" % self.hostname, - context=room_id, - content={}, - origin_server_ts=0, - pdu_id="a", - origin="b", - ) - - yield self.handlers.federation_handler.on_receive_pdu(pdu, False) - - mem_handler = self.handlers.room_member_handler - self.assertEquals(0, mem_handler.change_membership.call_count) From dfb3d21a6d2560668a637bba149bf34c7d413125 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 17:12:39 +0000 Subject: [PATCH 117/265] Fix room handler tests --- synapse/handlers/room.py | 1 - tests/handlers/test_room.py | 146 +++++------------------------------- 2 files changed, 19 insertions(+), 128 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index ffc0892f1..f176ad39b 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -408,7 +408,6 @@ class RoomMemberHandler(BaseHandler): defer.returnValue({}) return - yield self.state_handler.handle_new_event(event, snapshot) yield self._do_local_membership_update( event, membership=event.content["membership"], diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index c88d1c884..57a8a1b13 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -18,7 +18,7 @@ from twisted.internet import defer from tests import unittest from synapse.api.events.room import ( - InviteJoinEvent, RoomMemberEvent, RoomConfigEvent + RoomMemberEvent, ) from synapse.api.constants import Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler @@ -34,6 +34,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def setUp(self): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + self.hostname = "red" hs = HomeServer( self.hostname, @@ -58,7 +59,10 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "federation_handler", ]), auth=NonCallableMock(spec_set=["check"]), - state_handler=NonCallableMock(spec_set=["handle_new_event"]), + state_handler=NonCallableMock(spec_set=[ + "handle_new_event", + "annotate_state_groups", + ]), config=self.mock_config, ) @@ -114,6 +118,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) + self.datastore.get_room_member.return_value = defer.succeed(None) + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -202,133 +208,16 @@ class RoomMemberHandlerTestCase(unittest.TestCase): join_signal_observer.assert_called_with( user=user, room_id=room_id) - @defer.inlineCallbacks - def STALE_test_invite_join(self): - room_id = "foo" - user_id = "@bob:red" - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, - user_id=user_id, - target_user_id=target_user_id, - room_id=room_id, - membership=Membership.JOIN, - content=content, - ) - - joined = ["red", "blue", "green"] - - self.state_handler.handle_new_event.return_value = defer.succeed(True) - self.datastore.get_joined_hosts_for_room.return_value = ( - defer.succeed(joined) - ) - - store_id = "store_id_fooo" - self.datastore.store_room_member.return_value = defer.succeed(store_id) - self.datastore.get_room.return_value = defer.succeed(None) - - prev_state = NonCallableMock(name="prev_state") - prev_state.membership = Membership.INVITE - prev_state.sender = "@foo:blue" - self.datastore.get_room_member.return_value = defer.succeed(prev_state) - - # Actual invocation - yield self.room_member_handler.change_membership(event) - - self.datastore.get_room_member.assert_called_once_with( - target_user_id, room_id - ) - - self.assertTrue(self.federation.handle_new_event.called) - args = self.federation.handle_new_event.call_args[0] - invite_join_event = args[0] - - self.assertTrue(InviteJoinEvent.TYPE, invite_join_event.TYPE) - self.assertTrue("blue", invite_join_event.target_host) - self.assertTrue(room_id, invite_join_event.room_id) - self.assertTrue(user_id, invite_join_event.user_id) - self.assertFalse(hasattr(invite_join_event, "state_key")) - - self.assertEquals( - set(["blue"]), - set(invite_join_event.destinations) - ) - - self.federation.get_state_for_room.assert_called_once_with( - "blue", room_id - ) - - self.assertFalse(self.datastore.store_room_member.called) - - self.assertFalse(self.notifier.on_new_room_event.called) - self.assertFalse(self.state_handler.handle_new_event.called) - - @defer.inlineCallbacks - def STALE_test_invite_join_public(self): - room_id = "#foo:blue" - user_id = "@bob:red" - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, - user_id=user_id, - target_user_id=target_user_id, - room_id=room_id, - membership=Membership.JOIN, - content=content, - ) - - joined = ["red", "blue", "green"] - - self.state_handler.handle_new_event.return_value = defer.succeed(True) - self.datastore.get_joined_hosts_for_room.return_value = ( - defer.succeed(joined) - ) - - store_id = "store_id_fooo" - self.datastore.store_room_member.return_value = defer.succeed(store_id) - self.datastore.get_room.return_value = defer.succeed(None) - - prev_state = NonCallableMock(name="prev_state") - prev_state.membership = Membership.INVITE - prev_state.sender = "@foo:blue" - self.datastore.get_room_member.return_value = defer.succeed(prev_state) - - # Actual invocation - yield self.room_member_handler.change_membership(event) - - self.assertTrue(self.federation.handle_new_event.called) - args = self.federation.handle_new_event.call_args[0] - invite_join_event = args[0] - - self.assertTrue(InviteJoinEvent.TYPE, invite_join_event.TYPE) - self.assertTrue("blue", invite_join_event.target_host) - self.assertTrue("foo", invite_join_event.room_id) - self.assertTrue(user_id, invite_join_event.user_id) - self.assertFalse(hasattr(invite_join_event, "state_key")) - - self.assertEquals( - set(["blue"]), - set(invite_join_event.destinations) - ) - - self.federation.get_state_for_room.assert_called_once_with( - "blue", "foo" - ) - - self.assertFalse(self.datastore.store_room_member.called) - - self.assertFalse(self.notifier.on_new_room_event.called) - self.assertFalse(self.state_handler.handle_new_event.called) - class RoomCreationTest(unittest.TestCase): def setUp(self): self.hostname = "red" + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + hs = HomeServer( self.hostname, db_pool=None, @@ -346,11 +235,14 @@ class RoomCreationTest(unittest.TestCase): "federation_handler", ]), auth=NonCallableMock(spec_set=["check"]), - state_handler=NonCallableMock(spec_set=["handle_new_event"]), + state_handler=NonCallableMock(spec_set=[ + "handle_new_event", + "annotate_state_groups", + ]), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.federation = NonCallableMock(spec_set=[ @@ -400,6 +292,6 @@ class RoomCreationTest(unittest.TestCase): self.assertEquals(user_id, join_event.user_id) self.assertEquals(user_id, join_event.state_key) - self.assertTrue(self.state_handler.handle_new_event.called) + self.assertTrue(self.state_handler.annotate_state_groups.called) self.assertTrue(self.federation.handle_new_event.called) From 4a5e95511e31aa1836259e94d501a9cfe359d466 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 17:13:21 +0000 Subject: [PATCH 118/265] PEP8 --- tests/handlers/test_room.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 57a8a1b13..f0b77a249 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -171,7 +171,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_joined_hosts_for_room.side_effect = get_joined - store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) self.datastore.get_room.return_value = defer.succeed(1) # Not None. @@ -203,10 +202,12 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[user]) + event, extra_users=[user] + ) join_signal_observer.assert_called_with( - user=user, room_id=room_id) + user=user, room_id=room_id + ) class RoomCreationTest(unittest.TestCase): @@ -217,7 +218,6 @@ class RoomCreationTest(unittest.TestCase): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] - hs = HomeServer( self.hostname, db_pool=None, From a70765ed90196a418023d080dd6ba313c6305633 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 17:19:49 +0000 Subject: [PATCH 119/265] Add matrix-service unit tests. Update angular-mocks. --- syweb/webclient/js/angular-mocks.js | 310 +++++++++++------- .../test/unit/matrix-service.spec.js | 64 ++++ 2 files changed, 262 insertions(+), 112 deletions(-) create mode 100644 syweb/webclient/test/unit/matrix-service.spec.js diff --git a/syweb/webclient/js/angular-mocks.js b/syweb/webclient/js/angular-mocks.js index 48c0b5dec..24bbcd413 100755 --- a/syweb/webclient/js/angular-mocks.js +++ b/syweb/webclient/js/angular-mocks.js @@ -1,10 +1,3 @@ -/** - * @license AngularJS v1.2.22 - * (c) 2010-2014 Google, Inc. http://angularjs.org - * License: MIT - */ -(function(window, angular, undefined) { - 'use strict'; /** @@ -63,6 +56,8 @@ angular.mock.$Browser = function() { return listener; }; + self.$$checkUrlChange = angular.noop; + self.cookieHash = {}; self.lastCookieHash = {}; self.deferredFns = []; @@ -125,7 +120,7 @@ angular.mock.$Browser = function() { } }; - self.$$baseHref = ''; + self.$$baseHref = '/'; self.baseHref = function() { return this.$$baseHref; }; @@ -774,13 +769,22 @@ angular.mock.animate = angular.module('ngAnimateMock', ['ng']) }; }); - $provide.decorator('$animate', function($delegate, $$asyncCallback) { + $provide.decorator('$animate', ['$delegate', '$$asyncCallback', '$timeout', '$browser', + function($delegate, $$asyncCallback, $timeout, $browser) { var animate = { queue : [], + cancel : $delegate.cancel, enabled : $delegate.enabled, - triggerCallbacks : function() { + triggerCallbackEvents : function() { $$asyncCallback.flush(); }, + triggerCallbackPromise : function() { + $timeout.flush(0); + }, + triggerCallbacks : function() { + this.triggerCallbackEvents(); + this.triggerCallbackPromise(); + }, triggerReflow : function() { angular.forEach(reflowQueue, function(fn) { fn(); @@ -797,12 +801,12 @@ angular.mock.animate = angular.module('ngAnimateMock', ['ng']) element : arguments[0], args : arguments }); - $delegate[method].apply($delegate, arguments); + return $delegate[method].apply($delegate, arguments); }; }); return animate; - }); + }]); }]); @@ -888,7 +892,7 @@ angular.mock.dump = function(object) { * development please see {@link ngMockE2E.$httpBackend e2e $httpBackend mock}. * * During unit testing, we want our unit tests to run quickly and have no external dependencies so - * we don’t want to send [XHR](https://developer.mozilla.org/en/xmlhttprequest) or + * we don’t want to send [XHR](https://developer.mozilla.org/en/xmlhttprequest) or * [JSONP](http://en.wikipedia.org/wiki/JSONP) requests to a real server. All we really need is * to verify whether a certain request has been sent or not, or alternatively just let the * application make requests, respond with pre-trained responses and assert that the end result is @@ -1007,13 +1011,14 @@ angular.mock.dump = function(object) { ```js // testing controller describe('MyController', function() { - var $httpBackend, $rootScope, createController; + var $httpBackend, $rootScope, createController, authRequestHandler; beforeEach(inject(function($injector) { // Set up the mock http service responses $httpBackend = $injector.get('$httpBackend'); // backend definition common for all tests - $httpBackend.when('GET', '/auth.py').respond({userId: 'userX'}, {'A-Token': 'xxx'}); + authRequestHandler = $httpBackend.when('GET', '/auth.py') + .respond({userId: 'userX'}, {'A-Token': 'xxx'}); // Get hold of a scope (i.e. the root scope) $rootScope = $injector.get('$rootScope'); @@ -1039,11 +1044,23 @@ angular.mock.dump = function(object) { }); + it('should fail authentication', function() { + + // Notice how you can change the response even after it was set + authRequestHandler.respond(401, ''); + + $httpBackend.expectGET('/auth.py'); + var controller = createController(); + $httpBackend.flush(); + expect($rootScope.status).toBe('Failed...'); + }); + + it('should send msg to server', function() { var controller = createController(); $httpBackend.flush(); - // now you don’t care about the authentication, but + // now you don’t care about the authentication, but // the controller will still send the request and // $httpBackend will respond without you having to // specify the expectation and response for this request @@ -1186,32 +1203,39 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * Creates a new backend definition. * * @param {string} method HTTP method. - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp|function(string))=} data HTTP request body or function that receives * data string and returns true if the data is as expected. * @param {(Object|function(Object))=} headers HTTP headers or function that receives http header * object and returns true if the headers match the current definition. * @returns {requestHandler} Returns an object with `respond` method that controls how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. * - * - respond – + * - respond – * `{function([status,] data[, headers, statusText]) * | function(function(method, url, data, headers)}` - * – The respond method takes a set of static data to be returned or a function that can + * – The respond method takes a set of static data to be returned or a function that can * return an array containing response status (number), response data (string), response - * headers (Object), and the text for the status (string). + * headers (Object), and the text for the status (string). The respond method returns the + * `requestHandler` object for possible overrides. */ $httpBackend.when = function(method, url, data, headers) { var definition = new MockHttpExpectation(method, url, data, headers), chain = { respond: function(status, data, headers, statusText) { + definition.passThrough = undefined; definition.response = createResponse(status, data, headers, statusText); + return chain; } }; if ($browser) { chain.passThrough = function() { + definition.response = undefined; definition.passThrough = true; + return chain; }; } @@ -1225,10 +1249,12 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new backend definition for GET requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1237,10 +1263,12 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new backend definition for HEAD requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1249,10 +1277,12 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new backend definition for DELETE requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1261,12 +1291,14 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new backend definition for POST requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp|function(string))=} data HTTP request body or function that receives * data string and returns true if the data is as expected. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1275,12 +1307,14 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new backend definition for PUT requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp|function(string))=} data HTTP request body or function that receives * data string and returns true if the data is as expected. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1289,9 +1323,11 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new backend definition for JSONP requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ createShortMethods('when'); @@ -1303,30 +1339,36 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * Creates a new request expectation. * * @param {string} method HTTP method. - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp|function(string)|Object)=} data HTTP request body or function that * receives data string and returns true if the data is as expected, or Object if request body * is in JSON format. * @param {(Object|function(Object))=} headers HTTP headers or function that receives http header * object and returns true if the headers match the current expectation. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. * - * - respond – + * - respond – * `{function([status,] data[, headers, statusText]) * | function(function(method, url, data, headers)}` - * – The respond method takes a set of static data to be returned or a function that can + * – The respond method takes a set of static data to be returned or a function that can * return an array containing response status (number), response data (string), response - * headers (Object), and the text for the status (string). + * headers (Object), and the text for the status (string). The respond method returns the + * `requestHandler` object for possible overrides. */ $httpBackend.expect = function(method, url, data, headers) { - var expectation = new MockHttpExpectation(method, url, data, headers); + var expectation = new MockHttpExpectation(method, url, data, headers), + chain = { + respond: function (status, data, headers, statusText) { + expectation.response = createResponse(status, data, headers, statusText); + return chain; + } + }; + expectations.push(expectation); - return { - respond: function (status, data, headers, statusText) { - expectation.response = createResponse(status, data, headers, statusText); - } - }; + return chain; }; @@ -1336,10 +1378,12 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new request expectation for GET requests. For more info see `expect()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {Object=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. See #expect for more info. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. See #expect for more info. */ /** @@ -1348,10 +1392,12 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new request expectation for HEAD requests. For more info see `expect()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {Object=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1360,10 +1406,12 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new request expectation for DELETE requests. For more info see `expect()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {Object=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1372,13 +1420,15 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new request expectation for POST requests. For more info see `expect()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp|function(string)|Object)=} data HTTP request body or function that * receives data string and returns true if the data is as expected, or Object if request body * is in JSON format. * @param {Object=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1387,13 +1437,15 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new request expectation for PUT requests. For more info see `expect()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp|function(string)|Object)=} data HTTP request body or function that * receives data string and returns true if the data is as expected, or Object if request body * is in JSON format. * @param {Object=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1402,13 +1454,15 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new request expectation for PATCH requests. For more info see `expect()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp|function(string)|Object)=} data HTTP request body or function that * receives data string and returns true if the data is as expected, or Object if request body * is in JSON format. * @param {Object=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ /** @@ -1417,9 +1471,11 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * @description * Creates a new request expectation for JSONP requests. For more info see `expect()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @returns {requestHandler} Returns an object with `respond` method that control how a matched - * request is handled. + * request is handled. You can save this object for later use and invoke `respond` again in + * order to change how a matched request is handled. */ createShortMethods('expect'); @@ -1434,11 +1490,11 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * all pending requests will be flushed. If there are no pending requests when the flush method * is called an exception is thrown (as this typically a sign of programming error). */ - $httpBackend.flush = function(count) { - $rootScope.$digest(); + $httpBackend.flush = function(count, digest) { + if (digest !== false) $rootScope.$digest(); if (!responses.length) throw new Error('No pending request to flush !'); - if (angular.isDefined(count)) { + if (angular.isDefined(count) && count !== null) { while (count--) { if (!responses.length) throw new Error('No more pending request to flush !'); responses.shift()(); @@ -1448,7 +1504,7 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { responses.shift()(); } } - $httpBackend.verifyNoOutstandingExpectation(); + $httpBackend.verifyNoOutstandingExpectation(digest); }; @@ -1466,8 +1522,8 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { * afterEach($httpBackend.verifyNoOutstandingExpectation); * ``` */ - $httpBackend.verifyNoOutstandingExpectation = function() { - $rootScope.$digest(); + $httpBackend.verifyNoOutstandingExpectation = function(digest) { + if (digest !== false) $rootScope.$digest(); if (expectations.length) { throw new Error('Unsatisfied requests: ' + expectations.join(', ')); } @@ -1511,7 +1567,7 @@ function createHttpBackendMock($rootScope, $delegate, $browser) { function createShortMethods(prefix) { - angular.forEach(['GET', 'DELETE', 'JSONP'], function(method) { + angular.forEach(['GET', 'DELETE', 'JSONP', 'HEAD'], function(method) { $httpBackend[prefix + method] = function(url, headers) { return $httpBackend[prefix](method, url, undefined, headers); }; @@ -1541,6 +1597,7 @@ function MockHttpExpectation(method, url, data, headers) { this.matchUrl = function(u) { if (!url) return true; if (angular.isFunction(url.test)) return url.test(u); + if (angular.isFunction(url)) return url(u); return url == u; }; @@ -1627,7 +1684,7 @@ function MockXhr() { * that adds a "flush" and "verifyNoPendingTasks" methods. */ -angular.mock.$TimeoutDecorator = function($delegate, $browser) { +angular.mock.$TimeoutDecorator = ['$delegate', '$browser', function ($delegate, $browser) { /** * @ngdoc method @@ -1666,9 +1723,9 @@ angular.mock.$TimeoutDecorator = function($delegate, $browser) { } return $delegate; -}; +}]; -angular.mock.$RAFDecorator = function($delegate) { +angular.mock.$RAFDecorator = ['$delegate', function($delegate) { var queue = []; var rafFn = function(fn) { var index = queue.length; @@ -1694,9 +1751,9 @@ angular.mock.$RAFDecorator = function($delegate) { }; return rafFn; -}; +}]; -angular.mock.$AsyncCallbackDecorator = function($delegate) { +angular.mock.$AsyncCallbackDecorator = ['$delegate', function($delegate) { var callbacks = []; var addFn = function(fn) { callbacks.push(fn); @@ -1708,7 +1765,7 @@ angular.mock.$AsyncCallbackDecorator = function($delegate) { callbacks = []; }; return addFn; -}; +}]; /** * @@ -1822,22 +1879,25 @@ angular.module('ngMockE2E', ['ng']).config(['$provide', function($provide) { * Creates a new backend definition. * * @param {string} method HTTP method. - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp)=} data HTTP request body. * @param {(Object|function(Object))=} headers HTTP headers or function that receives http header * object and returns true if the headers match the current definition. * @returns {requestHandler} Returns an object with `respond` and `passThrough` methods that - * control how a matched request is handled. + * control how a matched request is handled. You can save this object for later use and invoke + * `respond` or `passThrough` again in order to change how a matched request is handled. * - * - respond – + * - respond – * `{function([status,] data[, headers, statusText]) * | function(function(method, url, data, headers)}` - * – The respond method takes a set of static data to be returned or a function that can return + * – The respond method takes a set of static data to be returned or a function that can return * an array containing response status (number), response data (string), response headers * (Object), and the text for the status (string). - * - passThrough – `{function()}` – Any request matching a backend definition with + * - passThrough – `{function()}` – Any request matching a backend definition with * `passThrough` handler will be passed through to the real backend (an XHR request will be made * to the server.) + * - Both methods return the `requestHandler` object for possible overrides. */ /** @@ -1847,10 +1907,12 @@ angular.module('ngMockE2E', ['ng']).config(['$provide', function($provide) { * @description * Creates a new backend definition for GET requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` and `passThrough` methods that - * control how a matched request is handled. + * control how a matched request is handled. You can save this object for later use and invoke + * `respond` or `passThrough` again in order to change how a matched request is handled. */ /** @@ -1860,10 +1922,12 @@ angular.module('ngMockE2E', ['ng']).config(['$provide', function($provide) { * @description * Creates a new backend definition for HEAD requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` and `passThrough` methods that - * control how a matched request is handled. + * control how a matched request is handled. You can save this object for later use and invoke + * `respond` or `passThrough` again in order to change how a matched request is handled. */ /** @@ -1873,10 +1937,12 @@ angular.module('ngMockE2E', ['ng']).config(['$provide', function($provide) { * @description * Creates a new backend definition for DELETE requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` and `passThrough` methods that - * control how a matched request is handled. + * control how a matched request is handled. You can save this object for later use and invoke + * `respond` or `passThrough` again in order to change how a matched request is handled. */ /** @@ -1886,11 +1952,13 @@ angular.module('ngMockE2E', ['ng']).config(['$provide', function($provide) { * @description * Creates a new backend definition for POST requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp)=} data HTTP request body. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` and `passThrough` methods that - * control how a matched request is handled. + * control how a matched request is handled. You can save this object for later use and invoke + * `respond` or `passThrough` again in order to change how a matched request is handled. */ /** @@ -1900,11 +1968,13 @@ angular.module('ngMockE2E', ['ng']).config(['$provide', function($provide) { * @description * Creates a new backend definition for PUT requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp)=} data HTTP request body. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` and `passThrough` methods that - * control how a matched request is handled. + * control how a matched request is handled. You can save this object for later use and invoke + * `respond` or `passThrough` again in order to change how a matched request is handled. */ /** @@ -1914,11 +1984,13 @@ angular.module('ngMockE2E', ['ng']).config(['$provide', function($provide) { * @description * Creates a new backend definition for PATCH requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @param {(string|RegExp)=} data HTTP request body. * @param {(Object|function(Object))=} headers HTTP headers. * @returns {requestHandler} Returns an object with `respond` and `passThrough` methods that - * control how a matched request is handled. + * control how a matched request is handled. You can save this object for later use and invoke + * `respond` or `passThrough` again in order to change how a matched request is handled. */ /** @@ -1928,30 +2000,17 @@ angular.module('ngMockE2E', ['ng']).config(['$provide', function($provide) { * @description * Creates a new backend definition for JSONP requests. For more info see `when()`. * - * @param {string|RegExp} url HTTP url. + * @param {string|RegExp|function(string)} url HTTP url or function that receives the url + * and returns true if the url match the current definition. * @returns {requestHandler} Returns an object with `respond` and `passThrough` methods that - * control how a matched request is handled. + * control how a matched request is handled. You can save this object for later use and invoke + * `respond` or `passThrough` again in order to change how a matched request is handled. */ angular.mock.e2e = {}; angular.mock.e2e.$httpBackendDecorator = ['$rootScope', '$delegate', '$browser', createHttpBackendMock]; -angular.mock.clearDataCache = function() { - var key, - cache = angular.element.cache; - - for(key in cache) { - if (Object.prototype.hasOwnProperty.call(cache,key)) { - var handle = cache[key].handle; - - handle && angular.element(handle.elem).off(); - delete cache[key]; - } - } -}; - - if(window.jasmine || window.mocha) { var currentSpec = null, @@ -1982,8 +2041,6 @@ if(window.jasmine || window.mocha) { injector.get('$browser').pollFns.length = 0; } - angular.mock.clearDataCache(); - // clean up jquery's fragment cache angular.forEach(angular.element.fragments, function(val, key) { delete angular.element.fragments[key]; @@ -2003,6 +2060,7 @@ if(window.jasmine || window.mocha) { * @description * * *NOTE*: This function is also published on window for easy access.
+ * *NOTE*: This function is declared ONLY WHEN running tests with jasmine or mocha * * This function registers a module configuration code. It collects the configuration information * which will be used when the injector is created by {@link angular.mock.inject inject}. @@ -2045,6 +2103,7 @@ if(window.jasmine || window.mocha) { * @description * * *NOTE*: This function is also published on window for easy access.
+ * *NOTE*: This function is declared ONLY WHEN running tests with jasmine or mocha * * The inject function wraps a function into an injectable function. The inject() creates new * instance of {@link auto.$injector $injector} per test, which is then used for @@ -2144,14 +2203,28 @@ if(window.jasmine || window.mocha) { ///////////////////// function workFn() { var modules = currentSpec.$modules || []; - + var strictDi = !!currentSpec.$injectorStrict; modules.unshift('ngMock'); modules.unshift('ng'); var injector = currentSpec.$injector; if (!injector) { - injector = currentSpec.$injector = angular.injector(modules); + if (strictDi) { + // If strictDi is enabled, annotate the providerInjector blocks + angular.forEach(modules, function(moduleFn) { + if (typeof moduleFn === "function") { + angular.injector.$$annotate(moduleFn); + } + }); + } + injector = currentSpec.$injector = angular.injector(modules, strictDi); + currentSpec.$injectorStrict = strictDi; } for(var i = 0, ii = blockFns.length; i < ii; i++) { + if (currentSpec.$injectorStrict) { + // If the injector is strict / strictDi, and the spec wants to inject using automatic + // annotation, then annotate the function here. + injector.annotate(blockFns[i]); + } try { /* jshint -W040 *//* Jasmine explicitly provides a `this` object when calling functions */ injector.invoke(blockFns[i] || angular.noop, this); @@ -2167,7 +2240,20 @@ if(window.jasmine || window.mocha) { } } }; + + + angular.mock.inject.strictDi = function(value) { + value = arguments.length ? !!value : true; + return isSpecRunning() ? workFn() : workFn; + + function workFn() { + if (value !== currentSpec.$injectorStrict) { + if (currentSpec.$injector) { + throw new Error('Injector already created, can not modify strict annotations'); + } else { + currentSpec.$injectorStrict = value; + } + } + } + }; } - - -})(window, window.angular); \ No newline at end of file diff --git a/syweb/webclient/test/unit/matrix-service.spec.js b/syweb/webclient/test/unit/matrix-service.spec.js new file mode 100644 index 000000000..3c5163e47 --- /dev/null +++ b/syweb/webclient/test/unit/matrix-service.spec.js @@ -0,0 +1,64 @@ +describe('MatrixService', function() { + var scope, httpBackend, createController; + var BASE = "http://example.com"; + var PREFIX = "/_matrix/client/api/v1"; + var URL = BASE + PREFIX; + var roomId = "!wejigf387t34:matrix.org"; + + beforeEach(module('matrixService')); + + beforeEach(inject(function($rootScope, $httpBackend, $controller) { + httpBackend = $httpBackend; + scope = $rootScope; + })); + + afterEach(function() { + httpBackend.verifyNoOutstandingExpectation(); + httpBackend.verifyNoOutstandingRequest(); + }); + + it('should be able to GET /rooms/$roomid/state', inject(function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + matrixService.roomState(roomId).then(function(response) { + expect(response.data).toEqual([]); + }); + + httpBackend.expect('GET', + URL + "/rooms/" + roomId + "/state?access_token=foobar") + .respond([]); + httpBackend.flush(); + })); + + it('should be able to POST /join', inject(function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + matrixService.joinAlias(roomId).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expect('POST', + URL + "/join/" + encodeURIComponent(roomId) + "?access_token=foobar") + .respond({}); + httpBackend.flush(); + })); + + it('should be able to POST /rooms/$roomid/join', inject(function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + matrixService.join(roomId).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expect('POST', + URL + "/rooms/" + encodeURIComponent(roomId) + "/join?access_token=foobar") + .respond({}); + httpBackend.flush(); + })); +}); From 4facbe02fbe53aafebfd68d88ae09c9ae77f3cd3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 4 Nov 2014 17:48:47 +0000 Subject: [PATCH 120/265] URL encoding bugfix and add more tests. --- .../components/matrix/matrix-service.js | 2 +- .../test/unit/matrix-service.spec.js | 46 ++++++++++++++++--- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/syweb/webclient/components/matrix/matrix-service.js b/syweb/webclient/components/matrix/matrix-service.js index fedfb8910..5b63fb4a3 100644 --- a/syweb/webclient/components/matrix/matrix-service.js +++ b/syweb/webclient/components/matrix/matrix-service.js @@ -267,7 +267,7 @@ angular.module('matrixService', []) // get room state for a specific room roomState: function(room_id) { - var path = "/rooms/" + room_id + "/state"; + var path = "/rooms/" + encodeURIComponent(room_id) + "/state"; return doRequest("GET", path); }, diff --git a/syweb/webclient/test/unit/matrix-service.spec.js b/syweb/webclient/test/unit/matrix-service.spec.js index 3c5163e47..29d2ca7be 100644 --- a/syweb/webclient/test/unit/matrix-service.spec.js +++ b/syweb/webclient/test/unit/matrix-service.spec.js @@ -1,5 +1,5 @@ describe('MatrixService', function() { - var scope, httpBackend, createController; + var scope, httpBackend; var BASE = "http://example.com"; var PREFIX = "/_matrix/client/api/v1"; var URL = BASE + PREFIX; @@ -7,7 +7,7 @@ describe('MatrixService', function() { beforeEach(module('matrixService')); - beforeEach(inject(function($rootScope, $httpBackend, $controller) { + beforeEach(inject(function($rootScope, $httpBackend) { httpBackend = $httpBackend; scope = $rootScope; })); @@ -17,6 +17,40 @@ describe('MatrixService', function() { httpBackend.verifyNoOutstandingRequest(); }); + it('should be able to POST /createRoom with an alias', inject(function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var alias = "flibble"; + matrixService.create(alias).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPOST(URL + "/createRoom?access_token=foobar", + { + room_alias_name: alias + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to GET /initialSync', inject(function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var limit = 15; + matrixService.initialSync(limit).then(function(response) { + expect(response.data).toEqual([]); + }); + + httpBackend.expectGET( + URL + "/initialSync?access_token=foobar&limit=15") + .respond([]); + httpBackend.flush(); + })); + it('should be able to GET /rooms/$roomid/state', inject(function(matrixService) { matrixService.setConfig({ access_token: "foobar", @@ -26,8 +60,8 @@ describe('MatrixService', function() { expect(response.data).toEqual([]); }); - httpBackend.expect('GET', - URL + "/rooms/" + roomId + "/state?access_token=foobar") + httpBackend.expectGET( + URL + "/rooms/" + encodeURIComponent(roomId) + "/state?access_token=foobar") .respond([]); httpBackend.flush(); })); @@ -41,7 +75,7 @@ describe('MatrixService', function() { expect(response.data).toEqual({}); }); - httpBackend.expect('POST', + httpBackend.expectPOST( URL + "/join/" + encodeURIComponent(roomId) + "?access_token=foobar") .respond({}); httpBackend.flush(); @@ -56,7 +90,7 @@ describe('MatrixService', function() { expect(response.data).toEqual({}); }); - httpBackend.expect('POST', + httpBackend.expectPOST( URL + "/rooms/" + encodeURIComponent(roomId) + "/join?access_token=foobar") .respond({}); httpBackend.flush(); From 96c001e6688617cc365f548a3152a32c647ebc59 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:07:54 +0000 Subject: [PATCH 121/265] Fix auth checks to all use the given old_event_state --- synapse/api/auth.py | 53 +++++++++++++++++++--------------- synapse/handlers/_base.py | 19 ++++++++---- synapse/handlers/federation.py | 6 ++-- synapse/state.py | 8 +++-- 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index c68426510..9eb0491c9 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -22,7 +22,7 @@ from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, InviteJoinEvent, - RoomCreateEvent, + RoomCreateEvent, RoomSendEventLevelEvent, RoomAddStateLevelEvent, ) from synapse.util.logutils import log_function @@ -37,8 +37,7 @@ class Auth(object): self.hs = hs self.store = hs.get_datastore() - @defer.inlineCallbacks - def check(self, event, snapshot, raises=False): + def check(self, event, raises=False): """ Checks if this event is correctly authed. Returns: @@ -52,17 +51,17 @@ class Auth(object): if event.old_state_events is None: # Oh, we don't know what the state of the room was, so we # are trusting that this is allowed (at least for now) - defer.returnValue(True) + return True if hasattr(event, "outlier") and event.outlier is True: # TODO (erikj): Auth for outliers is done differently. - defer.returnValue(True) + return True is_state = hasattr(event, "state_key") if event.type == RoomCreateEvent.TYPE: # FIXME - defer.returnValue(True) + return True if event.type == RoomMemberEvent.TYPE: self._can_replace_state(event) @@ -71,8 +70,7 @@ class Auth(object): logger.debug("Allowing! %s", event) else: logger.debug("Denying! %s", event) - defer.returnValue(allowed) - return + return allowed if not event.type == InviteJoinEvent.TYPE: self.check_event_sender_in_room(event) @@ -80,10 +78,10 @@ class Auth(object): if is_state: # TODO (erikj): This really only should be called for *new* # state - yield self._can_add_state(event) + self._can_add_state(event) self._can_replace_state(event) else: - yield self._can_send_event(event) + self._can_send_event(event) if event.type == RoomPowerLevelsEvent.TYPE: self._check_power_levels(event) @@ -91,9 +89,8 @@ class Auth(object): if event.type == RoomRedactionEvent.TYPE: self._check_redaction(event) - logger.debug("Allowing! %s", event) - defer.returnValue(True) + return True else: raise AuthError(500, "Unknown event: %s" % event) except AuthError as e: @@ -103,7 +100,7 @@ class Auth(object): if raises: raise e - defer.returnValue(False) + return False @defer.inlineCallbacks def check_joined_room(self, room_id, user_id): @@ -326,10 +323,15 @@ class Auth(object): def is_server_admin(self, user): return self.store.is_server_admin(user) - @defer.inlineCallbacks @log_function def _can_send_event(self, event): - send_level = yield self.store.get_send_event_level(event.room_id) + key = (RoomSendEventLevelEvent.TYPE, "", ) + send_level_event = event.old_state_events.get(key) + send_level = None + if send_level_event: + send_level = send_level_event.content.get(event.user_id) + if not send_level: + send_level = send_level_event.content.get("level", 0) if send_level: send_level = int(send_level) @@ -351,16 +353,21 @@ class Auth(object): 403, "You don't have permission to post to the room" ) - defer.returnValue(True) + return True - @defer.inlineCallbacks def _can_add_state(self, event): - add_level = yield self.store.get_add_state_level(event.room_id) + key = (RoomAddStateLevelEvent.TYPE, "", ) + add_level_event = event.old_state_events.get(key) + add_level = None + if add_level_event: + add_level = add_level_event.content.get(event.user_id) + if not add_level: + add_level = add_level_event.content.get("level", 0) - if not add_level: - defer.returnValue(True) - - add_level = int(add_level) + if add_level: + add_level = int(add_level) + else: + add_level = 0 user_level = self._get_power_level_from_event_state( event, @@ -374,7 +381,7 @@ class Auth(object): 403, "You don't have permission to add state to the room" ) - defer.returnValue(True) + return True def _can_replace_state(self, event): user_level = self._get_power_level_from_event_state( diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 28b64565a..509f7b550 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -20,6 +20,12 @@ from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures +import logging + + +logger = logging.getLogger(__name__) + + class BaseHandler(object): def __init__(self, hs): @@ -58,15 +64,18 @@ class BaseHandler(object): yield self.state_handler.annotate_state_groups(event) - yield add_hashes_and_signatures( + logger.debug("Signing event...") + + add_hashes_and_signatures( event, self.server_name, self.signing_key ) - if not suppress_auth: - yield self.auth.check(event, snapshot, raises=True) + logger.debug("Signed event.") - if hasattr(event, "state_key"): - yield self.state_handler.handle_new_event(event, snapshot) + if not suppress_auth: + logger.debug("Authing...") + self.auth.check(event, raises=True) + logger.debug("Authed") yield self.store.persist_event(event) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 06a2dabae..1464a6093 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -118,7 +118,7 @@ class FederationHandler(BaseHandler): logger.debug("Event: %s", event) try: - yield self.auth.check(event, None, raises=True) + self.auth.check(event, raises=True) except AuthError as e: raise FederationError( "ERROR", @@ -319,7 +319,7 @@ class FederationHandler(BaseHandler): snapshot.fill_out_prev_events(event) yield self.state_handler.annotate_state_groups(event) - yield self.auth.check(event, None, raises=True) + self.auth.check(event, raises=True) pdu = self.pdu_codec.pdu_from_event(event) @@ -333,7 +333,7 @@ class FederationHandler(BaseHandler): event.outlier = False is_new_state = yield self.state_handler.annotate_state_groups(event) - yield self.auth.check(event, None, raises=True) + self.auth.check(event, raises=True) # FIXME (erikj): All this is duplicated above :( diff --git a/synapse/state.py b/synapse/state.py index 9771883bc..32744e047 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -188,11 +188,15 @@ class StateHandler(object): consumeErrors=True ) - max_power = max([int(p) for p in new_powers]) + new_powers = [ + int(p) if p else 0 for p in new_powers + ] + + max_power = max(new_powers) curr_events = [ z[0] for z in zip(curr_events, new_powers) - if int(z[1]) == max_power + if z[1] == max_power ] if not curr_events: From 9e429239ab0c304cce4d2f076be5b30691fd93a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:10:36 +0000 Subject: [PATCH 122/265] Fix Federation test --- tests/handlers/test_federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 63a9990f7..a9d6b2bb1 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -100,7 +100,7 @@ class FederationTestCase(unittest.TestCase): old_state=None, ) - self.auth.check.assert_called_once_with(ANY, None, raises=True) + self.auth.check.assert_called_once_with(ANY, raises=True) self.notifier.on_new_room_event.assert_called_once_with( ANY, From 34c4614682b08fa89a9f94ded4ecaf882fdc4160 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:10:54 +0000 Subject: [PATCH 123/265] Fix presence test --- tests/handlers/test_presence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index fdc2e8de4..a6af648de 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -51,6 +51,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "content": content, } ], + "pdu_failures": [], } def _make_edu_json(origin, edu_type, content): From a2aafeb959075acd76374d2a7d01830f59930996 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 5 Nov 2014 11:11:36 +0000 Subject: [PATCH 124/265] Add a bunch more unit tests for matrixService. --- .../test/unit/matrix-service.spec.js | 197 +++++++++++++++++- 1 file changed, 191 insertions(+), 6 deletions(-) diff --git a/syweb/webclient/test/unit/matrix-service.spec.js b/syweb/webclient/test/unit/matrix-service.spec.js index 29d2ca7be..95a43057c 100644 --- a/syweb/webclient/test/unit/matrix-service.spec.js +++ b/syweb/webclient/test/unit/matrix-service.spec.js @@ -17,7 +17,8 @@ describe('MatrixService', function() { httpBackend.verifyNoOutstandingRequest(); }); - it('should be able to POST /createRoom with an alias', inject(function(matrixService) { + it('should be able to POST /createRoom with an alias', inject( + function(matrixService) { matrixService.setConfig({ access_token: "foobar", homeserver: "http://example.com" @@ -51,7 +52,8 @@ describe('MatrixService', function() { httpBackend.flush(); })); - it('should be able to GET /rooms/$roomid/state', inject(function(matrixService) { + it('should be able to GET /rooms/$roomid/state', inject( + function(matrixService) { matrixService.setConfig({ access_token: "foobar", homeserver: "http://example.com" @@ -61,7 +63,8 @@ describe('MatrixService', function() { }); httpBackend.expectGET( - URL + "/rooms/" + encodeURIComponent(roomId) + "/state?access_token=foobar") + URL + "/rooms/" + encodeURIComponent(roomId) + + "/state?access_token=foobar") .respond([]); httpBackend.flush(); })); @@ -76,12 +79,15 @@ describe('MatrixService', function() { }); httpBackend.expectPOST( - URL + "/join/" + encodeURIComponent(roomId) + "?access_token=foobar") + URL + "/join/" + encodeURIComponent(roomId) + + "?access_token=foobar", + {}) .respond({}); httpBackend.flush(); })); - it('should be able to POST /rooms/$roomid/join', inject(function(matrixService) { + it('should be able to POST /rooms/$roomid/join', inject( + function(matrixService) { matrixService.setConfig({ access_token: "foobar", homeserver: "http://example.com" @@ -91,7 +97,186 @@ describe('MatrixService', function() { }); httpBackend.expectPOST( - URL + "/rooms/" + encodeURIComponent(roomId) + "/join?access_token=foobar") + URL + "/rooms/" + encodeURIComponent(roomId) + + "/join?access_token=foobar", + {}) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to POST /rooms/$roomid/invite', inject( + function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var inviteUserId = "@user:example.com"; + matrixService.invite(roomId, inviteUserId).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPOST( + URL + "/rooms/" + encodeURIComponent(roomId) + + "/invite?access_token=foobar", + { + user_id: inviteUserId + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to POST /rooms/$roomid/leave', inject( + function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + matrixService.leave(roomId).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPOST( + URL + "/rooms/" + encodeURIComponent(roomId) + + "/leave?access_token=foobar", + {}) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to POST /rooms/$roomid/ban', inject( + function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var userId = "@example:example.com"; + var reason = "Because."; + matrixService.ban(roomId, userId, reason).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPOST( + URL + "/rooms/" + encodeURIComponent(roomId) + + "/ban?access_token=foobar", + { + user_id: userId, + reason: reason + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to GET /directory/room/$alias', inject( + function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var alias = "#test:example.com"; + var roomId = "!wefuhewfuiw:example.com"; + matrixService.resolveRoomAlias(alias).then(function(response) { + expect(response.data).toEqual({ + room_id: roomId + }); + }); + + httpBackend.expectGET( + URL + "/directory/room/" + encodeURIComponent(alias) + + "?access_token=foobar") + .respond({ + room_id: roomId + }); + httpBackend.flush(); + })); + + it('should be able to send m.room.name', inject(function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var roomId = "!fh38hfwfwef:example.com"; + var name = "Room Name"; + matrixService.setName(roomId, name).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPUT( + URL + "/rooms/" + encodeURIComponent(roomId) + + "/state/m.room.name?access_token=foobar", + { + name: name + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to send m.room.topic', inject(function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var roomId = "!fh38hfwfwef:example.com"; + var topic = "A room topic can go here."; + matrixService.setTopic(roomId, topic).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPUT( + URL + "/rooms/" + encodeURIComponent(roomId) + + "/state/m.room.topic?access_token=foobar", + { + topic: topic + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to send generic state events without a state key', inject( + function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var roomId = "!fh38hfwfwef:example.com"; + var eventType = "com.example.events.test"; + var content = { + testing: "1 2 3" + }; + matrixService.sendStateEvent(roomId, eventType, content).then( + function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPUT( + URL + "/rooms/" + encodeURIComponent(roomId) + "/state/" + + encodeURIComponent(eventType) + "?access_token=foobar", + content) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to send generic state events with a state key', inject( + function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var roomId = "!fh38hfwfwef:example.com"; + var eventType = "com.example.events.test"; + var content = { + testing: "1 2 3" + }; + var stateKey = "version1"; + matrixService.sendStateEvent(roomId, eventType, content, stateKey).then( + function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPUT( + URL + "/rooms/" + encodeURIComponent(roomId) + "/state/" + + encodeURIComponent(eventType) + "/" + encodeURIComponent(stateKey)+ + "?access_token=foobar", + content) .respond({}); httpBackend.flush(); })); From 416ab4ebf0cd12a14f9d6d98226ae52d7d86f3a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:12:47 +0000 Subject: [PATCH 125/265] Don't execute empty tests. Formatting. --- tests/rest/test_events.py | 45 ++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 362c7bc01..82f5d277e 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -25,9 +25,6 @@ import synapse.rest.room from synapse.server import HomeServer -# python imports -import json - from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase @@ -49,7 +46,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): def tearDown(self): pass - def test_long_poll(self): + def TODO_test_long_poll(self): # stream from 'end' key, send (self+other) message, expect message. # stream from 'END', send (self+other) message, expect message. @@ -64,7 +61,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): pass - def test_stream_forward(self): + def TODO_test_stream_forward(self): # stream from START, expect injected items # stream from 'start' key, expect same content @@ -80,14 +77,14 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): # returned as end key pass - def test_limits(self): + def TODO_test_limits(self): # stream from a key, expect limit_num items # stream from START, expect limit_num items pass - def test_range(self): + def TODO_test_range(self): # stream from key to key, expect X items # stream from key to END, expect X items @@ -97,7 +94,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): # stream from START to END, expect all items pass - def test_direction(self): + def TODO_test_direction(self): # stream from END to START and fwds, expect newest first # stream from END to START and bwds, expect oldest first @@ -116,7 +113,10 @@ class EventStreamPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - state_handler = Mock(spec=["handle_new_event"]) + state_handler = Mock(spec=[ + "handle_new_event", + "annotate_state_groups" + ]) state_handler.handle_new_event.return_value = True persistence_service = Mock(spec=["get_latest_pdus_in_context"]) @@ -151,6 +151,7 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() hs.get_clock().time_msec.return_value = 1000000 + hs.get_clock().time.return_value = 1000 synapse.rest.register.register_servlets(hs, self.mock_resource) synapse.rest.events.register_servlets(hs, self.mock_resource) @@ -175,12 +176,14 @@ class EventStreamPermissionsTestCase(RestTestCase): def test_stream_basic_permissions(self): # invalid token, expect 403 (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s" % ("invalid" + self.token)) + "/events?access_token=%s" % ("invalid" + self.token, ) + ) self.assertEquals(403, code, msg=str(response)) # valid token, expect content (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s&timeout=0" % (self.token)) + "/events?access_token=%s&timeout=0" % (self.token,) + ) self.assertEquals(200, code, msg=str(response)) self.assertTrue("chunk" in response) self.assertTrue("start" in response) @@ -188,15 +191,23 @@ class EventStreamPermissionsTestCase(RestTestCase): @defer.inlineCallbacks def test_stream_room_permissions(self): - room_id = yield self.create_room_as(self.other_user, - tok=self.other_token) + room_id = yield self.create_room_as( + self.other_user, + tok=self.other_token + ) yield self.send(room_id, tok=self.other_token) # invited to room (expect no content for room) - yield self.invite(room_id, src=self.other_user, targ=self.user_id, - tok=self.other_token) + yield self.invite( + room_id, + src=self.other_user, + targ=self.user_id, + tok=self.other_token + ) + (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s&timeout=0" % (self.token)) + "/events?access_token=%s&timeout=0" % (self.token,) + ) self.assertEquals(200, code, msg=str(response)) self.assertEquals(0, len(response["chunk"])) @@ -206,7 +217,7 @@ class EventStreamPermissionsTestCase(RestTestCase): # left to room (expect no content for room) - def test_stream_items(self): + def TODO_test_stream_items(self): # new user, no content # join room, expect 1 item (join) From 24e2da455783009df1664cf94edf7653b80391a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:13:01 +0000 Subject: [PATCH 126/265] Fix presencelike test --- tests/handlers/test_presencelike.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 047752ad6..532ecf0f2 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -21,7 +21,7 @@ from twisted.internet import defer from mock import Mock, call, ANY -from ..utils import MockClock +from ..utils import MockClock, MockKey from synapse.server import HomeServer from synapse.api.constants import PresenceState @@ -57,6 +57,9 @@ class PresenceAndProfileHandlers(object): class PresenceProfilelikeDataTestCase(unittest.TestCase): def setUp(self): + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", clock=MockClock(), db_pool=None, @@ -72,6 +75,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): resource_for_federation=Mock(), http_client=None, replication_layer=MockReplication(), + config=self.mock_config, ) hs.handlers = PresenceAndProfileHandlers(hs) From f48fce8bd37add27a43306eae255ca40b774a041 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:13:58 +0000 Subject: [PATCH 127/265] Fix profile tests --- tests/handlers/test_profile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 5dc9b456e..1660e7e92 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -24,7 +24,7 @@ from synapse.server import HomeServer from synapse.handlers.profile import ProfileHandler from synapse.api.constants import Membership -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey class ProfileHandlers(object): @@ -49,12 +49,16 @@ class ProfileTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=db_pool, http_client=None, handlers=None, resource_for_federation=Mock(), replication_layer=self.mock_federation, + config=self.mock_config, ) hs.handlers = ProfileHandlers(hs) From c670ce416bcbdeb929ce06dabcd4bce841ec815d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:15:15 +0000 Subject: [PATCH 128/265] Fix room tests --- tests/handlers/test_room.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index f0b77a249..146ed21b6 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -123,9 +123,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): # Actual invocation yield self.room_member_handler.change_membership(event) - self.state_handler.handle_new_event.assert_called_once_with( - event, self.snapshot, - ) self.federation.handle_new_event.assert_called_once_with( event, self.snapshot, ) @@ -186,9 +183,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): # Actual invocation yield self.room_member_handler.change_membership(event) - self.state_handler.handle_new_event.assert_called_once_with( - event, self.snapshot - ) self.federation.handle_new_event.assert_called_once_with( event, self.snapshot ) From 1616df2f610ea2cc7c1fa32b98f5a63004d978de Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:15:56 +0000 Subject: [PATCH 129/265] Fix typing tests --- tests/handlers/test_typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index f1d3b27f7..07acda5ee 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -40,6 +40,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "content": content, } ], + "pdu_failures": [], } From 9f6d1b10ad5a9098a8f72157875ce97fc44bc423 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 5 Nov 2014 11:21:55 +0000 Subject: [PATCH 130/265] Be sure to urlencode/decode event types correctly in both the web client and HS. --- synapse/rest/room.py | 2 +- syweb/webclient/components/matrix/matrix-service.js | 4 ++-- syweb/webclient/test/unit/matrix-service.spec.js | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index ec0ce78fd..d97babea0 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -148,7 +148,7 @@ class RoomStateEventRestServlet(RestServlet): content = _parse_json(request) event = self.event_factory.create_event( - etype=event_type, + etype=urllib.unquote(event_type), content=content, room_id=urllib.unquote(room_id), user_id=user.to_string(), diff --git a/syweb/webclient/components/matrix/matrix-service.js b/syweb/webclient/components/matrix/matrix-service.js index 5b63fb4a3..e1e5b88b3 100644 --- a/syweb/webclient/components/matrix/matrix-service.js +++ b/syweb/webclient/components/matrix/matrix-service.js @@ -375,9 +375,9 @@ angular.module('matrixService', []) sendStateEvent: function(room_id, eventType, content, state_key) { - var path = "/rooms/$room_id/state/"+eventType; + var path = "/rooms/$room_id/state/"+ encodeURIComponent(eventType); if (state_key !== undefined) { - path += "/" + state_key; + path += "/" + encodeURIComponent(state_key); } room_id = encodeURIComponent(room_id); path = path.replace("$room_id", room_id); diff --git a/syweb/webclient/test/unit/matrix-service.spec.js b/syweb/webclient/test/unit/matrix-service.spec.js index 95a43057c..b54163a64 100644 --- a/syweb/webclient/test/unit/matrix-service.spec.js +++ b/syweb/webclient/test/unit/matrix-service.spec.js @@ -238,7 +238,7 @@ describe('MatrixService', function() { homeserver: "http://example.com" }); var roomId = "!fh38hfwfwef:example.com"; - var eventType = "com.example.events.test"; + var eventType = "com.example.events.test:special@characters"; var content = { testing: "1 2 3" }; @@ -262,11 +262,11 @@ describe('MatrixService', function() { homeserver: "http://example.com" }); var roomId = "!fh38hfwfwef:example.com"; - var eventType = "com.example.events.test"; + var eventType = "com.example.events.test:special@characters"; var content = { testing: "1 2 3" }; - var stateKey = "version1"; + var stateKey = "version:1"; matrixService.sendStateEvent(roomId, eventType, content, stateKey).then( function(response) { expect(response.data).toEqual({}); From 42081b1937127979f3fb0a673eefb866cb4de64e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 5 Nov 2014 11:28:22 +0000 Subject: [PATCH 131/265] Don't urlencode event types just yet so older HSes don't 500. Skip the tests which test for urlencoding, and add a TODO in matrixService. --- syweb/webclient/components/matrix/matrix-service.js | 4 +++- syweb/webclient/test/unit/matrix-service.spec.js | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/syweb/webclient/components/matrix/matrix-service.js b/syweb/webclient/components/matrix/matrix-service.js index e1e5b88b3..8ff2999e2 100644 --- a/syweb/webclient/components/matrix/matrix-service.js +++ b/syweb/webclient/components/matrix/matrix-service.js @@ -375,7 +375,9 @@ angular.module('matrixService', []) sendStateEvent: function(room_id, eventType, content, state_key) { - var path = "/rooms/$room_id/state/"+ encodeURIComponent(eventType); + var path = "/rooms/$room_id/state/"+ eventType; + // TODO: uncomment this when matrix.org is updated, else all state events 500. + // var path = "/rooms/$room_id/state/"+ encodeURIComponent(eventType); if (state_key !== undefined) { path += "/" + encodeURIComponent(state_key); } diff --git a/syweb/webclient/test/unit/matrix-service.spec.js b/syweb/webclient/test/unit/matrix-service.spec.js index b54163a64..ed290f2ff 100644 --- a/syweb/webclient/test/unit/matrix-service.spec.js +++ b/syweb/webclient/test/unit/matrix-service.spec.js @@ -231,7 +231,7 @@ describe('MatrixService', function() { httpBackend.flush(); })); - it('should be able to send generic state events without a state key', inject( + xit('should be able to send generic state events without a state key', inject( function(matrixService) { matrixService.setConfig({ access_token: "foobar", @@ -255,7 +255,7 @@ describe('MatrixService', function() { httpBackend.flush(); })); - it('should be able to send generic state events with a state key', inject( + xit('should be able to send generic state events with a state key', inject( function(matrixService) { matrixService.setConfig({ access_token: "foobar", From d3a02ec03806d7629681895a0b960be24ff9ae1e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 5 Nov 2014 12:05:11 +0000 Subject: [PATCH 132/265] Fix url decoding bugs with /redact and /send APIs. --- synapse/rest/room.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index d97babea0..772496706 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -182,7 +182,7 @@ class RoomSendEventRestServlet(RestServlet): content = _parse_json(request) event = self.event_factory.create_event( - etype=event_type, + etype=urllib.unquote(event_type), room_id=urllib.unquote(room_id), user_id=user.to_string(), content=content @@ -458,7 +458,7 @@ class RoomRedactEventRestServlet(RestServlet): room_id=urllib.unquote(room_id), user_id=user.to_string(), content=content, - redacts=event_id, + redacts=urllib.unquote(event_id), ) msg_handler = self.handlers.message_handler From 0881a8ae6f6468af8345fc9e6e46bdb32dfb405b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 5 Nov 2014 12:32:28 +0000 Subject: [PATCH 133/265] Add more tests and a TODO. --- .../test/unit/matrix-service.spec.js | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/syweb/webclient/test/unit/matrix-service.spec.js b/syweb/webclient/test/unit/matrix-service.spec.js index ed290f2ff..2ca9a2432 100644 --- a/syweb/webclient/test/unit/matrix-service.spec.js +++ b/syweb/webclient/test/unit/matrix-service.spec.js @@ -231,14 +231,14 @@ describe('MatrixService', function() { httpBackend.flush(); })); - xit('should be able to send generic state events without a state key', inject( + it('should be able to send generic state events without a state key', inject( function(matrixService) { matrixService.setConfig({ access_token: "foobar", homeserver: "http://example.com" }); var roomId = "!fh38hfwfwef:example.com"; - var eventType = "com.example.events.test:special@characters"; + var eventType = "com.example.events.test"; var content = { testing: "1 2 3" }; @@ -255,6 +255,8 @@ describe('MatrixService', function() { httpBackend.flush(); })); + // TODO: Skipped since the webclient is purposefully broken so as not to + // 500 matrix.org xit('should be able to send generic state events with a state key', inject( function(matrixService) { matrixService.setConfig({ @@ -280,4 +282,55 @@ describe('MatrixService', function() { .respond({}); httpBackend.flush(); })); + + it('should be able to PUT generic events ', inject( + function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var roomId = "!fh38hfwfwef:example.com"; + var eventType = "com.example.events.test"; + var txnId = "42"; + var content = { + testing: "1 2 3" + }; + matrixService.sendEvent(roomId, eventType, txnId, content).then( + function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPUT( + URL + "/rooms/" + encodeURIComponent(roomId) + "/send/" + + encodeURIComponent(eventType) + "/" + encodeURIComponent(txnId)+ + "?access_token=foobar", + content) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to PUT text messages ', inject( + function(matrixService) { + matrixService.setConfig({ + access_token: "foobar", + homeserver: "http://example.com" + }); + var roomId = "!fh38hfwfwef:example.com"; + var body = "ABC 123"; + matrixService.sendTextMessage(roomId, body).then( + function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPUT( + new RegExp(URL + "/rooms/" + encodeURIComponent(roomId) + + "/send/m.room.message/(.*)" + + "?access_token=foobar"), + { + body: body, + msgtype: "m.text" + }) + .respond({}); + httpBackend.flush(); + })); }); From cc44ecc62f69436a9217745292af6c55b5f8fe81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:23:35 +0000 Subject: [PATCH 134/265] Get correct prev_events --- synapse/storage/_base.py | 11 +++++------ synapse/storage/event_federation.py | 30 ++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5e00c23fd..7d445b463 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -439,7 +439,7 @@ class SQLBaseStore(object): def _parse_events_txn(self, txn, rows): events = [self._parse_event_from_row(r) for r in rows] - sql = "SELECT * FROM events WHERE event_id = ?" + select_event_sql = "SELECT * FROM events WHERE event_id = ?" for ev in events: signatures = self._get_event_origin_signatures_txn( @@ -450,13 +450,12 @@ class SQLBaseStore(object): k: encode_base64(v) for k, v in signatures.items() } - prev_events = self._get_latest_events_in_room(txn, ev.room_id) - ev.prev_events = [(e_id, s,) for e_id, s, _ in prev_events] + ev.prev_events = self._get_prev_events(txn, ev.event_id) if hasattr(ev, "prev_state"): # Load previous state_content. # TODO: Should we be pulling this out above? - cursor = txn.execute(sql, (ev.prev_state,)) + cursor = txn.execute(select_event_sql, (ev.prev_state,)) prevs = self.cursor_to_dict(cursor) if prevs: prev = self._parse_event_from_row(prevs[0]) @@ -468,8 +467,8 @@ class SQLBaseStore(object): if ev.redacted: # Get the redaction event. - sql = "SELECT * FROM events WHERE event_id = ?" - txn.execute(sql, (ev.redacted,)) + select_event_sql = "SELECT * FROM events WHERE event_id = ?" + txn.execute(select_event_sql, (ev.redacted,)) del_evs = self._parse_events_txn( txn, self.cursor_to_dict(txn) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index dcc116bad..f427aba87 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -49,15 +49,6 @@ class EventFederationStore(SQLBaseStore): ) def _get_latest_events_in_room(self, txn, room_id): - self._simple_select_onecol_txn( - txn, - table="event_forward_extremities", - keyvalues={ - "room_id": room_id, - }, - retcol="event_id", - ) - sql = ( "SELECT e.event_id, e.depth FROM events as e " "INNER JOIN event_forward_extremities as f " @@ -78,6 +69,27 @@ class EventFederationStore(SQLBaseStore): return results + def _get_prev_events(self, txn, event_id): + prev_ids = self._simple_select_onecol_txn( + txn, + table="event_edges", + keyvalues={ + "event_id": event_id, + }, + retcol="prev_event_id", + ) + + results = [] + for prev_event_id in prev_ids: + hashes = self._get_event_reference_hashes_txn(txn, prev_event_id) + prev_hashes = { + k: encode_base64(v) for k, v in hashes.items() + if k == "sha256" + } + results.append((event_id, prev_hashes)) + + return results + def get_min_depth(self, room_id): return self.runInteraction( "get_min_depth", From da80ebcc6b0bd15095d092eacd0ea55c81bb03b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:27:20 +0000 Subject: [PATCH 135/265] Fix redaction storage test --- tests/storage/test_redaction.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index dae1641ea..fb175a5c4 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -61,6 +61,7 @@ class RedactionTestCase(unittest.TestCase): membership=membership, content={"membership": membership}, depth=self.depth, + prev_events=[], ) event.content.update(extra_content) @@ -68,6 +69,9 @@ class RedactionTestCase(unittest.TestCase): if prev_state: event.prev_state = prev_state + event.state_events = None + event.hashes = {} + # Have to create a join event using the eventfactory yield self.store.persist_event( event @@ -85,8 +89,12 @@ class RedactionTestCase(unittest.TestCase): room_id=room.to_string(), content={"body": body, "msgtype": u"message"}, depth=self.depth, + prev_events=[], ) + event.state_events = None + event.hashes = {} + yield self.store.persist_event( event ) @@ -102,8 +110,12 @@ class RedactionTestCase(unittest.TestCase): content={"reason": reason}, depth=self.depth, redacts=event_id, + prev_events=[], ) + event.state_events = None + event.hashes = {} + yield self.store.persist_event( event ) From 2fcce3b3c508ba1b10117e16511fb577da82f0b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:43:36 +0000 Subject: [PATCH 136/265] Remove stale tests --- tests/storage/test_room.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index 369a73d91..4ff02c306 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -127,7 +127,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_room_name(self): + def STALE_test_room_name(self): name = u"A-Room-Name" yield self.inject_room_event( @@ -150,7 +150,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_room_name(self): + def STALE_test_room_topic(self): topic = u"A place for things" yield self.inject_room_event( From 3791b75000f9787806dbdce3e6e216da1ac07388 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:48:36 +0000 Subject: [PATCH 137/265] Fix a couple more storage tests --- tests/storage/test_roommember.py | 24 +++++++++++++++--------- tests/storage/test_stream.py | 24 +++++++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index eae278ee8..9e2b2e31c 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -51,16 +51,22 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_room_member(self, room, user, membership): # Have to create a join event using the eventfactory + event = self.event_factory.create_event( + etype=RoomMemberEvent.TYPE, + user_id=user.to_string(), + state_key=user.to_string(), + room_id=room.to_string(), + membership=membership, + content={"membership": membership}, + depth=1, + prev_events=[], + ) + + event.state_events = None + event.hashes = {} + yield self.store.persist_event( - self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - user_id=user.to_string(), - state_key=user.to_string(), - room_id=room.to_string(), - membership=membership, - content={"membership": membership}, - depth=1, - ) + event ) @defer.inlineCallbacks diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index ab30e6ea2..b034fd8f1 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -59,8 +59,12 @@ class StreamStoreTestCase(unittest.TestCase): membership=membership, content={"membership": membership}, depth=self.depth, + prev_events=[], ) + event.state_events = None + event.hashes = {} + if prev_state: event.prev_state = prev_state @@ -75,15 +79,21 @@ class StreamStoreTestCase(unittest.TestCase): def inject_message(self, room, user, body): self.depth += 1 + event = self.event_factory.create_event( + etype=MessageEvent.TYPE, + user_id=user.to_string(), + room_id=room.to_string(), + content={"body": body, "msgtype": u"message"}, + depth=self.depth, + prev_events=[], + ) + + event.state_events = None + event.hashes = {} + # Have to create a join event using the eventfactory yield self.store.persist_event( - self.event_factory.create_event( - etype=MessageEvent.TYPE, - user_id=user.to_string(), - room_id=room.to_string(), - content={"body": body, "msgtype": u"message"}, - depth=self.depth, - ) + event ) @defer.inlineCallbacks From 988a8526b5a75a988fffd9ab5c3b4abbd2a41840 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 5 Nov 2014 14:35:41 +0000 Subject: [PATCH 138/265] Finish matrixService unit tests. Add missing encodeURIComponent to path args. --- .../components/matrix/matrix-service.js | 11 +- .../test/unit/matrix-service.spec.js | 288 ++++++++++++++---- 2 files changed, 234 insertions(+), 65 deletions(-) diff --git a/syweb/webclient/components/matrix/matrix-service.js b/syweb/webclient/components/matrix/matrix-service.js index 8ff2999e2..63051c4f4 100644 --- a/syweb/webclient/components/matrix/matrix-service.js +++ b/syweb/webclient/components/matrix/matrix-service.js @@ -443,7 +443,8 @@ angular.module('matrixService', []) redactEvent: function(room_id, event_id) { var path = "/rooms/$room_id/redact/$event_id"; - path = path.replace("$room_id", room_id); + path = path.replace("$room_id", encodeURIComponent(room_id)); + // TODO: encodeURIComponent when HS updated. path = path.replace("$event_id", event_id); var content = {}; return doRequest("POST", path, undefined, content); @@ -461,7 +462,7 @@ angular.module('matrixService', []) paginateBackMessages: function(room_id, from_token, limit) { var path = "/rooms/$room_id/messages"; - path = path.replace("$room_id", room_id); + path = path.replace("$room_id", encodeURIComponent(room_id)); var params = { from: from_token, limit: limit, @@ -509,12 +510,12 @@ angular.module('matrixService', []) setProfileInfo: function(data, info_segment) { var path = "/profile/$user/" + info_segment; - path = path.replace("$user", config.user_id); + path = path.replace("$user", encodeURIComponent(config.user_id)); return doRequest("PUT", path, undefined, data); }, getProfileInfo: function(userId, info_segment) { - var path = "/profile/"+userId + var path = "/profile/"+encodeURIComponent(userId); if (info_segment) path += '/' + info_segment; return doRequest("GET", path); }, @@ -633,7 +634,7 @@ angular.module('matrixService', []) // Set the logged in user presence state setUserPresence: function(presence) { var path = "/presence/$user_id/status"; - path = path.replace("$user_id", config.user_id); + path = path.replace("$user_id", encodeURIComponent(config.user_id)); return doRequest("PUT", path, undefined, { presence: presence }); diff --git a/syweb/webclient/test/unit/matrix-service.spec.js b/syweb/webclient/test/unit/matrix-service.spec.js index 2ca9a2432..4959f2395 100644 --- a/syweb/webclient/test/unit/matrix-service.spec.js +++ b/syweb/webclient/test/unit/matrix-service.spec.js @@ -5,6 +5,11 @@ describe('MatrixService', function() { var URL = BASE + PREFIX; var roomId = "!wejigf387t34:matrix.org"; + var CONFIG = { + access_token: "foobar", + homeserver: BASE + }; + beforeEach(module('matrixService')); beforeEach(inject(function($rootScope, $httpBackend) { @@ -19,10 +24,7 @@ describe('MatrixService', function() { it('should be able to POST /createRoom with an alias', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var alias = "flibble"; matrixService.create(alias).then(function(response) { expect(response.data).toEqual({}); @@ -37,10 +39,7 @@ describe('MatrixService', function() { })); it('should be able to GET /initialSync', inject(function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var limit = 15; matrixService.initialSync(limit).then(function(response) { expect(response.data).toEqual([]); @@ -54,10 +53,7 @@ describe('MatrixService', function() { it('should be able to GET /rooms/$roomid/state', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); matrixService.roomState(roomId).then(function(response) { expect(response.data).toEqual([]); }); @@ -70,10 +66,7 @@ describe('MatrixService', function() { })); it('should be able to POST /join', inject(function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); matrixService.joinAlias(roomId).then(function(response) { expect(response.data).toEqual({}); }); @@ -88,10 +81,7 @@ describe('MatrixService', function() { it('should be able to POST /rooms/$roomid/join', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); matrixService.join(roomId).then(function(response) { expect(response.data).toEqual({}); }); @@ -106,10 +96,7 @@ describe('MatrixService', function() { it('should be able to POST /rooms/$roomid/invite', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var inviteUserId = "@user:example.com"; matrixService.invite(roomId, inviteUserId).then(function(response) { expect(response.data).toEqual({}); @@ -127,10 +114,7 @@ describe('MatrixService', function() { it('should be able to POST /rooms/$roomid/leave', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); matrixService.leave(roomId).then(function(response) { expect(response.data).toEqual({}); }); @@ -145,10 +129,7 @@ describe('MatrixService', function() { it('should be able to POST /rooms/$roomid/ban', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var userId = "@example:example.com"; var reason = "Because."; matrixService.ban(roomId, userId, reason).then(function(response) { @@ -168,10 +149,7 @@ describe('MatrixService', function() { it('should be able to GET /directory/room/$alias', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var alias = "#test:example.com"; var roomId = "!wefuhewfuiw:example.com"; matrixService.resolveRoomAlias(alias).then(function(response) { @@ -190,10 +168,7 @@ describe('MatrixService', function() { })); it('should be able to send m.room.name', inject(function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var roomId = "!fh38hfwfwef:example.com"; var name = "Room Name"; matrixService.setName(roomId, name).then(function(response) { @@ -211,10 +186,7 @@ describe('MatrixService', function() { })); it('should be able to send m.room.topic', inject(function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var roomId = "!fh38hfwfwef:example.com"; var topic = "A room topic can go here."; matrixService.setTopic(roomId, topic).then(function(response) { @@ -233,10 +205,7 @@ describe('MatrixService', function() { it('should be able to send generic state events without a state key', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var roomId = "!fh38hfwfwef:example.com"; var eventType = "com.example.events.test"; var content = { @@ -259,10 +228,7 @@ describe('MatrixService', function() { // 500 matrix.org xit('should be able to send generic state events with a state key', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var roomId = "!fh38hfwfwef:example.com"; var eventType = "com.example.events.test:special@characters"; var content = { @@ -285,10 +251,7 @@ describe('MatrixService', function() { it('should be able to PUT generic events ', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var roomId = "!fh38hfwfwef:example.com"; var eventType = "com.example.events.test"; var txnId = "42"; @@ -311,10 +274,7 @@ describe('MatrixService', function() { it('should be able to PUT text messages ', inject( function(matrixService) { - matrixService.setConfig({ - access_token: "foobar", - homeserver: "http://example.com" - }); + matrixService.setConfig(CONFIG); var roomId = "!fh38hfwfwef:example.com"; var body = "ABC 123"; matrixService.sendTextMessage(roomId, body).then( @@ -333,4 +293,212 @@ describe('MatrixService', function() { .respond({}); httpBackend.flush(); })); + + it('should be able to PUT emote messages ', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + var roomId = "!fh38hfwfwef:example.com"; + var body = "ABC 123"; + matrixService.sendEmoteMessage(roomId, body).then( + function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPUT( + new RegExp(URL + "/rooms/" + encodeURIComponent(roomId) + + "/send/m.room.message/(.*)" + + "?access_token=foobar"), + { + body: body, + msgtype: "m.emote" + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to POST redactions', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + var roomId = "!fh38hfwfwef:example.com"; + var eventId = "fwefwexample.com"; + matrixService.redactEvent(roomId, eventId).then( + function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectPOST(URL + "/rooms/" + encodeURIComponent(roomId) + + "/redact/" + encodeURIComponent(eventId) + + "?access_token=foobar") + .respond({}); + httpBackend.flush(); + })); + + it('should be able to GET /directory/room/$alias', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + var alias = "#test:example.com"; + var roomId = "!wefuhewfuiw:example.com"; + matrixService.resolveRoomAlias(alias).then(function(response) { + expect(response.data).toEqual({ + room_id: roomId + }); + }); + + httpBackend.expectGET( + URL + "/directory/room/" + encodeURIComponent(alias) + + "?access_token=foobar") + .respond({ + room_id: roomId + }); + httpBackend.flush(); + })); + + it('should be able to GET /rooms/$roomid/members', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + var roomId = "!wefuhewfuiw:example.com"; + matrixService.getMemberList(roomId).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectGET( + URL + "/rooms/" + encodeURIComponent(roomId) + + "/members?access_token=foobar") + .respond({}); + httpBackend.flush(); + })); + + it('should be able to paginate a room', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + var roomId = "!wefuhewfuiw:example.com"; + var from = "3t_44e_54z"; + var limit = 20; + matrixService.paginateBackMessages(roomId, from, limit).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectGET( + URL + "/rooms/" + encodeURIComponent(roomId) + + "/messages?access_token=foobar&dir=b&from="+ + encodeURIComponent(from)+"&limit="+limit) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to GET /publicRooms', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + matrixService.publicRooms().then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectGET( + new RegExp(URL + "/publicRooms(.*)")) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to GET /profile/$userid/displayname', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + var userId = "@foo:example.com"; + matrixService.getDisplayName(userId).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectGET(URL + "/profile/" + encodeURIComponent(userId) + + "/displayname?access_token=foobar") + .respond({}); + httpBackend.flush(); + })); + + it('should be able to GET /profile/$userid/avatar_url', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + var userId = "@foo:example.com"; + matrixService.getProfilePictureUrl(userId).then(function(response) { + expect(response.data).toEqual({}); + }); + + httpBackend.expectGET(URL + "/profile/" + encodeURIComponent(userId) + + "/avatar_url?access_token=foobar") + .respond({}); + httpBackend.flush(); + })); + + it('should be able to PUT /profile/$me/avatar_url', inject( + function(matrixService) { + var testConfig = angular.copy(CONFIG); + testConfig.user_id = "@bob:example.com"; + matrixService.setConfig(testConfig); + var url = "http://example.com/mypic.jpg"; + matrixService.setProfilePictureUrl(url).then(function(response) { + expect(response.data).toEqual({}); + }); + httpBackend.expectPUT(URL + "/profile/" + + encodeURIComponent(testConfig.user_id) + + "/avatar_url?access_token=foobar", + { + avatar_url: url + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to PUT /profile/$me/displayname', inject( + function(matrixService) { + var testConfig = angular.copy(CONFIG); + testConfig.user_id = "@bob:example.com"; + matrixService.setConfig(testConfig); + var displayname = "Bob Smith"; + matrixService.setDisplayName(displayname).then(function(response) { + expect(response.data).toEqual({}); + }); + httpBackend.expectPUT(URL + "/profile/" + + encodeURIComponent(testConfig.user_id) + + "/displayname?access_token=foobar", + { + displayname: displayname + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to login with password', inject( + function(matrixService) { + matrixService.setConfig(CONFIG); + var userId = "@bob:example.com"; + var password = "monkey"; + matrixService.login(userId, password).then(function(response) { + expect(response.data).toEqual({}); + }); + httpBackend.expectPOST(new RegExp(URL+"/login(.*)"), + { + user: userId, + password: password, + type: "m.login.password" + }) + .respond({}); + httpBackend.flush(); + })); + + it('should be able to PUT presence status', inject( + function(matrixService) { + var testConfig = angular.copy(CONFIG); + testConfig.user_id = "@bob:example.com"; + matrixService.setConfig(testConfig); + var status = "unavailable"; + matrixService.setUserPresence(status).then(function(response) { + expect(response.data).toEqual({}); + }); + httpBackend.expectPUT(URL+"/presence/"+ + encodeURIComponent(testConfig.user_id)+ + "/status?access_token=foobar", + { + presence: status + }) + .respond({}); + httpBackend.flush(); + })); }); From 6aba43f6cc2350f0fce3eb15c4bdd38d7a9c17f6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 5 Nov 2014 15:32:35 +0000 Subject: [PATCH 139/265] Add a few eventHandlerService tests. --- .../test/unit/event-handler-service.spec.js | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 syweb/webclient/test/unit/event-handler-service.spec.js diff --git a/syweb/webclient/test/unit/event-handler-service.spec.js b/syweb/webclient/test/unit/event-handler-service.spec.js new file mode 100644 index 000000000..023abec98 --- /dev/null +++ b/syweb/webclient/test/unit/event-handler-service.spec.js @@ -0,0 +1,105 @@ +describe('EventHandlerService', function() { + var scope; + + var modelService = {}; + + // setup the service and mocked dependencies + beforeEach(function() { + // dependencies + module('matrixService'); + module('notificationService'); + module('mPresence'); + + // cleanup mocked methods + modelService = {}; + + // mocked dependencies + module(function ($provide) { + $provide.value('modelService', modelService); + }); + + // tested service + module('eventHandlerService'); + }); + + beforeEach(inject(function($rootScope) { + scope = $rootScope; + })); + + it('should be able to get the number of joined users in a room', inject( + function(eventHandlerService) { + var roomId = "!foo:matrix.org"; + // set mocked data + modelService.getRoom = function(roomId) { + return { + room_id: roomId, + current_room_state: { + members: { + "@adam:matrix.org": { + content: { membership: "join" }, + user_id: "@adam:matrix.org" + }, + "@beth:matrix.org": { + content: { membership: "invite" }, + user_id: "@beth:matrix.org" + }, + "@charlie:matrix.org": { + content: { membership: "join" }, + user_id: "@charlie:matrix.org" + }, + "@danice:matrix.org": { + content: { membership: "leave" }, + user_id: "@danice:matrix.org" + } + } + } + }; + } + + var num = eventHandlerService.getUsersCountInRoom(roomId); + expect(num).toEqual(2); + })); + + it('should be able to get a users power level', inject( + function(eventHandlerService) { + var roomId = "!foo:matrix.org"; + // set mocked data + modelService.getRoom = function(roomId) { + return { + room_id: roomId, + current_room_state: { + members: { + "@adam:matrix.org": { + content: { membership: "join" }, + user_id: "@adam:matrix.org" + }, + "@beth:matrix.org": { + content: { membership: "join" }, + user_id: "@beth:matrix.org" + } + }, + s: { + "m.room.power_levels": { + content: { + "@adam:matrix.org": 90, + "default": 50 + } + } + }, + state: function(type, key) { + return key ? this.s[type+key] : this.s[type] + } + } + }; + }; + + var num = eventHandlerService.getUserPowerLevel(roomId, "@beth:matrix.org"); + expect(num).toEqual(50); + + num = eventHandlerService.getUserPowerLevel(roomId, "@adam:matrix.org"); + expect(num).toEqual(90); + + num = eventHandlerService.getUserPowerLevel(roomId, "@unknown:matrix.org"); + expect(num).toEqual(50); + })); +}); \ No newline at end of file From 69c396825b28074346e3cfe3f239b5b1f2c143df Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 5 Nov 2014 17:49:03 +0000 Subject: [PATCH 140/265] Add duration filter unit tests. --- syweb/webclient/app-filter.js | 15 +------ syweb/webclient/test/unit/filters.spec.js | 50 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 syweb/webclient/test/unit/filters.spec.js diff --git a/syweb/webclient/app-filter.js b/syweb/webclient/app-filter.js index f19db4141..65da0d312 100644 --- a/syweb/webclient/app-filter.js +++ b/syweb/webclient/app-filter.js @@ -29,10 +29,10 @@ angular.module('matrixWebClient') return s + "s"; } if (t < 60 * 60) { - return m + "m "; // + s + "s"; + return m + "m"; // + s + "s"; } if (t < 24 * 60 * 60) { - return h + "h "; // + m + "m"; + return h + "h"; // + m + "m"; } return d + "d "; // + h + "h"; }; @@ -76,17 +76,6 @@ angular.module('matrixWebClient') return filtered; }; }) -.filter('stateEventsFilter', function($sce) { - return function(events) { - var filtered = {}; - angular.forEach(events, function(value, key) { - if (value && typeof(value.state_key) === "string") { - filtered[key] = value; - } - }); - return filtered; - }; -}) .filter('unsafe', ['$sce', function($sce) { return function(text) { return $sce.trustAsHtml(text); diff --git a/syweb/webclient/test/unit/filters.spec.js b/syweb/webclient/test/unit/filters.spec.js new file mode 100644 index 000000000..3dc735b2c --- /dev/null +++ b/syweb/webclient/test/unit/filters.spec.js @@ -0,0 +1,50 @@ +describe('durationFilter', function() { + var filter, durationFilter; + + beforeEach(module('matrixWebClient')); + beforeEach(module('matrixFilter')); + beforeEach(inject(function($filter) { + filter = $filter; + durationFilter = filter("duration"); + })); + + it("should represent 15000 ms as '15s'", function() { + var output = durationFilter(15000); + expect(output).toEqual("15s"); + }); + + it("should represent 60000 ms as '1m'", function() { + var output = durationFilter(60000); + expect(output).toEqual("1m"); + }); + + it("should represent 65000 ms as '1m'", function() { + var output = durationFilter(65000); + expect(output).toEqual("1m"); + }); + + it("should represent 10 ms as '0s'", function() { + var output = durationFilter(10); + expect(output).toEqual("0s"); + }); + + it("should represent 4m as '4m'", function() { + var output = durationFilter(1000*60*4); + expect(output).toEqual("4m"); + }); + + it("should represent 4m30s as '4m'", function() { + var output = durationFilter(1000*60*4 + 1000*30); + expect(output).toEqual("4m"); + }); + + it("should represent 2h as '2h'", function() { + var output = durationFilter(1000*60*60*2); + expect(output).toEqual("2h"); + }); + + it("should represent 2h35m as '2h'", function() { + var output = durationFilter(1000*60*60*2 + 1000*60*35); + expect(output).toEqual("2h"); + }); +}); \ No newline at end of file From 746525014110b4aef2eb00ba9b09714c3c8c5c77 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 09:34:35 +0000 Subject: [PATCH 141/265] State *.js in karma.conf rather than *.* so *.js~ files are ignored. --- syweb/webclient/test/karma.conf.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/syweb/webclient/test/karma.conf.js b/syweb/webclient/test/karma.conf.js index 083c7c720..7ce958adc 100644 --- a/syweb/webclient/test/karma.conf.js +++ b/syweb/webclient/test/karma.conf.js @@ -25,13 +25,13 @@ module.exports = function(config) { '../js/ng-infinite-scroll-matrix.js', '../js/ui-bootstrap*', '../js/elastic.js', - '../login/**/*.*', - '../room/**/*.*', - '../components/**/*.*', - '../user/**/*.*', - '../home/**/*.*', - '../recents/**/*.*', - '../settings/**/*.*', + '../login/**/*.js', + '../room/**/*.js', + '../components/**/*.js', + '../user/**/*.js', + '../home/**/*.js', + '../recents/**/*.js', + '../settings/**/*.js', '../app.js', '../app*', './unit/**/*.js' From c5eec32c583f2d577579dac9bc9ecb278b512d98 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 11:04:43 +0000 Subject: [PATCH 142/265] Add mRoomName and orderMembersList filter tests. Mark FIXME on broken tests for now. --- syweb/webclient/test/unit/filters.spec.js | 447 +++++++++++++++++++++- 1 file changed, 444 insertions(+), 3 deletions(-) diff --git a/syweb/webclient/test/unit/filters.spec.js b/syweb/webclient/test/unit/filters.spec.js index 3dc735b2c..f03742520 100644 --- a/syweb/webclient/test/unit/filters.spec.js +++ b/syweb/webclient/test/unit/filters.spec.js @@ -1,8 +1,243 @@ -describe('durationFilter', function() { +describe('mRoomName filter', function() { + var filter, mRoomName; + + var roomId = "!weufhewifu:matrix.org"; + + // test state values (f.e. test) + var testUserId, testAlias, testDisplayName, testOtherDisplayName, testRoomState; + + // mocked services which return the test values above. + var matrixService = { + getRoomIdToAliasMapping: function(room_id) { + return testAlias; + }, + + config: function() { + return { + user_id: testUserId + }; + } + }; + + var eventHandlerService = { + getUserDisplayName: function(room_id, user_id) { + if (user_id === testUserId) { + return testDisplayName; + } + return testOtherDisplayName; + } + }; + + var modelService = { + getRoom: function(room_id) { + return { + current_room_state: testRoomState + }; + } + }; + + beforeEach(function() { + // inject mocked dependencies + module(function ($provide) { + $provide.value('matrixService', matrixService); + $provide.value('eventHandlerService', eventHandlerService); + $provide.value('modelService', modelService); + }); + + module('matrixFilter'); + }); + + beforeEach(inject(function($filter) { + filter = $filter; + mRoomName = filter("mRoomName"); + + // purge the previous test values + testUserId = undefined; + testAlias = undefined; + testDisplayName = undefined; + testOtherDisplayName = undefined; + + // mock up a stub room state + testRoomState = { + s:{}, // internal; stores the state events + state: function(type, key) { + // accessor used by filter + return key ? this.s[type+key] : this.s[type]; + }, + members: {}, // struct used by filter + + // test helper methods + setJoinRule: function(rule) { + this.s["m.room.join_rules"] = { + content: { + join_rule: rule + } + }; + }, + setRoomName: function(name) { + this.s["m.room.name"] = { + content: { + name: name + } + }; + }, + setMember: function(user_id, membership, inviter_user_id) { + if (!inviter_user_id) { + inviter_user_id = user_id; + } + this.s["m.room.member" + user_id] = { + content: { + membership: membership + }, + state_key: user_id, + user_id: inviter_user_id + }; + this.members[user_id] = this.s["m.room.member" + user_id]; + } + }; + })); + + /**** ROOM NAME ****/ + + it("should show the room name if one exists for private (invite join_rules) rooms.", function() { + var roomName = "The Room Name"; + testUserId = "@me:matrix.org"; + testRoomState.setJoinRule("invite"); + testRoomState.setRoomName(roomName); + testRoomState.setMember(testUserId, "join"); + var output = mRoomName(roomId); + expect(output).toEqual(roomName); + }); + + it("should show the room name if one exists for public (public join_rules) rooms.", function() { + var roomName = "The Room Name"; + testUserId = "@me:matrix.org"; + testRoomState.setJoinRule("public"); + testRoomState.setRoomName(roomName); + testRoomState.setMember(testUserId, "join"); + var output = mRoomName(roomId); + expect(output).toEqual(roomName); + }); + + /**** ROOM ALIAS ****/ + + // FIXME + xit("should show the room alias if one exists for private (invite join_rules) rooms if a room name doesn't exist.", function() { + var testAlias = "#thealias:matrix.org"; + testUserId = "@me:matrix.org"; + testRoomState.setJoinRule("invite"); + testRoomState.setMember(testUserId, "join"); + var output = mRoomName(roomId); + expect(output).toEqual(testAlias); + }); + + // FIXME + xit("should show the room alias if one exists for public (public join_rules) rooms if a room name doesn't exist.", function() { + var testAlias = "#thealias:matrix.org"; + testUserId = "@me:matrix.org"; + testRoomState.setJoinRule("public"); + testRoomState.setMember(testUserId, "join"); + var output = mRoomName(roomId); + expect(output).toEqual(testAlias); + }); + + /**** ROOM ID ****/ + + it("should show the room ID for public (public join_rules) rooms if a room name and alias don't exist.", function() { + testUserId = "@me:matrix.org"; + testRoomState.setJoinRule("public"); + testRoomState.setMember(testUserId, "join"); + var output = mRoomName(roomId); + expect(output).toEqual(roomId); + }); + + it("should show the room ID for private (invite join_rules) rooms if a room name and alias don't exist and there are >2 members.", function() { + testUserId = "@me:matrix.org"; + testRoomState.setJoinRule("public"); + testRoomState.setMember(testUserId, "join"); + testRoomState.setMember("@alice:matrix.org", "join"); + testRoomState.setMember("@bob:matrix.org", "join"); + var output = mRoomName(roomId); + expect(output).toEqual(roomId); + }); + + /**** SELF-CHAT ****/ + + it("should show your display name for private (invite join_rules) rooms if a room name and alias don't exist and it is a self-chat.", function() { + testUserId = "@me:matrix.org"; + testDisplayName = "Me"; + testRoomState.setJoinRule("private"); + testRoomState.setMember(testUserId, "join"); + var output = mRoomName(roomId); + expect(output).toEqual(testDisplayName); + }); + + // FIXME + xit("should show your user ID for private (invite join_rules) rooms if a room name and alias don't exist and it is a self-chat and they don't have a display name set.", function() { + testUserId = "@me:matrix.org"; + testRoomState.setJoinRule("private"); + testRoomState.setMember(testUserId, "join"); + var output = mRoomName(roomId); + expect(output).toEqual(testUserId); + }); + + /**** ONE-TO-ONE CHAT ****/ + + it("should show the other user's display name for private (invite join_rules) rooms if a room name and alias don't exist and it is a 1:1-chat.", function() { + testUserId = "@me:matrix.org"; + otherUserId = "@alice:matrix.org"; + testOtherDisplayName = "Alice"; + testRoomState.setJoinRule("private"); + testRoomState.setMember(testUserId, "join"); + testRoomState.setMember("@alice:matrix.org", "join"); + var output = mRoomName(roomId); + expect(output).toEqual(testOtherDisplayName); + }); + + // FIXME + xit("should show the other user's ID for private (invite join_rules) rooms if a room name and alias don't exist and it is a 1:1-chat and they don't have a display name set.", function() { + testUserId = "@me:matrix.org"; + otherUserId = "@alice:matrix.org"; + testRoomState.setJoinRule("private"); + testRoomState.setMember(testUserId, "join"); + testRoomState.setMember("@alice:matrix.org", "join"); + var output = mRoomName(roomId); + expect(output).toEqual(otherUserId); + }); + + /**** INVITED TO ROOM ****/ + + it("should show the other user's display name for private (invite join_rules) rooms if you are invited to it.", function() { + testUserId = "@me:matrix.org"; + testDisplayName = "Me"; + otherUserId = "@alice:matrix.org"; + testOtherDisplayName = "Alice"; + testRoomState.setJoinRule("private"); + testRoomState.setMember(testUserId, "join"); + testRoomState.setMember(otherUserId, "join"); + testRoomState.setMember(testUserId, "invite"); + var output = mRoomName(roomId); + expect(output).toEqual(testOtherDisplayName); + }); + + // FIXME + xit("should show the other user's ID for private (invite join_rules) rooms if you are invited to it and the inviter doesn't have a display name.", function() { + testUserId = "@me:matrix.org"; + testDisplayName = "Me"; + otherUserId = "@alice:matrix.org"; + testRoomState.setJoinRule("private"); + testRoomState.setMember(testUserId, "join"); + testRoomState.setMember(otherUserId, "join"); + testRoomState.setMember(testUserId, "invite"); + var output = mRoomName(roomId); + expect(output).toEqual(otherUserId); + }); +}); + +describe('duration filter', function() { var filter, durationFilter; beforeEach(module('matrixWebClient')); - beforeEach(module('matrixFilter')); beforeEach(inject(function($filter) { filter = $filter; durationFilter = filter("duration"); @@ -47,4 +282,210 @@ describe('durationFilter', function() { var output = durationFilter(1000*60*60*2 + 1000*60*35); expect(output).toEqual("2h"); }); -}); \ No newline at end of file +}); + +describe('orderMembersList filter', function() { + var filter, orderMembersList; + + beforeEach(module('matrixWebClient')); + beforeEach(inject(function($filter) { + filter = $filter; + orderMembersList = filter("orderMembersList"); + })); + + it("should sort a single entry", function() { + var output = orderMembersList({ + "@a:example.com": { + last_active_ago: 50, + last_updated: 1415266943964 + } + }); + expect(output).toEqual([{ + id: "@a:example.com", + last_active_ago: 50, + last_updated: 1415266943964 + }]); + }); + + it("should sort by taking last_active_ago into account", function() { + var output = orderMembersList({ + "@a:example.com": { + last_active_ago: 1000, + last_updated: 1415266943964 + }, + "@b:example.com": { + last_active_ago: 50, + last_updated: 1415266943964 + }, + "@c:example.com": { + last_active_ago: 99999, + last_updated: 1415266943964 + } + }); + expect(output).toEqual([ + { + id: "@b:example.com", + last_active_ago: 50, + last_updated: 1415266943964 + }, + { + id: "@a:example.com", + last_active_ago: 1000, + last_updated: 1415266943964 + }, + { + id: "@c:example.com", + last_active_ago: 99999, + last_updated: 1415266943964 + }, + ]); + }); + + it("should sort by taking last_updated into account", function() { + var output = orderMembersList({ + "@a:example.com": { + last_active_ago: 1000, + last_updated: 1415266943964 + }, + "@b:example.com": { + last_active_ago: 1000, + last_updated: 1415266900000 + }, + "@c:example.com": { + last_active_ago: 1000, + last_updated: 1415266943000 + } + }); + expect(output).toEqual([ + { + id: "@a:example.com", + last_active_ago: 1000, + last_updated: 1415266943964 + }, + { + id: "@c:example.com", + last_active_ago: 1000, + last_updated: 1415266943000 + }, + { + id: "@b:example.com", + last_active_ago: 1000, + last_updated: 1415266900000 + }, + ]); + }); + + it("should sort by taking last_updated and last_active_ago into account", + function() { + var output = orderMembersList({ + "@a:example.com": { + last_active_ago: 1000, + last_updated: 1415266943000 + }, + "@b:example.com": { + last_active_ago: 100000, + last_updated: 1415266943900 + }, + "@c:example.com": { + last_active_ago: 1000, + last_updated: 1415266943964 + } + }); + expect(output).toEqual([ + { + id: "@c:example.com", + last_active_ago: 1000, + last_updated: 1415266943964 + }, + { + id: "@a:example.com", + last_active_ago: 1000, + last_updated: 1415266943000 + }, + { + id: "@b:example.com", + last_active_ago: 100000, + last_updated: 1415266943900 + }, + ]); + }); + + // SYWEB-26 comment + it("should sort members who do not have last_active_ago value at the end of the list", + function() { + // single undefined entry + var output = orderMembersList({ + "@a:example.com": { + last_active_ago: 1000, + last_updated: 1415266943964 + }, + "@b:example.com": { + last_active_ago: 100000, + last_updated: 1415266943964 + }, + "@c:example.com": { + last_active_ago: undefined, + last_updated: 1415266943964 + } + }); + expect(output).toEqual([ + { + id: "@a:example.com", + last_active_ago: 1000, + last_updated: 1415266943964 + }, + { + id: "@b:example.com", + last_active_ago: 100000, + last_updated: 1415266943964 + }, + { + id: "@c:example.com", + last_active_ago: undefined, + last_updated: 1415266943964 + }, + ]); + }); + + it("should sort multiple members who do not have last_active_ago according to presence", + function() { + // single undefined entry + var output = orderMembersList({ + "@a:example.com": { + last_active_ago: undefined, + last_updated: 1415266943964, + presence: "unavailable" + }, + "@b:example.com": { + last_active_ago: undefined, + last_updated: 1415266943964, + presence: "online" + }, + "@c:example.com": { + last_active_ago: undefined, + last_updated: 1415266943964, + presence: "offline" + } + }); + expect(output).toEqual([ + { + id: "@b:example.com", + last_active_ago: undefined, + last_updated: 1415266943964, + presence: "online" + }, + { + id: "@a:example.com", + last_active_ago: undefined, + last_updated: 1415266943964, + presence: "unavailable" + }, + { + id: "@c:example.com", + last_active_ago: undefined, + last_updated: 1415266943964, + presence: "offline" + }, + ]); + }); +}); From a92092340b5de022d1e48ecd4176cfb9b200b4d6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 11:14:31 +0000 Subject: [PATCH 143/265] Fix broken tests which were previously skipped. --- .../components/matrix/matrix-filter.js | 14 +++++++++++++- syweb/webclient/test/unit/filters.spec.js | 19 +++++++------------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/syweb/webclient/components/matrix/matrix-filter.js b/syweb/webclient/components/matrix/matrix-filter.js index 4d264e93f..e84c197c7 100644 --- a/syweb/webclient/components/matrix/matrix-filter.js +++ b/syweb/webclient/components/matrix/matrix-filter.js @@ -38,7 +38,7 @@ function($rootScope, matrixService, eventHandlerService, modelService) { if (room.state("m.room.join_rules") && room.state("m.room.join_rules").content) { isPublicRoom = ("public" === room.state("m.room.join_rules").content.join_rule); } - + if (room_name_event) { roomName = room_name_event.content.name; } @@ -56,6 +56,9 @@ function($rootScope, matrixService, eventHandlerService, modelService) { var member = room.members[i]; if (member.state_key !== user_id) { roomName = eventHandlerService.getUserDisplayName(room_id, member.state_key); + if (!roomName) { + roomName = member.state_key; + } break; } } @@ -70,13 +73,22 @@ function($rootScope, matrixService, eventHandlerService, modelService) { if (room.members[otherUserId].content.membership === "invite") { // someone invited us, use the right ID. roomName = eventHandlerService.getUserDisplayName(room_id, room.members[otherUserId].user_id); + if (!roomName) { + roomName = room.members[otherUserId].user_id; + } } else { roomName = eventHandlerService.getUserDisplayName(room_id, otherUserId); + if (!roomName) { + roomName = user_id; + } } } else { // it isn't us, so use their name if we know it. roomName = eventHandlerService.getUserDisplayName(room_id, otherUserId); + if (!roomName) { + roomName = otherUserId; + } } } else if (Object.keys(room.members).length === 0) { diff --git a/syweb/webclient/test/unit/filters.spec.js b/syweb/webclient/test/unit/filters.spec.js index f03742520..2e8d0c403 100644 --- a/syweb/webclient/test/unit/filters.spec.js +++ b/syweb/webclient/test/unit/filters.spec.js @@ -121,9 +121,8 @@ describe('mRoomName filter', function() { /**** ROOM ALIAS ****/ - // FIXME - xit("should show the room alias if one exists for private (invite join_rules) rooms if a room name doesn't exist.", function() { - var testAlias = "#thealias:matrix.org"; + it("should show the room alias if one exists for private (invite join_rules) rooms if a room name doesn't exist.", function() { + testAlias = "#thealias:matrix.org"; testUserId = "@me:matrix.org"; testRoomState.setJoinRule("invite"); testRoomState.setMember(testUserId, "join"); @@ -131,9 +130,8 @@ describe('mRoomName filter', function() { expect(output).toEqual(testAlias); }); - // FIXME - xit("should show the room alias if one exists for public (public join_rules) rooms if a room name doesn't exist.", function() { - var testAlias = "#thealias:matrix.org"; + it("should show the room alias if one exists for public (public join_rules) rooms if a room name doesn't exist.", function() { + testAlias = "#thealias:matrix.org"; testUserId = "@me:matrix.org"; testRoomState.setJoinRule("public"); testRoomState.setMember(testUserId, "join"); @@ -172,8 +170,7 @@ describe('mRoomName filter', function() { expect(output).toEqual(testDisplayName); }); - // FIXME - xit("should show your user ID for private (invite join_rules) rooms if a room name and alias don't exist and it is a self-chat and they don't have a display name set.", function() { + it("should show your user ID for private (invite join_rules) rooms if a room name and alias don't exist and it is a self-chat and they don't have a display name set.", function() { testUserId = "@me:matrix.org"; testRoomState.setJoinRule("private"); testRoomState.setMember(testUserId, "join"); @@ -194,8 +191,7 @@ describe('mRoomName filter', function() { expect(output).toEqual(testOtherDisplayName); }); - // FIXME - xit("should show the other user's ID for private (invite join_rules) rooms if a room name and alias don't exist and it is a 1:1-chat and they don't have a display name set.", function() { + it("should show the other user's ID for private (invite join_rules) rooms if a room name and alias don't exist and it is a 1:1-chat and they don't have a display name set.", function() { testUserId = "@me:matrix.org"; otherUserId = "@alice:matrix.org"; testRoomState.setJoinRule("private"); @@ -220,8 +216,7 @@ describe('mRoomName filter', function() { expect(output).toEqual(testOtherDisplayName); }); - // FIXME - xit("should show the other user's ID for private (invite join_rules) rooms if you are invited to it and the inviter doesn't have a display name.", function() { + it("should show the other user's ID for private (invite join_rules) rooms if you are invited to it and the inviter doesn't have a display name.", function() { testUserId = "@me:matrix.org"; testDisplayName = "Me"; otherUserId = "@alice:matrix.org"; From dd8af5565b5822dd2700ef8c2082134679fdc3ba Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 11:55:07 +0000 Subject: [PATCH 144/265] Start adding regression tests. First up, register-controller for SYWEB-109. --- syweb/webclient/login/register-controller.js | 2 +- .../test/unit/register-controller.spec.js | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 syweb/webclient/test/unit/register-controller.spec.js diff --git a/syweb/webclient/login/register-controller.js b/syweb/webclient/login/register-controller.js index be970ce1c..b23a72b18 100644 --- a/syweb/webclient/login/register-controller.js +++ b/syweb/webclient/login/register-controller.js @@ -124,7 +124,7 @@ angular.module('RegisterController', ['matrixService']) $location.url("home"); }, function(error) { - console.trace("Registration error: "+error); + console.error("Registration error: "+JSON.stringify(error)); if (useCaptcha) { Recaptcha.reload(); } diff --git a/syweb/webclient/test/unit/register-controller.spec.js b/syweb/webclient/test/unit/register-controller.spec.js new file mode 100644 index 000000000..369704c0d --- /dev/null +++ b/syweb/webclient/test/unit/register-controller.spec.js @@ -0,0 +1,84 @@ +describe("RegisterController ", function() { + var rootScope, scope, ctrl, $q, $timeout; + var userId = "@foo:bar"; + var displayName = "Foo"; + var avatarUrl = "avatar.url"; + + window.webClientConfig = { + useCapatcha: false + }; + + // test vars + var testRegisterData, testFailRegisterData; + + + // mock services + var matrixService = { + config: function() { + return { + user_id: userId + } + }, + setConfig: function(){}, + register: function(mxid, password, threepidCreds, useCaptcha) { + var d = $q.defer(); + if (testFailRegisterData) { + d.reject({ + data: testFailRegisterData + }); + } + else { + d.resolve({ + data: testRegisterData + }); + } + return d.promise; + } + }; + + var eventStreamService = {}; + + beforeEach(function() { + module('matrixWebClient'); + + // reset test vars + testRegisterData = undefined; + testFailRegisterData = undefined; + }); + + beforeEach(inject(function($rootScope, $injector, $location, $controller, _$q_, _$timeout_) { + $q = _$q_; + $timeout = _$timeout_; + scope = $rootScope.$new(); + rootScope = $rootScope; + routeParams = { + user_matrix_id: userId + }; + ctrl = $controller('RegisterController', { + '$scope': scope, + '$rootScope': $rootScope, + '$location': $location, + 'matrixService': matrixService, + 'eventStreamService': eventStreamService + }); + }) + ); + + // SYWEB-109 + it('should display an error if the HS rejects the username on registration', function() { + var prevFeedback = angular.copy(scope.feedback); + + testFailRegisterData = { + errcode: "M_UNKNOWN", + error: "I am rejecting you." + }; + + scope.account.pwd1 = "password"; + scope.account.pwd2 = "password"; + scope.account.desired_user_id = "bob"; + scope.register(); + rootScope.$digest(); + + expect(scope.feedback).toNotEqual(prevFeedback); + }); +}); From c9c2e395316a8f8ef905d1ca9d7e0bbab2c39f7e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 12:00:03 +0000 Subject: [PATCH 145/265] Use .not.toEqual instead of .toNotEqual which is in a newer version of Jasmine. --- syweb/webclient/test/unit/register-controller.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syweb/webclient/test/unit/register-controller.spec.js b/syweb/webclient/test/unit/register-controller.spec.js index 369704c0d..ce6ef1f4e 100644 --- a/syweb/webclient/test/unit/register-controller.spec.js +++ b/syweb/webclient/test/unit/register-controller.spec.js @@ -79,6 +79,6 @@ describe("RegisterController ", function() { scope.register(); rootScope.$digest(); - expect(scope.feedback).toNotEqual(prevFeedback); + expect(scope.feedback).not.toEqual(prevFeedback); }); }); From 8bcd36377a04bede2e2d74dcd7f18742d0982ad5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 13:37:05 +0000 Subject: [PATCH 146/265] Factor out room name logic: mRoomName is the canonical source. --- .../components/matrix/event-handler-service.js | 16 ++++------------ .../test/unit/register-controller.spec.js | 4 ++-- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/syweb/webclient/components/matrix/event-handler-service.js b/syweb/webclient/components/matrix/event-handler-service.js index 027c80a1b..38a6efced 100644 --- a/syweb/webclient/components/matrix/event-handler-service.js +++ b/syweb/webclient/components/matrix/event-handler-service.js @@ -26,8 +26,8 @@ Typically, this service will store events and broadcast them to any listeners (e.g. controllers) via $broadcast. */ angular.module('eventHandlerService', []) -.factory('eventHandlerService', ['matrixService', '$rootScope', '$q', '$timeout', 'mPresence', 'notificationService', 'modelService', -function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService, modelService) { +.factory('eventHandlerService', ['matrixService', '$rootScope', '$q', '$timeout', '$filter', 'mPresence', 'notificationService', 'modelService', +function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificationService, modelService) { var ROOM_CREATE_EVENT = "ROOM_CREATE_EVENT"; var MSG_EVENT = "MSG_EVENT"; var MEMBER_EVENT = "MEMBER_EVENT"; @@ -135,16 +135,8 @@ function(matrixService, $rootScope, $q, $timeout, mPresence, notificationService else if (event.content.msgtype === "m.image") { message = displayname + " sent an image."; } - - var roomTitle = matrixService.getRoomIdToAliasMapping(event.room_id); - var theRoom = modelService.getRoom(event.room_id); - if (!roomTitle && theRoom.current_room_state.state("m.room.name") && theRoom.current_room_state.state("m.room.name").content) { - roomTitle = theRoom.current_room_state.state("m.room.name").content.name; - } - - if (!roomTitle) { - roomTitle = event.room_id; - } + + var roomTitle = $filter("mRoomName")(event.room_id); notificationService.showNotification( displayname + " (" + roomTitle + ")", diff --git a/syweb/webclient/test/unit/register-controller.spec.js b/syweb/webclient/test/unit/register-controller.spec.js index ce6ef1f4e..b5c784235 100644 --- a/syweb/webclient/test/unit/register-controller.spec.js +++ b/syweb/webclient/test/unit/register-controller.spec.js @@ -76,8 +76,8 @@ describe("RegisterController ", function() { scope.account.pwd1 = "password"; scope.account.pwd2 = "password"; scope.account.desired_user_id = "bob"; - scope.register(); - rootScope.$digest(); + scope.register(); // this depends on the result of a deferred + rootScope.$digest(); // which is delivered after the digest expect(scope.feedback).not.toEqual(prevFeedback); }); From b77cce4ec56c3b4fd74ea65416e030fc321d4fa4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 14:18:23 +0000 Subject: [PATCH 147/265] Add modelService test. Thin for now but will be expanded upon. --- .../webclient/test/unit/model-service.spec.js | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 syweb/webclient/test/unit/model-service.spec.js diff --git a/syweb/webclient/test/unit/model-service.spec.js b/syweb/webclient/test/unit/model-service.spec.js new file mode 100644 index 000000000..2e012efe9 --- /dev/null +++ b/syweb/webclient/test/unit/model-service.spec.js @@ -0,0 +1,30 @@ +describe('ModelService', function() { + + // setup the dependencies + beforeEach(function() { + // dependencies + module('matrixService'); + + // tested service + module('modelService'); + }); + + it('should be able to get a member in a room', inject( + function(modelService) { + var roomId = "!wefiohwefuiow:matrix.org"; + var userId = "@bob:matrix.org"; + + modelService.getRoom(roomId).current_room_state.storeStateEvent({ + type: "m.room.member", + id: "fwefw:matrix.org", + user_id: userId, + state_key: userId, + content: { + membership: "join" + } + }); + + var user = modelService.getMember(roomId, userId); + expect(user.state_key).toEqual(userId); + })); +}); From d4c20c472b82e2a8851374743ec5075a58bb71ff Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 14:23:14 +0000 Subject: [PATCH 148/265] Use mRoomName on join notifications as well. --- syweb/webclient/room/room-controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index 2e0f5faff..a2bc23195 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -207,7 +207,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) } notificationService.showNotification( userName + - " (" + (matrixService.getRoomIdToAliasMapping(event.room_id) || event.room_id) + ")", + " (" + $filter("mRoomName")(event.room_id) + ")", userName + " joined", event.content.avatar_url ? event.content.avatar_url : undefined, function() { From e3c3f5a6d04bfbc0256010e9fb4dad7616ebbcc5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 6 Nov 2014 14:52:22 +0000 Subject: [PATCH 149/265] Swap from using raw m.room.member events for room members to using actual RoomMember objects, so User objects can be tacked on. Update tests. --- .../matrix/event-handler-service.js | 9 +++-- .../components/matrix/matrix-call.js | 2 +- .../components/matrix/matrix-filter.js | 8 ++-- .../components/matrix/model-service.js | 4 +- syweb/webclient/recents/recents-filter.js | 3 ++ syweb/webclient/room/room-controller.js | 4 +- .../test/unit/event-handler-service.spec.js | 38 ++++++++++++------- syweb/webclient/test/unit/filters.spec.js | 12 +++--- .../webclient/test/unit/model-service.spec.js | 2 +- 9 files changed, 52 insertions(+), 30 deletions(-) diff --git a/syweb/webclient/components/matrix/event-handler-service.js b/syweb/webclient/components/matrix/event-handler-service.js index 38a6efced..a9c6eb34c 100644 --- a/syweb/webclient/components/matrix/event-handler-service.js +++ b/syweb/webclient/components/matrix/event-handler-service.js @@ -141,7 +141,7 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati notificationService.showNotification( displayname + " (" + roomTitle + ")", message, - member ? member.avatar_url : undefined, + member ? member.event.content.avatar_url : undefined, function() { console.log("notification.onclick() room=" + event.room_id); $rootScope.goToPage('room/' + event.room_id); @@ -306,6 +306,9 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati // Get the user display name from the member list of the room var member = modelService.getMember(room_id, user_id); + if (member) { + member = member.event; + } if (member && member.content.displayname) { // Do not consider null displayname displayName = member.content.displayname; @@ -315,7 +318,7 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati for (var member_id in room.current_room_state.members) { if (room.current_room_state.members.hasOwnProperty(member_id) && member_id !== user_id) { - var member2 = room.current_room_state.members[member_id]; + var member2 = room.current_room_state.members[member_id].event; if (member2.content.displayname && member2.content.displayname === displayName) { displayName = displayName + " (" + user_id + ")"; break; @@ -551,7 +554,7 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati for (var i in room.current_room_state.members) { if (!room.current_room_state.members.hasOwnProperty(i)) continue; - var member = room.current_room_state.members[i]; + var member = room.current_room_state.members[i].event; if ("join" === member.content.membership) { memberCount = memberCount + 1; diff --git a/syweb/webclient/components/matrix/matrix-call.js b/syweb/webclient/components/matrix/matrix-call.js index 5a2807c75..465b2b780 100644 --- a/syweb/webclient/components/matrix/matrix-call.js +++ b/syweb/webclient/components/matrix/matrix-call.js @@ -214,7 +214,7 @@ angular.module('MatrixCall', []) var self = this; var roomMembers = modelService.getRoom(this.room_id).current_room_state.members; - if (roomMembers[matrixService.config().user_id].membership != 'join') { + if (roomMembers[matrixService.config().user_id].event.content.membership != 'join') { console.log("We need to join the room before we can accept this call"); matrixService.join(this.room_id).then(function() { self.answer(); diff --git a/syweb/webclient/components/matrix/matrix-filter.js b/syweb/webclient/components/matrix/matrix-filter.js index e84c197c7..aeebedc78 100644 --- a/syweb/webclient/components/matrix/matrix-filter.js +++ b/syweb/webclient/components/matrix/matrix-filter.js @@ -53,7 +53,7 @@ function($rootScope, matrixService, eventHandlerService, modelService) { for (var i in room.members) { if (!room.members.hasOwnProperty(i)) continue; - var member = room.members[i]; + var member = room.members[i].event; if (member.state_key !== user_id) { roomName = eventHandlerService.getUserDisplayName(room_id, member.state_key); if (!roomName) { @@ -70,11 +70,11 @@ function($rootScope, matrixService, eventHandlerService, modelService) { if (otherUserId === user_id) { // it's us, we may have been invited to this room or it could // be a self chat. - if (room.members[otherUserId].content.membership === "invite") { + if (room.members[otherUserId].event.content.membership === "invite") { // someone invited us, use the right ID. - roomName = eventHandlerService.getUserDisplayName(room_id, room.members[otherUserId].user_id); + roomName = eventHandlerService.getUserDisplayName(room_id, room.members[otherUserId].event.user_id); if (!roomName) { - roomName = room.members[otherUserId].user_id; + roomName = room.members[otherUserId].event.user_id; } } else { diff --git a/syweb/webclient/components/matrix/model-service.js b/syweb/webclient/components/matrix/model-service.js index 8b2ee877b..8e0ce8d1a 100644 --- a/syweb/webclient/components/matrix/model-service.js +++ b/syweb/webclient/components/matrix/model-service.js @@ -106,7 +106,9 @@ angular.module('modelService', []) storeStateEvent: function storeState(event) { this.state_events[event.type + event.state_key] = event; if (event.type === "m.room.member") { - this.members[event.state_key] = event; + var rm = new RoomMember(); + rm.event = event; + this.members[event.state_key] = rm; } }, diff --git a/syweb/webclient/recents/recents-filter.js b/syweb/webclient/recents/recents-filter.js index 39c235996..cfbc6f4bd 100644 --- a/syweb/webclient/recents/recents-filter.js +++ b/syweb/webclient/recents/recents-filter.js @@ -30,6 +30,9 @@ angular.module('RecentsController') // Show the room only if the user has joined it or has been invited // (ie, do not show it if he has been banned) var member = modelService.getMember(room_id, user_id); + if (member) { + member = member.event; + } room.recent.me = member; if (member && ("invite" === member.content.membership || "join" === member.content.membership)) { if ("invite" === member.content.membership) { diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index a2bc23195..d3fb85b9d 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -754,13 +754,13 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) for (var i in members) { if (!members.hasOwnProperty(i)) continue; - var member = members[i]; + var member = members[i].event; updateMemberList(member); } // Check if the user has already join the room if ($scope.state.user_id in members) { - if ("join" === members[$scope.state.user_id].membership) { + if ("join" === members[$scope.state.user_id].event.content.membership) { needsToJoin = false; } } diff --git a/syweb/webclient/test/unit/event-handler-service.spec.js b/syweb/webclient/test/unit/event-handler-service.spec.js index 023abec98..2a4dc3b5a 100644 --- a/syweb/webclient/test/unit/event-handler-service.spec.js +++ b/syweb/webclient/test/unit/event-handler-service.spec.js @@ -36,20 +36,28 @@ describe('EventHandlerService', function() { current_room_state: { members: { "@adam:matrix.org": { - content: { membership: "join" }, - user_id: "@adam:matrix.org" + event: { + content: { membership: "join" }, + user_id: "@adam:matrix.org" + } }, "@beth:matrix.org": { - content: { membership: "invite" }, - user_id: "@beth:matrix.org" + event: { + content: { membership: "invite" }, + user_id: "@beth:matrix.org" + } }, "@charlie:matrix.org": { - content: { membership: "join" }, - user_id: "@charlie:matrix.org" + event: { + content: { membership: "join" }, + user_id: "@charlie:matrix.org" + } }, "@danice:matrix.org": { - content: { membership: "leave" }, - user_id: "@danice:matrix.org" + event: { + content: { membership: "leave" }, + user_id: "@danice:matrix.org" + } } } } @@ -70,12 +78,16 @@ describe('EventHandlerService', function() { current_room_state: { members: { "@adam:matrix.org": { - content: { membership: "join" }, - user_id: "@adam:matrix.org" + event: { + content: { membership: "join" }, + user_id: "@adam:matrix.org" + } }, "@beth:matrix.org": { - content: { membership: "join" }, - user_id: "@beth:matrix.org" + event: { + content: { membership: "join" }, + user_id: "@beth:matrix.org" + } } }, s: { @@ -102,4 +114,4 @@ describe('EventHandlerService', function() { num = eventHandlerService.getUserPowerLevel(roomId, "@unknown:matrix.org"); expect(num).toEqual(50); })); -}); \ No newline at end of file +}); diff --git a/syweb/webclient/test/unit/filters.spec.js b/syweb/webclient/test/unit/filters.spec.js index 2e8d0c403..7324a8e02 100644 --- a/syweb/webclient/test/unit/filters.spec.js +++ b/syweb/webclient/test/unit/filters.spec.js @@ -86,11 +86,13 @@ describe('mRoomName filter', function() { inviter_user_id = user_id; } this.s["m.room.member" + user_id] = { - content: { - membership: membership - }, - state_key: user_id, - user_id: inviter_user_id + event: { + content: { + membership: membership + }, + state_key: user_id, + user_id: inviter_user_id + } }; this.members[user_id] = this.s["m.room.member" + user_id]; } diff --git a/syweb/webclient/test/unit/model-service.spec.js b/syweb/webclient/test/unit/model-service.spec.js index 2e012efe9..e2fa8ceba 100644 --- a/syweb/webclient/test/unit/model-service.spec.js +++ b/syweb/webclient/test/unit/model-service.spec.js @@ -25,6 +25,6 @@ describe('ModelService', function() { }); var user = modelService.getMember(roomId, userId); - expect(user.state_key).toEqual(userId); + expect(user.event.state_key).toEqual(userId); })); }); From 4317c8e5835f0c15bf882f737d3e3c2a5b85f73f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 15:10:55 +0000 Subject: [PATCH 150/265] Implement new replace_state and changed prev_state `prev_state` is now a list of previous state ids, similiar to prev_events. `replace_state` now points to what we think was replaced. --- synapse/api/events/__init__.py | 1 + synapse/handlers/directory.py | 5 +- synapse/handlers/federation.py | 4 +- synapse/handlers/message.py | 11 +-- synapse/handlers/profile.py | 6 +- synapse/handlers/room.py | 16 ++--- synapse/rest/room.py | 2 +- synapse/state.py | 39 ++--------- synapse/storage/__init__.py | 92 +++++++++++++++++++------- synapse/storage/_base.py | 64 +++++++++++++----- synapse/storage/event_federation.py | 64 ++++++++++++++++-- synapse/storage/schema/event_edges.sql | 40 +++++++---- synapse/util/jsonobject.py | 2 +- 13 files changed, 219 insertions(+), 127 deletions(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 168b81231..fc3f35057 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -60,6 +60,7 @@ class SynapseEvent(JsonEncodedObject): "age_ts", "prev_content", "prev_state", + "replaces_state", "redacted_because", "origin_server_ts", ] diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 6e897e915..164363cdc 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -147,10 +147,7 @@ class DirectoryHandler(BaseHandler): content={"aliases": aliases}, ) - snapshot = yield self.store.snapshot_room( - room_id=room_id, - user_id=user_id, - ) + snapshot = yield self.store.snapshot_room(event) yield self._on_new_room_event( event, snapshot, extra_users=[user_id], suppress_auth=True diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1464a6093..513ec9a5e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -313,9 +313,7 @@ class FederationHandler(BaseHandler): state_key=user_id, ) - snapshot = yield self.store.snapshot_room( - event.room_id, event.user_id, - ) + snapshot = yield self.store.snapshot_room(event) snapshot.fill_out_prev_events(event) yield self.state_handler.annotate_state_groups(event) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6f6ab14d..8394013df 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -81,7 +81,7 @@ class MessageHandler(BaseHandler): user = self.hs.parse_userid(event.user_id) assert user.is_mine, "User must be our own: %s" % (user,) - snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) + snapshot = yield self.store.snapshot_room(event) yield self._on_new_room_event( event, snapshot, suppress_auth=suppress_auth @@ -141,12 +141,7 @@ class MessageHandler(BaseHandler): SynapseError if something went wrong. """ - snapshot = yield self.store.snapshot_room( - event.room_id, - event.user_id, - state_type=event.type, - state_key=event.state_key, - ) + snapshot = yield self.store.snapshot_room(event) yield self._on_new_room_event(event, snapshot) @@ -214,7 +209,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def send_feedback(self, event): - snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) + snapshot = yield self.store.snapshot_room(event) # store message in db yield self._on_new_room_event(event, snapshot) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 4cd0a0609..e47814483 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException from synapse.api.constants import Membership -from synapse.api.events.room import RoomMemberEvent from ._base import BaseHandler @@ -196,10 +195,7 @@ class ProfileHandler(BaseHandler): ) for j in joins: - snapshot = yield self.store.snapshot_room( - j.room_id, j.state_key, RoomMemberEvent.TYPE, - j.state_key - ) + snapshot = yield self.store.snapshot_room(j) content = { "membership": j.content["membership"], diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f176ad39b..55c893eb5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -122,10 +122,7 @@ class RoomCreationHandler(BaseHandler): @defer.inlineCallbacks def handle_event(event): - snapshot = yield self.store.snapshot_room( - room_id=room_id, - user_id=user_id, - ) + snapshot = yield self.store.snapshot_room(event) logger.debug("Event: %s", event) @@ -364,10 +361,8 @@ class RoomMemberHandler(BaseHandler): """ target_user_id = event.state_key - snapshot = yield self.store.snapshot_room( - event.room_id, event.user_id, - RoomMemberEvent.TYPE, target_user_id - ) + snapshot = yield self.store.snapshot_room(event) + ## TODO(markjh): get prev state from snapshot. prev_state = yield self.store.get_room_member( target_user_id, event.room_id @@ -442,10 +437,7 @@ class RoomMemberHandler(BaseHandler): content=content, ) - snapshot = yield self.store.snapshot_room( - room_id, joinee.to_string(), RoomMemberEvent.TYPE, - joinee.to_string() - ) + snapshot = yield self.store.snapshot_room(new_event) yield self._do_join(new_event, snapshot, room_host=host, do_auth=True) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index ec0ce78fd..997895dab 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -138,7 +138,7 @@ class RoomStateEventRestServlet(RestServlet): raise SynapseError( 404, "Event not found.", errcode=Codes.NOT_FOUND ) - defer.returnValue((200, data[0].get_dict()["content"])) + defer.returnValue((200, data.get_dict()["content"])) @defer.inlineCallbacks def on_PUT(self, request, room_id, event_type, state_key): diff --git a/synapse/state.py b/synapse/state.py index 32744e047..97a8160a3 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -45,40 +45,6 @@ class StateHandler(object): self.server_name = hs.hostname self.hs = hs - @defer.inlineCallbacks - @log_function - def handle_new_event(self, event, snapshot): - """ Given an event this works out if a) we have sufficient power level - to update the state and b) works out what the prev_state should be. - - Returns: - Deferred: Resolved with a boolean indicating if we successfully - updated the state. - - Raised: - AuthError - """ - # This needs to be done in a transaction. - - if not hasattr(event, "state_key"): - return - - # Now I need to fill out the prev state and work out if it has auth - # (w.r.t. to power levels) - - snapshot.fill_out_prev_events(event) - yield self.annotate_state_groups(event) - - if event.old_state_events: - current_state = event.old_state_events.get( - (event.type, event.state_key) - ) - - if current_state: - event.prev_state = current_state.event_id - - defer.returnValue(True) - @defer.inlineCallbacks @log_function def annotate_state_groups(self, event, old_state=None): @@ -111,7 +77,10 @@ class StateHandler(object): event.old_state_events = copy.deepcopy(new_state) if hasattr(event, "state_key"): - new_state[(event.type, event.state_key)] = event + key = (event.type, event.state_key) + if key in new_state: + event.replaces_state = new_state[key].event_id + new_state[key] = event event.state_group = None event.state_events = new_state diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6b8fed450..2d62fc2ed 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -242,8 +242,8 @@ class DataStore(RoomMemberStore, RoomStore, "state_key": event.state_key, } - if hasattr(event, "prev_state"): - vals["prev_state"] = event.prev_state + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state self._simple_insert_txn(txn, "state_events", vals) @@ -258,6 +258,40 @@ class DataStore(RoomMemberStore, RoomStore, } ) + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": 1, + }, + or_ignore=True, + ) + + if not backfilled: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) + for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( @@ -357,7 +391,7 @@ class DataStore(RoomMemberStore, RoomStore, ], ) - def snapshot_room(self, room_id, user_id, state_type=None, state_key=None): + def snapshot_room(self, event): """Snapshot the room for an update by a user Args: room_id (synapse.types.RoomId): The room to snapshot. @@ -368,16 +402,29 @@ class DataStore(RoomMemberStore, RoomStore, synapse.storage.Snapshot: A snapshot of the state of the room. """ def _snapshot(txn): - membership_state = self._get_room_member(txn, user_id, room_id) - prev_events = self._get_latest_events_in_room(txn, room_id) + prev_events = self._get_latest_events_in_room( + txn, + event.room_id + ) + + prev_state = None + state_key = None + if hasattr(event, "state_key"): + state_key = event.state_key + prev_state = self._get_latest_state_in_room( + txn, + event.room_id, + type=event.type, + state_key=state_key, + ) return Snapshot( store=self, - room_id=room_id, - user_id=user_id, + room_id=event.room_id, + user_id=event.user_id, prev_events=prev_events, - membership_state=membership_state, - state_type=state_type, + prev_state=prev_state, + state_type=event.type, state_key=state_key, ) @@ -400,30 +447,29 @@ class Snapshot(object): """ def __init__(self, store, room_id, user_id, prev_events, - membership_state, state_type=None, state_key=None, - prev_state_pdu=None): + prev_state, state_type=None, state_key=None): self.store = store self.room_id = room_id self.user_id = user_id self.prev_events = prev_events - self.membership_state = membership_state + self.prev_state = prev_state self.state_type = state_type self.state_key = state_key - self.prev_state_pdu = prev_state_pdu def fill_out_prev_events(self, event): - if hasattr(event, "prev_events"): - return + if not hasattr(event, "prev_events"): + event.prev_events = [ + (event_id, hashes) + for event_id, hashes, _ in self.prev_events + ] - event.prev_events = [ - (event_id, hashes) - for event_id, hashes, _ in self.prev_events - ] + if self.prev_events: + event.depth = max([int(v) for _, _, v in self.prev_events]) + 1 + else: + event.depth = 0 - if self.prev_events: - event.depth = max([int(v) for _, _, v in self.prev_events]) + 1 - else: - event.depth = 0 + if not hasattr(event, "prev_state") and self.prev_state is not None: + event.prev_state = self.prev_state def schema_path(schema): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7d445b463..7821fc472 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -245,7 +245,6 @@ class SQLBaseStore(object): return [r[0] for r in txn.fetchall()] - def _simple_select_onecol(self, table, keyvalues, retcol): """Executes a SELECT query on the named table, which returns a list comprising of the values of the named column from the selected rows. @@ -273,17 +272,30 @@ class SQLBaseStore(object): keyvalues : dict of column names and values to select the rows with retcols : list of strings giving the names of the columns to return """ + return self.runInteraction( + "_simple_select_list", + self._simple_select_list_txn, + table, keyvalues, retcols + ) + + def _simple_select_list_txn(self, txn, table, keyvalues, retcols): + """Executes a SELECT query on the named table, which may return zero or + more rows, returning the result as a list of dicts. + + Args: + txn : Transaction object + table : string giving the table name + keyvalues : dict of column names and values to select the rows with + retcols : list of strings giving the names of the columns to return + """ sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, - " AND ".join("%s = ?" % (k) for k in keyvalues) + " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) - def func(txn): - txn.execute(sql, keyvalues.values()) - return self.cursor_to_dict(txn) - - return self.runInteraction("_simple_select_list", func) + txn.execute(sql, keyvalues.values()) + return self.cursor_to_dict(txn) def _simple_update_one(self, table, keyvalues, updatevalues, retcols=None): @@ -417,6 +429,10 @@ class SQLBaseStore(object): d.pop("topological_ordering", None) d.pop("processed", None) d["origin_server_ts"] = d.pop("ts", 0) + replaces_state = d.pop("prev_state", None) + + if replaces_state: + d["replaces_state"] = replaces_state d.update(json.loads(row_dict["unrecognized_keys"])) d["content"] = json.loads(d["content"]) @@ -450,16 +466,32 @@ class SQLBaseStore(object): k: encode_base64(v) for k, v in signatures.items() } - ev.prev_events = self._get_prev_events(txn, ev.event_id) + prevs = self._get_prev_events_and_state(txn, ev.event_id) - if hasattr(ev, "prev_state"): - # Load previous state_content. - # TODO: Should we be pulling this out above? - cursor = txn.execute(select_event_sql, (ev.prev_state,)) - prevs = self.cursor_to_dict(cursor) - if prevs: - prev = self._parse_event_from_row(prevs[0]) - ev.prev_content = prev.content + ev.prev_events = [ + (e_id, h) + for e_id, h, is_state in prevs + if is_state == 0 + ] + + if hasattr(ev, "state_key"): + ev.prev_state = [ + (e_id, h) + for e_id, h, is_state in prevs + if is_state == 1 + ] + + if hasattr(ev, "replaces_state"): + # Load previous state_content. + # FIXME (erikj): Handle multiple prev_states. + cursor = txn.execute( + select_event_sql, + (ev.replaces_state,) + ) + prevs = self.cursor_to_dict(cursor) + if prevs: + prev = self._parse_event_from_row(prevs[0]) + ev.prev_content = prev.content if not hasattr(ev, "redacted"): logger.debug("Doesn't have redacted key: %s", ev) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index f427aba87..180a76413 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -69,19 +69,21 @@ class EventFederationStore(SQLBaseStore): return results - def _get_prev_events(self, txn, event_id): - prev_ids = self._simple_select_onecol_txn( + def _get_latest_state_in_room(self, txn, room_id, type, state_key): + event_ids = self._simple_select_onecol_txn( txn, - table="event_edges", + table="state_forward_extremities", keyvalues={ - "event_id": event_id, + "room_id": room_id, + "type": type, + "state_key": state_key, }, - retcol="prev_event_id", + retcol="event_id", ) results = [] - for prev_event_id in prev_ids: - hashes = self._get_event_reference_hashes_txn(txn, prev_event_id) + for event_id in event_ids: + hashes = self._get_event_reference_hashes_txn(txn, event_id) prev_hashes = { k: encode_base64(v) for k, v in hashes.items() if k == "sha256" @@ -90,6 +92,53 @@ class EventFederationStore(SQLBaseStore): return results + def _get_prev_events(self, txn, event_id): + results = self._get_prev_events_and_state( + txn, + event_id, + is_state=0, + ) + + return [(e_id, h, ) for e_id, h, _ in results] + + def _get_prev_state(self, txn, event_id): + results = self._get_prev_events_and_state( + txn, + event_id, + is_state=1, + ) + + return [(e_id, h, ) for e_id, h, _ in results] + + def _get_prev_events_and_state(self, txn, event_id, is_state=None): + keyvalues = { + "event_id": event_id, + } + + if is_state is not None: + keyvalues["is_state"] = is_state + + res = self._simple_select_list_txn( + txn, + table="event_edges", + keyvalues=keyvalues, + retcols=["prev_event_id", "is_state"], + ) + + results = [] + for d in res: + hashes = self._get_event_reference_hashes_txn( + txn, + d["prev_event_id"] + ) + prev_hashes = { + k: encode_base64(v) for k, v in hashes.items() + if k == "sha256" + } + results.append((d["prev_event_id"], prev_hashes, d["is_state"])) + + return results + def get_min_depth(self, room_id): return self.runInteraction( "get_min_depth", @@ -135,6 +184,7 @@ class EventFederationStore(SQLBaseStore): "event_id": event_id, "prev_event_id": e_id, "room_id": room_id, + "is_state": 0, }, or_ignore=True, ) diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index e5f768c70..51695826a 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id TEXT, - room_id TEXT, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE ); @@ -10,8 +10,8 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id TEXT, - room_id TEXT, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE ); @@ -20,10 +20,11 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id CREATE TABLE IF NOT EXISTS event_edges( - event_id TEXT, - prev_event_id TEXT, - room_id TEXT, - CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id) + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + is_state INTEGER NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) ); CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); @@ -31,8 +32,8 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id TEXT, - min_depth INTEGER, + room_id TEXT NOT NULL, + min_depth INTEGER NOT NULL, CONSTRAINT uniqueness UNIQUE (room_id) ); @@ -40,10 +41,25 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id TEXT, - destination TEXT, + event_id TEXT NOT NULL, + destination TEXT NOT NULL, delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE ); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); + + +CREATE TABLE IF NOT EXISTS state_forward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( + room_id, type, state_key +); +CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + diff --git a/synapse/util/jsonobject.py b/synapse/util/jsonobject.py index c91eb897a..e79b68f66 100644 --- a/synapse/util/jsonobject.py +++ b/synapse/util/jsonobject.py @@ -80,7 +80,7 @@ class JsonEncodedObject(object): def get_full_dict(self): d = { - k: v for (k, v) in self.__dict__.items() + k: _encode(v) for (k, v) in self.__dict__.items() if k in self.valid_keys or k in self.internal_keys } d.update(self.unrecognized_keys) From c6766d45b53ba51121ca15ae5806a5ac30f4f5d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 15:19:00 +0000 Subject: [PATCH 151/265] Don't send prev_state to clients anymore --- synapse/api/events/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index fc3f35057..98a66144e 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -59,7 +59,6 @@ class SynapseEvent(JsonEncodedObject): "required_power_level", "age_ts", "prev_content", - "prev_state", "replaces_state", "redacted_because", "origin_server_ts", @@ -76,6 +75,7 @@ class SynapseEvent(JsonEncodedObject): "prev_events", "hashes", "signatures", + "prev_state", ] required_keys = [ From 233969bb586f2a5b0231c8e0675c8c8c3097ef03 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 15:25:03 +0000 Subject: [PATCH 152/265] Update to use replaces_state rather than prev_state --- synapse/handlers/federation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 513ec9a5e..f0448a05d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -386,8 +386,10 @@ class FederationHandler(BaseHandler): event = yield self.store.get_event(event_id) if hasattr(event, "state_key"): # Get previous state - if hasattr(event, "prev_state") and event.prev_state: - prev_event = yield self.store.get_event(event.prev_state) + if hasattr(event, "replaces_state") and event.replaces_state: + prev_event = yield self.store.get_event( + event.replaces_state + ) results[(event.type, event.state_key)] = prev_event else: del results[(event.type, event.state_key)] From 4b256cab317ab02da549ce64a33911743f1b9d6f Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 6 Nov 2014 16:48:01 +0000 Subject: [PATCH 153/265] Don't cache isWebRTCSupported because whether webRTC is supported might change part-way through the page's lifecycle if your webrtc support comes from some kind of injected content script (hello OpenWebRTC Sarafi extension) --- syweb/webclient/components/matrix/matrix-call.js | 9 +++------ .../components/matrix/matrix-phone-service.js | 2 +- syweb/webclient/index.html | 2 +- syweb/webclient/room/room.html | 12 ++++++------ 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/syweb/webclient/components/matrix/matrix-call.js b/syweb/webclient/components/matrix/matrix-call.js index 465b2b780..c13083298 100644 --- a/syweb/webclient/components/matrix/matrix-call.js +++ b/syweb/webclient/components/matrix/matrix-call.js @@ -40,14 +40,11 @@ window.RTCPeerConnection = window.RTCPeerConnection || window.webkitRTCPeerConne window.RTCSessionDescription = window.RTCSessionDescription || window.webkitRTCSessionDescription || window.mozRTCSessionDescription; window.RTCIceCandidate = window.RTCIceCandidate || window.webkitRTCIceCandidate || window.mozRTCIceCandidate; -// Returns true if the browser supports all required features to make WebRTC call -var isWebRTCSupported = function () { - return !!(navigator.getUserMedia || window.RTCPeerConnection || window.RTCSessionDescription || window.RTCIceCandidate); -}; - angular.module('MatrixCall', []) .factory('MatrixCall', ['matrixService', 'matrixPhoneService', 'modelService', '$rootScope', '$timeout', function MatrixCallFactory(matrixService, matrixPhoneService, modelService, $rootScope, $timeout) { - $rootScope.isWebRTCSupported = isWebRTCSupported(); + $rootScope.isWebRTCSupported = function () { + return !!(navigator.getUserMedia || window.RTCPeerConnection || window.RTCSessionDescription || window.RTCIceCandidate); + }; var MatrixCall = function(room_id) { this.room_id = room_id; diff --git a/syweb/webclient/components/matrix/matrix-phone-service.js b/syweb/webclient/components/matrix/matrix-phone-service.js index 06465ed82..55dbbf522 100644 --- a/syweb/webclient/components/matrix/matrix-phone-service.js +++ b/syweb/webclient/components/matrix/matrix-phone-service.js @@ -60,7 +60,7 @@ angular.module('matrixPhoneService', []) var MatrixCall = $injector.get('MatrixCall'); var call = new MatrixCall(event.room_id); - if (!isWebRTCSupported()) { + if (!$rootScope.isWebRTCSupported()) { console.log("Incoming call ID "+msg.call_id+" but this browser doesn't support WebRTC"); // don't hang up the call: there could be other clients connected that do support WebRTC and declining the // the call on their behalf would be really annoying. diff --git a/syweb/webclient/index.html b/syweb/webclient/index.html index 3ed968a5e..992e8d337 100644 --- a/syweb/webclient/index.html +++ b/syweb/webclient/index.html @@ -85,7 +85,7 @@
- + diff --git a/syweb/webclient/room/room.html b/syweb/webclient/room/room.html index ca5669a73..e59cc30ed 100644 --- a/syweb/webclient/room/room.html +++ b/syweb/webclient/room/room.html @@ -182,8 +182,8 @@ (msg.content.formatted_body | unsanitizedLinky) : (msg.content.msgtype === 'm.text' && msg.type === 'm.room.message') ? (msg.content.body | linky:'_blank') : '' "/> - Outgoing Call{{ isWebRTCSupported ? '' : ' (But your browser does not support VoIP)' }} - Incoming Call{{ isWebRTCSupported ? '' : ' (But your browser does not support VoIP)' }} + Outgoing Call{{ isWebRTCSupported() ? '' : ' (But your browser does not support VoIP)' }} + Incoming Call{{ isWebRTCSupported() ? '' : ' (But your browser does not support VoIP)' }}
@@ -248,15 +248,15 @@ From 1a62f1299dcf986cd70573b77ed2c7f05afa28f9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 6 Nov 2014 16:55:15 +0000 Subject: [PATCH 154/265] Detect call type by examining the SDP always rather than just in Firefox as it seems Chrome's behaviour is the odd one out here. --- syweb/webclient/components/matrix/matrix-call.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/syweb/webclient/components/matrix/matrix-call.js b/syweb/webclient/components/matrix/matrix-call.js index c13083298..063164994 100644 --- a/syweb/webclient/components/matrix/matrix-call.js +++ b/syweb/webclient/components/matrix/matrix-call.js @@ -175,7 +175,8 @@ angular.module('MatrixCall', []) this.state = 'ringing'; this.direction = 'inbound'; - if (window.mozRTCPeerConnection) { + // This also applied to the Safari OpenWebRTC extension so let's just do this all the time at least for now + //if (window.mozRTCPeerConnection) { // firefox's RTCPeerConnection doesn't add streams until it starts getting media on them // so we need to figure out whether a video channel has been offered by ourselves. if (this.msg.offer.sdp.indexOf('m=video') > -1) { @@ -183,7 +184,7 @@ angular.module('MatrixCall', []) } else { this.type = 'voice'; } - } + //} var self = this; $timeout(function() { From 351c64e99e5677096f9a2ae2cd7e84dbc1887878 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 16:59:13 +0000 Subject: [PATCH 155/265] Amalgamate all power levels. Remove concept of reqired power levels, something similiar can be done using the new power level event. --- synapse/api/auth.py | 223 +++++++++++---------------------- synapse/api/events/__init__.py | 3 +- synapse/api/events/factory.py | 7 +- synapse/api/events/room.py | 22 ---- synapse/api/events/utils.py | 23 ++-- synapse/handlers/room.py | 52 +++----- synapse/storage/__init__.py | 12 -- synapse/storage/room.py | 157 ----------------------- 8 files changed, 103 insertions(+), 396 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9eb0491c9..462e97bd9 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,8 +21,8 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, - RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, InviteJoinEvent, - RoomCreateEvent, RoomSendEventLevelEvent, RoomAddStateLevelEvent, + RoomJoinRulesEvent, InviteJoinEvent, + RoomCreateEvent, ) from synapse.util.logutils import log_function @@ -51,6 +51,7 @@ class Auth(object): if event.old_state_events is None: # Oh, we don't know what the state of the room was, so we # are trusting that this is allowed (at least for now) + logger.warn("Trusting event: %s", event.event_id) return True if hasattr(event, "outlier") and event.outlier is True: @@ -64,7 +65,7 @@ class Auth(object): return True if event.type == RoomMemberEvent.TYPE: - self._can_replace_state(event) + self._can_send_event(event) allowed = self.is_membership_change_allowed(event) if allowed: logger.debug("Allowing! %s", event) @@ -72,16 +73,7 @@ class Auth(object): logger.debug("Denying! %s", event) return allowed - if not event.type == InviteJoinEvent.TYPE: - self.check_event_sender_in_room(event) - - if is_state: - # TODO (erikj): This really only should be called for *new* - # state - self._can_add_state(event) - self._can_replace_state(event) - else: - self._can_send_event(event) + self._can_send_event(event) if event.type == RoomPowerLevelsEvent.TYPE: self._check_power_levels(event) @@ -239,21 +231,21 @@ class Auth(object): power_level_event = event.old_state_events.get(key) level = None if power_level_event: - level = power_level_event.content.get(user_id) + level = power_level_event.content.get("users", {}).get(user_id) if not level: - level = power_level_event.content.get("default", 0) + level = power_level_event.content.get("users_default", 0) return level def _get_ops_level_from_event_state(self, event): - key = (RoomOpsPowerLevelsEvent.TYPE, "", ) - ops_event = event.old_state_events.get(key) + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) - if ops_event: + if power_level_event: return ( - ops_event.content.get("ban_level"), - ops_event.content.get("kick_level"), - ops_event.content.get("redact_level"), + power_level_event.content.get("ban", 50), + power_level_event.content.get("kick", 50), + power_level_event.content.get("redact", 50), ) return None, None, None, @@ -325,13 +317,22 @@ class Auth(object): @log_function def _can_send_event(self, event): - key = (RoomSendEventLevelEvent.TYPE, "", ) + key = (RoomPowerLevelsEvent.TYPE, "", ) send_level_event = event.old_state_events.get(key) send_level = None if send_level_event: - send_level = send_level_event.content.get(event.user_id) + send_level = send_level_event.content.get("events", {}).get( + event.type + ) if not send_level: - send_level = send_level_event.content.get("level", 0) + if hasattr(event, "state_key"): + send_level = send_level_event.content.get( + "state_default", 50 + ) + else: + send_level = send_level_event.content.get( + "events_default", 0 + ) if send_level: send_level = int(send_level) @@ -350,85 +351,21 @@ class Auth(object): if user_level < send_level: raise AuthError( - 403, "You don't have permission to post to the room" + 403, "You don't have permission to post that to the room" ) return True - def _can_add_state(self, event): - key = (RoomAddStateLevelEvent.TYPE, "", ) - add_level_event = event.old_state_events.get(key) - add_level = None - if add_level_event: - add_level = add_level_event.content.get(event.user_id) - if not add_level: - add_level = add_level_event.content.get("level", 0) - - if add_level: - add_level = int(add_level) - else: - add_level = 0 - - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - ) - - user_level = int(user_level) - - if user_level < add_level: - raise AuthError( - 403, "You don't have permission to add state to the room" - ) - - return True - - def _can_replace_state(self, event): - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - ) - - if user_level: - user_level = int(user_level) - else: - user_level = 0 - - logger.debug( - "Checking power level for %s, %s", event.user_id, user_level - ) - - key = (event.type, event.state_key, ) - current_state = event.old_state_events.get(key) - - if current_state and hasattr(current_state, "required_power_level"): - req = current_state.required_power_level - - logger.debug("Checked power level for %s, %s", event.user_id, req) - if user_level < req: - raise AuthError( - 403, - "You don't have permission to change that state" - ) - def _check_redaction(self, event): user_level = self._get_power_level_from_event_state( event, event.user_id, ) - if user_level: - user_level = int(user_level) - else: - user_level = 0 - _, _, redact_level = self._get_ops_level_from_event_state( event ) - if not redact_level: - redact_level = 50 - if user_level < redact_level: raise AuthError( 403, @@ -436,14 +373,9 @@ class Auth(object): ) def _check_power_levels(self, event): - for k, v in event.content.items(): - if k == "default": - continue - - # FIXME (erikj): We don't want hsob_Ts in content. - if k == "hsob_ts": - continue - + user_list = event.content.get("users", {}) + # Validate users + for k, v in user_list.items(): try: self.hs.parse_userid(k) except: @@ -459,72 +391,63 @@ class Auth(object): if not current_state: return - else: - current_state = current_state[0] user_level = self._get_power_level_from_event_state( event, event.user_id, ) - if user_level: - user_level = int(user_level) - else: - user_level = 0 + # Check other levels: + levels_to_check = [ + ("users_default", []), + ("events_default", []), + ("ban", []), + ("redact", []), + ("kick", []), + ] - old_list = current_state.content + old_list = current_state.content.get("users") + for user in set(old_list.keys() + user_list.keys()): + levels_to_check.append( + (user, ["users"]) + ) - # FIXME (erikj) - old_people = {k: v for k, v in old_list.items() if k.startswith("@")} - new_people = { - k: v for k, v in event.content.items() - if k.startswith("@") - } + old_list = current_state.content.get("events") + new_list = event.content.get("events") + for ev_id in set(old_list.keys() + new_list.keys()): + levels_to_check.append( + (ev_id, ["events"]) + ) - removed = set(old_people.keys()) - set(new_people.keys()) - added = set(new_people.keys()) - set(old_people.keys()) - same = set(old_people.keys()) & set(new_people.keys()) + old_state = current_state.content + new_state = event.content - for r in removed: - if int(old_list[r]) > user_level: - raise AuthError( - 403, - "You don't have permission to remove user: %s" % (r, ) - ) + for level_to_check, dir in levels_to_check: + old_loc = old_state + for d in dir: + old_loc = old_loc.get(d, {}) - for n in added: - if int(event.content[n]) > user_level: + new_loc = new_state + for d in dir: + new_loc = new_loc.get(d, {}) + + if level_to_check in old_loc: + old_level = int(old_loc[level_to_check]) + else: + old_level = None + + if level_to_check in new_loc: + new_level = int(new_loc[level_to_check]) + else: + new_level = None + + if new_level is not None and old_level is not None: + if new_level == old_level: + continue + + if old_level > user_level or new_level > user_level: raise AuthError( 403, "You don't have permission to add ops level greater " "than your own" ) - - for s in same: - if int(event.content[s]) != int(old_list[s]): - if int(event.content[s]) > user_level: - raise AuthError( - 403, - "You don't have permission to add ops level greater " - "than your own" - ) - - if "default" in old_list: - old_default = int(old_list["default"]) - - if old_default > user_level: - raise AuthError( - 403, - "You don't have permission to add ops level greater than " - "your own" - ) - - if "default" in event.content: - new_default = int(event.content["default"]) - - if new_default > user_level: - raise AuthError( - 403, - "You don't have permission to add ops level greater " - "than your own" - ) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 98a66144e..84d3a9836 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -56,12 +56,12 @@ class SynapseEvent(JsonEncodedObject): "user_id", # sender/initiator "content", # HTTP body, JSON "state_key", - "required_power_level", "age_ts", "prev_content", "replaces_state", "redacted_because", "origin_server_ts", + "auth_chains", ] internal_keys = [ @@ -70,7 +70,6 @@ class SynapseEvent(JsonEncodedObject): "destinations", "origin", "outlier", - "power_level", "redacted", "prev_events", "hashes", diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 9134c82ef..a1ec708a8 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -16,8 +16,8 @@ from synapse.api.events.room import ( RoomTopicEvent, MessageEvent, RoomMemberEvent, FeedbackEvent, InviteJoinEvent, RoomConfigEvent, RoomNameEvent, GenericEvent, - RoomPowerLevelsEvent, RoomJoinRulesEvent, RoomOpsPowerLevelsEvent, - RoomCreateEvent, RoomAddStateLevelEvent, RoomSendEventLevelEvent, + RoomPowerLevelsEvent, RoomJoinRulesEvent, + RoomCreateEvent, RoomRedactionEvent, ) @@ -39,9 +39,6 @@ class EventFactory(object): RoomPowerLevelsEvent, RoomJoinRulesEvent, RoomCreateEvent, - RoomAddStateLevelEvent, - RoomSendEventLevelEvent, - RoomOpsPowerLevelsEvent, RoomRedactionEvent, ] diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index cd936074f..25bc88370 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -153,28 +153,6 @@ class RoomPowerLevelsEvent(SynapseStateEvent): def get_content_template(self): return {} - -class RoomAddStateLevelEvent(SynapseStateEvent): - TYPE = "m.room.add_state_level" - - def get_content_template(self): - return {} - - -class RoomSendEventLevelEvent(SynapseStateEvent): - TYPE = "m.room.send_event_level" - - def get_content_template(self): - return {} - - -class RoomOpsPowerLevelsEvent(SynapseStateEvent): - TYPE = "m.room.ops_levels" - - def get_content_template(self): - return {} - - class RoomAliasesEvent(SynapseStateEvent): TYPE = "m.room.aliases" diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index 31601fd3a..5fc79105b 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -15,7 +15,6 @@ from .room import ( RoomMemberEvent, RoomJoinRulesEvent, RoomPowerLevelsEvent, - RoomAddStateLevelEvent, RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent, RoomAliasesEvent, RoomCreateEvent, ) @@ -52,17 +51,17 @@ def _prune_event_or_pdu(event_type, event): elif event_type == RoomJoinRulesEvent.TYPE: add_fields("join_rule") elif event_type == RoomPowerLevelsEvent.TYPE: - # TODO: Actually check these are valid user_ids etc. - add_fields("default") - for k, v in event.content.items(): - if k.startswith("@") and isinstance(v, (int, long)): - new_content[k] = v - elif event_type == RoomAddStateLevelEvent.TYPE: - add_fields("level") - elif event_type == RoomSendEventLevelEvent.TYPE: - add_fields("level") - elif event_type == RoomOpsPowerLevelsEvent.TYPE: - add_fields("kick_level", "ban_level", "redact_level") + add_fields( + "users", + "users_default", + "events", + "events_default", + "events_default", + "state_default", + "ban", + "kick", + "redact", + ) elif event_type == RoomAliasesEvent.TYPE: add_fields("aliases") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 55c893eb5..42a6c9f9b 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -21,8 +21,7 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import StoreError, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomCreateEvent, RoomPowerLevelsEvent, - RoomJoinRulesEvent, RoomAddStateLevelEvent, RoomTopicEvent, - RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent, RoomNameEvent, + RoomTopicEvent, RoomNameEvent, RoomJoinRulesEvent, ) from synapse.util import stringutils from ._base import BaseHandler @@ -139,7 +138,6 @@ class RoomCreationHandler(BaseHandler): etype=RoomNameEvent.TYPE, room_id=room_id, user_id=user_id, - required_power_level=50, content={"name": name}, ) @@ -151,7 +149,6 @@ class RoomCreationHandler(BaseHandler): etype=RoomTopicEvent.TYPE, room_id=room_id, user_id=user_id, - required_power_level=50, content={"topic": topic}, ) @@ -196,7 +193,6 @@ class RoomCreationHandler(BaseHandler): event_keys = { "room_id": room_id, "user_id": creator.to_string(), - "required_power_level": 100, } def create(etype, **content): @@ -213,7 +209,21 @@ class RoomCreationHandler(BaseHandler): power_levels_event = self.event_factory.create_event( etype=RoomPowerLevelsEvent.TYPE, - content={creator.to_string(): 100, "default": 0}, + content={ + "users": { + creator.to_string(): 100, + }, + "users_default": 0, + "events": { + RoomNameEvent.TYPE: 100, + RoomPowerLevelsEvent.TYPE: 100, + }, + "events_default": 0, + "state_default": 50, + "ban": 50, + "kick": 50, + "redact": 50 + }, **event_keys ) @@ -223,30 +233,10 @@ class RoomCreationHandler(BaseHandler): join_rule=join_rule, ) - add_state_event = create( - etype=RoomAddStateLevelEvent.TYPE, - level=100, - ) - - send_event = create( - etype=RoomSendEventLevelEvent.TYPE, - level=0, - ) - - ops = create( - etype=RoomOpsPowerLevelsEvent.TYPE, - ban_level=50, - kick_level=50, - redact_level=50, - ) - return [ creation_event, power_levels_event, join_rules_event, - add_state_event, - send_event, - ops, ] @@ -388,16 +378,6 @@ class RoomMemberHandler(BaseHandler): else: # This is not a JOIN, so we can handle it normally. - # If we're banning someone, set a req power level - if event.membership == Membership.BAN: - if not hasattr(event, "required_power_level") or event.required_power_level is None: - # Add some default required_power_level - user_level = yield self.store.get_power_level( - event.room_id, - event.user_id, - ) - event.required_power_level = user_level - if prev_state and prev_state.membership == event.membership: # double same action, treat this event as a NOOP. defer.returnValue({}) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2d62fc2ed..2a1970914 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -17,13 +17,9 @@ from twisted.internet import defer from synapse.api.events.room import ( RoomMemberEvent, RoomTopicEvent, FeedbackEvent, -# RoomConfigEvent, RoomNameEvent, RoomJoinRulesEvent, RoomPowerLevelsEvent, - RoomAddStateLevelEvent, - RoomSendEventLevelEvent, - RoomOpsPowerLevelsEvent, RoomRedactionEvent, ) @@ -166,14 +162,6 @@ class DataStore(RoomMemberStore, RoomStore, self._store_room_topic_txn(txn, event) elif event.type == RoomJoinRulesEvent.TYPE: self._store_join_rule(txn, event) - elif event.type == RoomPowerLevelsEvent.TYPE: - self._store_power_levels(txn, event) - elif event.type == RoomAddStateLevelEvent.TYPE: - self._store_add_state_level(txn, event) - elif event.type == RoomSendEventLevelEvent.TYPE: - self._store_send_event_level(txn, event) - elif event.type == RoomOpsPowerLevelsEvent.TYPE: - self._store_ops_level(txn, event) elif event.type == RoomRedactionEvent.TYPE: self._store_redaction(txn, event) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 7e48ce9cc..0c83c11ad 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -148,85 +148,6 @@ class RoomStore(SQLBaseStore): else: defer.returnValue(None) - def get_power_level(self, room_id, user_id): - return self.runInteraction( - "get_power_level", - self._get_power_level, - room_id, user_id, - ) - - def _get_power_level(self, txn, room_id, user_id): - sql = ( - "SELECT level FROM room_power_levels as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? AND r.user_id = ? " - ) - - rows = txn.execute(sql, (room_id, user_id,)).fetchall() - - if len(rows) == 1: - return rows[0][0] - - sql = ( - "SELECT level FROM room_default_levels as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? " - ) - - rows = txn.execute(sql, (room_id,)).fetchall() - - if len(rows) == 1: - return rows[0][0] - else: - return None - - def get_ops_levels(self, room_id): - return self.runInteraction( - "get_ops_levels", - self._get_ops_levels, - room_id, - ) - - def _get_ops_levels(self, txn, room_id): - sql = ( - "SELECT ban_level, kick_level, redact_level " - "FROM room_ops_levels as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? " - ) - - rows = txn.execute(sql, (room_id,)).fetchall() - - if len(rows) == 1: - return OpsLevel(rows[0][0], rows[0][1], rows[0][2]) - else: - return OpsLevel(None, None) - - def get_add_state_level(self, room_id): - return self._get_level_from_table("room_add_state_levels", room_id) - - def get_send_event_level(self, room_id): - return self._get_level_from_table("room_send_event_levels", room_id) - - @defer.inlineCallbacks - def _get_level_from_table(self, table, room_id): - sql = ( - "SELECT level FROM %(table)s as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? " - ) % {"table": table} - - rows = yield self._execute(None, sql, room_id) - - if len(rows) == 1: - defer.returnValue(rows[0][0]) - else: - defer.returnValue(None) - def _store_room_topic_txn(self, txn, event): self._simple_insert_txn( txn, @@ -260,84 +181,6 @@ class RoomStore(SQLBaseStore): }, ) - def _store_power_levels(self, txn, event): - for user_id, level in event.content.items(): - if user_id == "default": - self._simple_insert_txn( - txn, - "room_default_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "level": level, - }, - ) - else: - self._simple_insert_txn( - txn, - "room_power_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "user_id": user_id, - "level": level - }, - ) - - def _store_default_level(self, txn, event): - self._simple_insert_txn( - txn, - "room_default_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "level": event.content["default_level"], - }, - ) - - def _store_add_state_level(self, txn, event): - self._simple_insert_txn( - txn, - "room_add_state_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "level": event.content["level"], - }, - ) - - def _store_send_event_level(self, txn, event): - self._simple_insert_txn( - txn, - "room_send_event_levels", - { - "event_id": event.event_id, - "room_id": event.room_id, - "level": event.content["level"], - }, - ) - - def _store_ops_level(self, txn, event): - content = { - "event_id": event.event_id, - "room_id": event.room_id, - } - - if "kick_level" in event.content: - content["kick_level"] = event.content["kick_level"] - - if "ban_level" in event.content: - content["ban_level"] = event.content["ban_level"] - - if "redact_level" in event.content: - content["redact_level"] = event.content["redact_level"] - - self._simple_insert_txn( - txn, - "room_ops_levels", - content, - ) - class RoomsTable(Table): table_name = "rooms" From 46de65cab9e449589e49b6cf7d37d4921363d934 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 17:15:09 +0000 Subject: [PATCH 156/265] Don't query the DB for user power levels --- synapse/state.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 97a8160a3..e2fd48bda 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor +from synapse.api.events.room import RoomPowerLevelsEvent from collections import namedtuple @@ -141,21 +142,26 @@ class StateHandler(object): defer.returnValue(new_state) + def _get_power_level_from_event_state(self, event, user_id): + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) + level = None + if power_level_event: + level = power_level_event.content.get("users", {}).get(user_id) + if not level: + level = power_level_event.content.get("users_default", 0) + + return level + @defer.inlineCallbacks @log_function def _resolve_state_events(self, events): curr_events = events - new_powers_deferreds = [] - for e in curr_events: - new_powers_deferreds.append( - self.store.get_power_level(e.room_id, e.user_id) - ) - - new_powers = yield defer.gatherResults( - new_powers_deferreds, - consumeErrors=True - ) + new_powers = [ + self._get_power_level_from_event_state(e, e.user_id) + for e in curr_events + ] new_powers = [ int(p) if p else 0 for p in new_powers From 8421cabb9db89a9d4d2158f1a4828b15c4ce18e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 17:26:13 +0000 Subject: [PATCH 157/265] Neaten things up a bit --- synapse/api/auth.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 462e97bd9..bb25c4ec5 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -58,14 +58,13 @@ class Auth(object): # TODO (erikj): Auth for outliers is done differently. return True - is_state = hasattr(event, "state_key") - if event.type == RoomCreateEvent.TYPE: # FIXME return True + self._can_send_event(event) + if event.type == RoomMemberEvent.TYPE: - self._can_send_event(event) allowed = self.is_membership_change_allowed(event) if allowed: logger.debug("Allowing! %s", event) @@ -73,8 +72,6 @@ class Auth(object): logger.debug("Denying! %s", event) return allowed - self._can_send_event(event) - if event.type == RoomPowerLevelsEvent.TYPE: self._check_power_levels(event) From bf6b72eb558cca94e209a541188079750bfefea0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Nov 2014 18:42:18 +0000 Subject: [PATCH 158/265] Start implementing auth chains --- synapse/api/auth.py | 3 +- synapse/api/events/__init__.py | 2 +- synapse/handlers/_base.py | 59 +++++++++++++++++++++++++- synapse/storage/__init__.py | 12 +++++- synapse/storage/_base.py | 2 + synapse/storage/event_federation.py | 21 +++++++++ synapse/storage/schema/event_edges.sql | 10 +++++ synapse/storage/signatures.py | 12 ++++++ 8 files changed, 115 insertions(+), 6 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index bb25c4ec5..e1302553d 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -21,8 +21,7 @@ from synapse.api.constants import Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError from synapse.api.events.room import ( RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, - RoomJoinRulesEvent, InviteJoinEvent, - RoomCreateEvent, + RoomJoinRulesEvent, RoomCreateEvent, ) from synapse.util.logutils import log_function diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 84d3a9836..513a48f56 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -61,7 +61,7 @@ class SynapseEvent(JsonEncodedObject): "replaces_state", "redacted_because", "origin_server_ts", - "auth_chains", + "auth_events", ] internal_keys = [ diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 509f7b550..2613fa7fc 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -14,11 +14,15 @@ # limitations under the License. from twisted.internet import defer + from synapse.api.errors import LimitExceededError - from synapse.util.async import run_on_reactor - from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.api.events.room import ( + RoomCreateEvent, RoomMemberEvent, RoomPowerLevelsEvent, RoomJoinRulesEvent, +) +from synapse.api.constants import Membership, JoinRules +from syutil.base64util import encode_base64 import logging @@ -55,6 +59,53 @@ class BaseHandler(object): retry_after_ms=int(1000*(time_allowed - time_now)), ) + @defer.inlineCallbacks + def _add_auth(self, event): + if event.type == RoomCreateEvent.TYPE: + event.auth_events = [] + return + + auth_events = [] + + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) + + if power_level_event: + auth_events.append(power_level_event.event_id) + + key = (RoomJoinRulesEvent.TYPE, "", ) + join_rule_event = event.old_state_events.get(key) + + key = (RoomMemberEvent.TYPE, event.user_id, ) + member_event = event.old_state_events.get(key) + + if join_rule_event: + join_rule = join_rule_event.content.get("join_rule") + is_public = join_rule == JoinRules.PUBLIC if join_rule else False + + if event.type == RoomMemberEvent.TYPE: + if event.content["membership"] == Membership.JOIN: + if is_public: + auth_events.append(join_rule_event.event_id) + elif member_event: + auth_events.append(member_event.event_id) + + if member_event: + if member_event.content["membership"] == Membership.JOIN: + auth_events.append(member_event.event_id) + + hashes = yield self.store.get_event_reference_hashes( + auth_events + ) + hashes = [ + { + k: encode_base64(v) for k, v in h.items() + if k == "sha256" + } + for h in hashes + ] + event.auth_events = zip(auth_events, hashes) + @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], extra_users=[], suppress_auth=False): @@ -64,6 +115,8 @@ class BaseHandler(object): yield self.state_handler.annotate_state_groups(event) + yield self._add_auth(event) + logger.debug("Signing event...") add_hashes_and_signatures( @@ -76,6 +129,8 @@ class BaseHandler(object): logger.debug("Authing...") self.auth.check(event, raises=True) logger.debug("Authed") + else: + logger.debug("Suppressed auth.") yield self.store.persist_event(event) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2a1970914..48ad4d864 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -19,7 +19,6 @@ from synapse.api.events.room import ( RoomMemberEvent, RoomTopicEvent, FeedbackEvent, RoomNameEvent, RoomJoinRulesEvent, - RoomPowerLevelsEvent, RoomRedactionEvent, ) @@ -302,6 +301,17 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, prev_event_id, alg, hash_bytes ) + for auth_id, _ in event.auth_events: + self._simple_insert_txn( + txn, + table="event_auth", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "auth_id": auth_id, + }, + ) + (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) self._store_event_reference_hash_txn( txn, event.event_id, ref_alg, ref_hash_bytes diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7821fc472..9aa404695 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -474,6 +474,8 @@ class SQLBaseStore(object): if is_state == 0 ] + ev.auth_events = self._get_auth_events(txn, ev.event_id) + if hasattr(ev, "state_key"): ev.prev_state = [ (e_id, h) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 180a76413..86c68ebf8 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -139,6 +139,27 @@ class EventFederationStore(SQLBaseStore): return results + def _get_auth_events(self, txn, event_id): + auth_ids = self._simple_select_onecol_txn( + txn, + table="event_auth", + keyvalues={ + "event_id": event_id, + }, + retcol="auth_id", + ) + + results = [] + for auth_id in auth_ids: + hashes = self._get_event_reference_hashes_txn(txn, auth_id) + prev_hashes = { + k: encode_base64(v) for k, v in hashes.items() + if k == "sha256" + } + results.append((auth_id, prev_hashes)) + + return results + def get_min_depth(self, room_id): return self.runInteraction( "get_min_depth", diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/event_edges.sql index 51695826a..be1c72a77 100644 --- a/synapse/storage/schema/event_edges.sql +++ b/synapse/storage/schema/event_edges.sql @@ -63,3 +63,13 @@ CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( ); CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + +CREATE TABLE IF NOT EXISTS event_auth( + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, + CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) +); + +CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index b4b3d5d7e..84a49088a 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -55,6 +55,18 @@ class SignatureStore(SQLBaseStore): or_ignore=True, ) + def get_event_reference_hashes(self, event_ids): + def f(txn): + return [ + self._get_event_reference_hashes_txn(txn, ev) + for ev in event_ids + ] + + return self.runInteraction( + "get_event_reference_hashes", + f + ) + def _get_event_reference_hashes_txn(self, txn, event_id): """Get all the hashes for a given PDU. Args: From 8b0e96474b3a18866f28f112de7e56979b401768 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 10:42:31 +0000 Subject: [PATCH 159/265] Implement method to get auth_chain from a given event_id --- synapse/storage/event_federation.py | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 86c68ebf8..7140ea3d5 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -24,6 +24,41 @@ logger = logging.getLogger(__name__) class EventFederationStore(SQLBaseStore): + def get_auth_chain(self, event_id): + return self.runInteraction( + "get_auth_chain", + self._get_auth_chain_txn, + event_id + ) + + def _get_auth_chain_txn(self, txn, event_id): + results = set([event_id]) + + front = set([event_id]) + while front: + for ev_id in front: + new_front = set() + auth_ids = self._simple_select_onecol_txn( + txn, + table="event_auth", + keyvalues={ + "event_id": ev_id, + }, + retcol="auth_id", + ) + + new_front.update(auth_ids) + front = new_front + new_front.clear() + + sql = "SELECT * FROM events WHERE event_id = ?" + rows = [] + for ev_id in results: + c = txn.execute(sql, (ev_id,)) + rows.extend(self.cursor_to_dict(c)) + + return self._parse_events_txn(txn, rows) + def get_oldest_events_in_room(self, room_id): return self.runInteraction( "get_oldest_events_in_room", From 49948d72f3c76f0ca32b844955d3add7922180ea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 10:42:44 +0000 Subject: [PATCH 160/265] Fix joining over federation --- synapse/api/auth.py | 53 +++++++++++++++++++++++++++++++-- synapse/api/events/__init__.py | 2 +- synapse/handlers/_base.py | 54 +--------------------------------- synapse/handlers/federation.py | 1 + synapse/storage/__init__.py | 1 + 5 files changed, 55 insertions(+), 56 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e1302553d..d4f284bd6 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -24,6 +24,7 @@ from synapse.api.events.room import ( RoomJoinRulesEvent, RoomCreateEvent, ) from synapse.util.logutils import log_function +from syutil.base64util import encode_base64 import logging @@ -61,8 +62,6 @@ class Auth(object): # FIXME return True - self._can_send_event(event) - if event.type == RoomMemberEvent.TYPE: allowed = self.is_membership_change_allowed(event) if allowed: @@ -71,6 +70,8 @@ class Auth(object): logger.debug("Denying! %s", event) return allowed + self._can_send_event(event) + if event.type == RoomPowerLevelsEvent.TYPE: self._check_power_levels(event) @@ -311,6 +312,54 @@ class Auth(object): def is_server_admin(self, user): return self.store.is_server_admin(user) + @defer.inlineCallbacks + def add_auth_events(self, event): + if event.type == RoomCreateEvent.TYPE: + event.auth_events = [] + return + + auth_events = [] + + key = (RoomPowerLevelsEvent.TYPE, "", ) + power_level_event = event.old_state_events.get(key) + + if power_level_event: + auth_events.append(power_level_event.event_id) + + key = (RoomJoinRulesEvent.TYPE, "", ) + join_rule_event = event.old_state_events.get(key) + + key = (RoomMemberEvent.TYPE, event.user_id, ) + member_event = event.old_state_events.get(key) + + if join_rule_event: + join_rule = join_rule_event.content.get("join_rule") + is_public = join_rule == JoinRules.PUBLIC if join_rule else False + + if event.type == RoomMemberEvent.TYPE: + if event.content["membership"] == Membership.JOIN: + if is_public: + auth_events.append(join_rule_event.event_id) + elif member_event: + auth_events.append(member_event.event_id) + + if member_event: + if member_event.content["membership"] == Membership.JOIN: + auth_events.append(member_event.event_id) + + hashes = yield self.store.get_event_reference_hashes( + auth_events + ) + hashes = [ + { + k: encode_base64(v) for k, v in h.items() + if k == "sha256" + } + for h in hashes + ] + event.auth_events = zip(auth_events, hashes) + + @log_function def _can_send_event(self, event): key = (RoomPowerLevelsEvent.TYPE, "", ) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 513a48f56..e5980c4be 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -61,7 +61,6 @@ class SynapseEvent(JsonEncodedObject): "replaces_state", "redacted_because", "origin_server_ts", - "auth_events", ] internal_keys = [ @@ -75,6 +74,7 @@ class SynapseEvent(JsonEncodedObject): "hashes", "signatures", "prev_state", + "auth_events", ] required_keys = [ diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 2613fa7fc..f63028003 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,11 +18,6 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.api.events.room import ( - RoomCreateEvent, RoomMemberEvent, RoomPowerLevelsEvent, RoomJoinRulesEvent, -) -from synapse.api.constants import Membership, JoinRules -from syutil.base64util import encode_base64 import logging @@ -59,53 +54,6 @@ class BaseHandler(object): retry_after_ms=int(1000*(time_allowed - time_now)), ) - @defer.inlineCallbacks - def _add_auth(self, event): - if event.type == RoomCreateEvent.TYPE: - event.auth_events = [] - return - - auth_events = [] - - key = (RoomPowerLevelsEvent.TYPE, "", ) - power_level_event = event.old_state_events.get(key) - - if power_level_event: - auth_events.append(power_level_event.event_id) - - key = (RoomJoinRulesEvent.TYPE, "", ) - join_rule_event = event.old_state_events.get(key) - - key = (RoomMemberEvent.TYPE, event.user_id, ) - member_event = event.old_state_events.get(key) - - if join_rule_event: - join_rule = join_rule_event.content.get("join_rule") - is_public = join_rule == JoinRules.PUBLIC if join_rule else False - - if event.type == RoomMemberEvent.TYPE: - if event.content["membership"] == Membership.JOIN: - if is_public: - auth_events.append(join_rule_event.event_id) - elif member_event: - auth_events.append(member_event.event_id) - - if member_event: - if member_event.content["membership"] == Membership.JOIN: - auth_events.append(member_event.event_id) - - hashes = yield self.store.get_event_reference_hashes( - auth_events - ) - hashes = [ - { - k: encode_base64(v) for k, v in h.items() - if k == "sha256" - } - for h in hashes - ] - event.auth_events = zip(auth_events, hashes) - @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], extra_users=[], suppress_auth=False): @@ -115,7 +63,7 @@ class BaseHandler(object): yield self.state_handler.annotate_state_groups(event) - yield self._add_auth(event) + yield self.auth.add_auth_events(event) logger.debug("Signing event...") diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f0448a05d..09593303a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -317,6 +317,7 @@ class FederationHandler(BaseHandler): snapshot.fill_out_prev_events(event) yield self.state_handler.annotate_state_groups(event) + yield self.auth.add_auth_events(event) self.auth.check(event, raises=True) pdu = self.pdu_codec.pdu_from_event(event) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 48ad4d864..96adf20c8 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -310,6 +310,7 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "auth_id": auth_id, }, + or_ignore=True, ) (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) From 16a0815fac750863588c3c1f72c5e445d14bbf43 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 11:21:20 +0000 Subject: [PATCH 161/265] Fix bug in _get_auth_chain_txn --- synapse/storage/event_federation.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7140ea3d5..d66a49e9f 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,24 +32,21 @@ class EventFederationStore(SQLBaseStore): ) def _get_auth_chain_txn(self, txn, event_id): - results = set([event_id]) + results = set() + + base_sql = ( + "SELECT auth_id FROM event_auth WHERE %s" + ) front = set([event_id]) while front: - for ev_id in front: - new_front = set() - auth_ids = self._simple_select_onecol_txn( - txn, - table="event_auth", - keyvalues={ - "event_id": ev_id, - }, - retcol="auth_id", - ) + sql = base_sql % ( + " OR ".join(["event_id=?"] * len(front)), + ) - new_front.update(auth_ids) - front = new_front - new_front.clear() + txn.execute(sql, list(front)) + front = [r[0] for r in txn.fetchall()] + results.update(front) sql = "SELECT * FROM events WHERE event_id = ?" rows = [] From 3b4dec442da51c6c999dd946db6ea6ce5f07ff0c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 11:22:12 +0000 Subject: [PATCH 162/265] Return auth chain when handling send_join --- synapse/federation/replication.py | 20 +++++++++++++++----- synapse/handlers/federation.py | 15 ++++++++++++--- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 92a9678e2..d1eddf249 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -426,8 +426,12 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_send_join_request(self, origin, content): pdu = Pdu(**content) - state = yield self.handler.on_send_join_request(origin, pdu) - defer.returnValue((200, self._transaction_from_pdus(state).get_dict())) + res_pdus = yield self.handler.on_send_join_request(origin, pdu) + + defer.returnValue((200, { + "state": [p.get_dict() for p in res_pdus["state"]], + "auth_chain": [p.get_dict() for p in res_pdus["auth_chain"]], + })) @defer.inlineCallbacks def make_join(self, destination, context, user_id): @@ -451,11 +455,17 @@ class ReplicationLayer(object): ) logger.debug("Got content: %s", content) - pdus = [Pdu(outlier=True, **p) for p in content.get("pdus", [])] - for pdu in pdus: + state = [Pdu(outlier=True, **p) for p in content.get("state", [])] + for pdu in state: yield self._handle_new_pdu(destination, pdu) - defer.returnValue(pdus) + auth_chain = [ + Pdu(outlier=True, **p) for p in content.get("auth_chain", []) + ] + for pdu in auth_chain: + yield self._handle_new_pdu(destination, pdu) + + defer.returnValue(state) @log_function def _get_persisted_pdu(self, event_id): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 09593303a..c193da12b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -366,10 +366,19 @@ class FederationHandler(BaseHandler): yield self.replication_layer.send_pdu(new_pdu) - defer.returnValue([ + auth_chain = yield self.store.get_auth_chain(event.event_id) + pdu_auth_chain = [ self.pdu_codec.pdu_from_event(e) - for e in event.state_events.values() - ]) + for e in auth_chain + ] + + defer.returnValue({ + "state": [ + self.pdu_codec.pdu_from_event(e) + for e in event.state_events.values() + ], + "auth_chain": pdu_auth_chain, + }) @defer.inlineCallbacks def get_state_for_pdu(self, event_id): From 97a096b507b92d70a2e07d49122b4f5d93b7dac4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 11:36:40 +0000 Subject: [PATCH 163/265] Add hash of current state to events --- synapse/api/events/__init__.py | 1 + synapse/crypto/event_signing.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index e5980c4be..8d65c29ac 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -75,6 +75,7 @@ class SynapseEvent(JsonEncodedObject): "signatures", "prev_state", "auth_events", + "state_hash", ] required_keys = [ diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index de5d2e746..7d800615f 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -45,7 +45,7 @@ def check_event_content_hash(event, hash_algorithm=hashlib.sha256): def _compute_content_hash(event, hash_algorithm): event_json = event.get_full_dict() - #TODO: We need to sign the JSON that is going out via fedaration. + # TODO: We need to sign the JSON that is going out via fedaration. event_json.pop("age_ts", None) event_json.pop("unsigned", None) event_json.pop("signatures", None) @@ -81,6 +81,15 @@ def compute_event_signature(event, signature_name, signing_key): def add_hashes_and_signatures(event, signature_name, signing_key, hash_algorithm=hashlib.sha256): + if hasattr(event, "old_state_events"): + state_json_bytes = encode_canonical_json( + [e.event_id for e in event.old_state_events.values()] + ) + hashed = hash_algorithm(state_json_bytes) + event.state_hash = { + hashed.name: encode_base64(hashed.digest()) + } + hashed = _compute_content_hash(event, hash_algorithm=hash_algorithm) if not hasattr(event, "hashes"): From 328dab246376814c141095d63e5cef3b4cb52068 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 11:40:38 +0000 Subject: [PATCH 164/265] Remove /context/ request --- synapse/federation/replication.py | 7 ------- synapse/federation/transport.py | 9 --------- 2 files changed, 16 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index d1eddf249..37e7db053 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -272,13 +272,6 @@ class ReplicationLayer(object): defer.returnValue(pdus) - @defer.inlineCallbacks - @log_function - def on_context_pdus_request(self, context): - raise NotImplementedError( - "on_context_pdus_request is a security violation" - ) - @defer.inlineCallbacks @log_function def on_backfill_request(self, context, versions, limit): diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 04ad7e63a..b9f7d54c7 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -403,15 +403,6 @@ class TransportLayer(object): ) ) - self.server.register_path( - "GET", - re.compile("^" + PREFIX + "/context/([^/]*)/$"), - self._with_authentication( - lambda origin, content, query, context: - handler.on_context_pdus_request(context) - ) - ) - # This is when we receive a server-server Query self.server.register_path( "GET", From d2fb2b8095ec7f5d00b51418e84b05a1b23b79b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 13:41:00 +0000 Subject: [PATCH 165/265] Implement invite part of invite join dance --- synapse/federation/replication.py | 15 ++++++++++++- synapse/handlers/_base.py | 13 ++++++++++- synapse/handlers/federation.py | 37 +++++++++++++++++++++++++++++++ synapse/handlers/room.py | 30 +++++++++---------------- 4 files changed, 74 insertions(+), 21 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 37e7db053..e358de942 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -413,7 +413,7 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_invite_request(self, origin, content): pdu = Pdu(**content) - ret_pdu = yield self.handler.on_send_join_request(origin, pdu) + ret_pdu = yield self.handler.on_invite_request(origin, pdu) defer.returnValue((200, ret_pdu.get_dict())) @defer.inlineCallbacks @@ -460,6 +460,19 @@ class ReplicationLayer(object): defer.returnValue(state) + @defer.inlineCallbacks + def send_invite(self, destination, context, event_id, pdu): + code, pdu_dict = yield self.transport_layer.send_invite( + destination=destination, + context=context, + event_id=event_id, + content=pdu.get_dict(), + ) + + logger.debug("Got response to send_invite: %s", pdu_dict) + + defer.returnValue(Pdu(**pdu_dict)) + @log_function def _get_persisted_pdu(self, event_id): """ Get a PDU from the database with given origin and id. diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index f63028003..9dc2fc2e0 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -56,7 +56,8 @@ class BaseHandler(object): @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], - extra_users=[], suppress_auth=False): + extra_users=[], suppress_auth=False, + do_invite_host=None): yield run_on_reactor() snapshot.fill_out_prev_events(event) @@ -80,6 +81,16 @@ class BaseHandler(object): else: logger.debug("Suppressed auth.") + if do_invite_host: + federation_handler = self.hs.get_handlers().federation_handler + invite_event = yield federation_handler.send_invite( + do_invite_host, + event + ) + + # FIXME: We need to check if the remote changed anything else + event.signatures = invite_event.signatures + yield self.store.persist_event(event) destinations = set(extra_destinations) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c193da12b..e6afd95a5 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -23,6 +23,7 @@ from synapse.api.constants import Membership from synapse.util.logutils import log_function from synapse.federation.pdu_codec import PduCodec from synapse.util.async import run_on_reactor +from synapse.crypto.event_signing import compute_event_signature from twisted.internet import defer @@ -212,6 +213,17 @@ class FederationHandler(BaseHandler): defer.returnValue(events) + @defer.inlineCallbacks + def send_invite(self, target_host, event): + pdu = yield self.replication_layer.send_invite( + destination=target_host, + context=event.room_id, + event_id=event.event_id, + pdu=self.pdu_codec.pdu_from_event(event) + ) + + defer.returnValue(self.pdu_codec.event_from_pdu(pdu)) + @log_function @defer.inlineCallbacks def do_invite_join(self, target_host, room_id, joinee, content, snapshot): @@ -380,6 +392,31 @@ class FederationHandler(BaseHandler): "auth_chain": pdu_auth_chain, }) + @defer.inlineCallbacks + def on_invite_request(self, origin, pdu): + event = self.pdu_codec.event_from_pdu(pdu) + + event.outlier = True + + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) + + yield self.state_handler.annotate_state_groups(event) + + yield self.store.persist_event( + event, + backfilled=False, + ) + + yield self.notifier.on_new_room_event(event) + + defer.returnValue(self.pdu_codec.pdu_from_event(event)) + @defer.inlineCallbacks def get_state_for_pdu(self, event_id): yield run_on_reactor() diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 42a6c9f9b..3642fcfc6 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -361,13 +361,6 @@ class RoomMemberHandler(BaseHandler): if prev_state: event.content["prev"] = prev_state.membership -# if prev_state and prev_state.membership == event.membership: -# # treat this event as a NOOP. -# if do_auth: # This is mainly to fix a unit test. -# yield self.auth.check(event, raises=True) -# defer.returnValue({}) -# return - room_id = event.room_id # If we're trying to join a room then we have to do this differently @@ -521,25 +514,24 @@ class RoomMemberHandler(BaseHandler): defer.returnValue([r.room_id for r in rooms]) + @defer.inlineCallbacks def _do_local_membership_update(self, event, membership, snapshot, do_auth): - destinations = [] - # If we're inviting someone, then we should also send it to that # HS. target_user_id = event.state_key target_user = self.hs.parse_userid(target_user_id) - if membership == Membership.INVITE: - host = target_user.domain - destinations.append(host) + if membership == Membership.INVITE and not target_user.is_mine: + do_invite_host = target_user.domain + else: + do_invite_host = None - # Always include target domain - host = target_user.domain - destinations.append(host) - - return self._on_new_room_event( - event, snapshot, extra_destinations=destinations, - extra_users=[target_user], suppress_auth=(not do_auth), + yield self._on_new_room_event( + event, + snapshot, + extra_users=[target_user], + suppress_auth=(not do_auth), + do_invite_host=do_invite_host, ) From 02c3b1c9e2afa27753e9ce898e5455b6489541b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 15:35:53 +0000 Subject: [PATCH 166/265] Add '/event_auth/' federation api --- synapse/federation/replication.py | 5 +++++ synapse/federation/transport.py | 26 ++++++++++++++++++++++++++ synapse/handlers/federation.py | 5 +++++ synapse/storage/event_federation.py | 26 +++++++++++++++++++------- 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index e358de942..719bfcc42 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -426,6 +426,11 @@ class ReplicationLayer(object): "auth_chain": [p.get_dict() for p in res_pdus["auth_chain"]], })) + @defer.inlineCallbacks + def on_event_auth(self, origin, context, event_id): + auth_pdus = yield self.handler.on_event_auth(event_id) + defer.returnValue((200, [a.get_dict() for a in auth_pdus])) + @defer.inlineCallbacks def make_join(self, destination, context, user_id): pdu_dict = yield self.transport_layer.make_join( diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index b9f7d54c7..babe8447e 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -256,6 +256,21 @@ class TransportLayer(object): defer.returnValue(json.loads(content)) + @defer.inlineCallbacks + @log_function + def get_event_auth(self, destination, context, event_id): + path = PREFIX + "/event_auth/%s/%s" % ( + context, + event_id, + ) + + response = yield self.client.get_json( + destination=destination, + path=path, + ) + + defer.returnValue(response) + @defer.inlineCallbacks def _authenticate_request(self, request): json_request = { @@ -426,6 +441,17 @@ class TransportLayer(object): ) ) + self.server.register_path( + "GET", + re.compile("^" + PREFIX + "/event_auth/([^/]*)/([^/]*)$"), + self._with_authentication( + lambda origin, content, query, context, event_id: + handler.on_event_auth( + origin, context, event_id, + ) + ) + ) + self.server.register_path( "PUT", re.compile("^" + PREFIX + "/send_join/([^/]*)/([^/]*)$"), diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e6afd95a5..ce65bbcd6 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -224,6 +224,11 @@ class FederationHandler(BaseHandler): defer.returnValue(self.pdu_codec.event_from_pdu(pdu)) + @defer.inlineCallbacks + def on_event_auth(self, event_id): + auth = yield self.store.get_auth_chain(event_id) + defer.returnValue([self.pdu_codec.pdu_from_event(e) for e in auth]) + @log_function @defer.inlineCallbacks def do_invite_join(self, target_host, room_id, joinee, content, snapshot): diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index d66a49e9f..06e32d592 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,6 +32,24 @@ class EventFederationStore(SQLBaseStore): ) def _get_auth_chain_txn(self, txn, event_id): + results = self._get_auth_chain_ids_txn(txn, event_id) + + sql = "SELECT * FROM events WHERE event_id = ?" + rows = [] + for ev_id in results: + c = txn.execute(sql, (ev_id,)) + rows.extend(self.cursor_to_dict(c)) + + return self._parse_events_txn(txn, rows) + + def get_auth_chain_ids(self, event_id): + return self.runInteraction( + "get_auth_chain_ids", + self._get_auth_chain_ids_txn, + event_id + ) + + def _get_auth_chain_ids_txn(self, txn, event_id): results = set() base_sql = ( @@ -48,13 +66,7 @@ class EventFederationStore(SQLBaseStore): front = [r[0] for r in txn.fetchall()] results.update(front) - sql = "SELECT * FROM events WHERE event_id = ?" - rows = [] - for ev_id in results: - c = txn.execute(sql, (ev_id,)) - rows.extend(self.cursor_to_dict(c)) - - return self._parse_events_txn(txn, rows) + return list(results) def get_oldest_events_in_room(self, room_id): return self.runInteraction( From 07286a73b179c7b3888f7cbf15dcf7c4f5a0c3a3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Nov 2014 16:03:31 +0000 Subject: [PATCH 167/265] Use current state to get room hosts, rather than querying the database --- synapse/handlers/_base.py | 18 +++++++++++++++--- synapse/handlers/federation.py | 21 +++++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 9dc2fc2e0..07a846410 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,6 +18,8 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.api.events.room import RoomMemberEvent +from synapse.api.constants import Membership import logging @@ -95,9 +97,19 @@ class BaseHandler(object): destinations = set(extra_destinations) # Send a PDU to all hosts who have joined the room. - destinations.update((yield self.store.get_joined_hosts_for_room( - event.room_id - ))) + + for k, s in event.state_events.items(): + try: + if k[0] == RoomMemberEvent.TYPE: + if s.content["membership"] == Membership.JOIN: + destinations.add( + self.hs.parse_userid(s.state_key).domain + ) + except: + logger.warn( + "Failed to get destination from event %s", s.event_id + ) + event.destinations = list(destinations) self.notifier.on_new_room_event(event, extra_users=extra_users) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ce65bbcd6..7e1058390 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -376,10 +376,23 @@ class FederationHandler(BaseHandler): "user_joined_room", user=user, room_id=event.room_id ) - new_pdu = self.pdu_codec.pdu_from_event(event); - new_pdu.destinations = yield self.store.get_joined_hosts_for_room( - event.room_id - ) + new_pdu = self.pdu_codec.pdu_from_event(event) + + destinations = set() + + for k, s in event.state_events.items(): + try: + if k[0] == RoomMemberEvent.TYPE: + if s.content["membership"] == Membership.JOIN: + destinations.add( + self.hs.parse_userid(s.state_key).domain + ) + except: + logger.warn( + "Failed to get destination from event %s", s.event_id + ) + + new_pdu.destinations = list(destinations) yield self.replication_layer.send_pdu(new_pdu) From 7d15452c3037b887a85d0b65916de90b2d2c4573 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 7 Nov 2014 17:56:28 +0000 Subject: [PATCH 168/265] Various fixes to try & make openwebrtc safari extension work (still doesn't work). --- syweb/webclient/app-controller.js | 8 +- syweb/webclient/app.css | 12 +- .../components/matrix/matrix-call.js | 146 +++++++++++------- syweb/webclient/index.html | 4 +- syweb/webclient/room/room-controller.js | 4 +- 5 files changed, 107 insertions(+), 67 deletions(-) diff --git a/syweb/webclient/app-controller.js b/syweb/webclient/app-controller.js index 2d82a42cf..bbcf4ab5f 100644 --- a/syweb/webclient/app-controller.js +++ b/syweb/webclient/app-controller.js @@ -112,8 +112,8 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even if (!$rootScope.currentCall) { // This causes the still frame to be flushed out of the video elements, // avoiding a flash of the last frame of the previous call when starting the next - angular.element('#localVideo')[0].load(); - angular.element('#remoteVideo')[0].load(); + if (angular.element('#localVideo')[0].load) angular.element('#localVideo')[0].load(); + if (angular.element('#remoteVideo')[0].load) angular.element('#remoteVideo')[0].load(); return; } @@ -187,8 +187,8 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even } call.onError = $scope.onCallError; call.onHangup = $scope.onCallHangup; - call.localVideoElement = angular.element('#localVideo')[0]; - call.remoteVideoElement = angular.element('#remoteVideo')[0]; + call.localVideoSelector = '#localVideo'; + call.remoteVideoSelector = '#remoteVideo'; $rootScope.currentCall = call; }); diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css index 5ab8e2b8f..be2a73872 100755 --- a/syweb/webclient/app.css +++ b/syweb/webclient/app.css @@ -136,17 +136,17 @@ textarea, input { transition: left linear 500ms, top linear 500ms, width linear 500ms, height linear 500ms; } -#localVideo.mini { +.mini #localVideo { top: 0px; left: 130px; } -#localVideo.large { +.large #localVideo { top: 70px; left: 20px; } -#localVideo.ended { +.ended #localVideo { -webkit-filter: grayscale(1); filter: grayscale(1); } @@ -157,19 +157,19 @@ textarea, input { transition: left linear 500ms, top linear 500ms, width linear 500ms, height linear 500ms; } -#remoteVideo.mini { +.mini #remoteVideo { left: 260px; top: 0px; width: 128px; } -#remoteVideo.large { +.large #remoteVideo { left: 0px; top: 50px; width: 100%; } -#remoteVideo.ended { +.ended #remoteVideo { -webkit-filter: grayscale(1); filter: grayscale(1); } diff --git a/syweb/webclient/components/matrix/matrix-call.js b/syweb/webclient/components/matrix/matrix-call.js index 063164994..b560cf7da 100644 --- a/syweb/webclient/components/matrix/matrix-call.js +++ b/syweb/webclient/components/matrix/matrix-call.js @@ -35,14 +35,14 @@ var forAllTracksOnStream = function(s, f) { forAllAudioTracksOnStream(s, f); } -navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia; -window.RTCPeerConnection = window.RTCPeerConnection || window.webkitRTCPeerConnection; // but not mozRTCPeerConnection because its interface is not compatible -window.RTCSessionDescription = window.RTCSessionDescription || window.webkitRTCSessionDescription || window.mozRTCSessionDescription; -window.RTCIceCandidate = window.RTCIceCandidate || window.webkitRTCIceCandidate || window.mozRTCIceCandidate; - angular.module('MatrixCall', []) .factory('MatrixCall', ['matrixService', 'matrixPhoneService', 'modelService', '$rootScope', '$timeout', function MatrixCallFactory(matrixService, matrixPhoneService, modelService, $rootScope, $timeout) { $rootScope.isWebRTCSupported = function () { + navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia; + window.RTCPeerConnection = window.RTCPeerConnection || window.webkitRTCPeerConnection; // but not mozRTCPeerConnection because its interface is not compatible + window.RTCSessionDescription = window.RTCSessionDescription || window.webkitRTCSessionDescription || window.mozRTCSessionDescription; + window.RTCIceCandidate = window.RTCIceCandidate || window.webkitRTCIceCandidate || window.mozRTCIceCandidate; + return !!(navigator.getUserMedia || window.RTCPeerConnection || window.RTCSessionDescription || window.RTCIceCandidate); }; @@ -57,7 +57,7 @@ angular.module('MatrixCall', []) this.candidateSendTries = 0; var self = this; - $rootScope.$watch(this.remoteVideoElement, function (oldValue, newValue) { + $rootScope.$watch(this.getRemoteVideoElement(), function (oldValue, newValue) { self.tryPlayRemoteStream(); }); @@ -252,8 +252,8 @@ angular.module('MatrixCall', []) // pausing now keeps the last frame (ish) of the video call in the video element // rather than it just turning black straight away - if (this.remoteVideoElement) this.remoteVideoElement.pause(); - if (this.localVideoElement) this.localVideoElement.pause(); + if (this.getRemoteVideoElement() && this.getRemoteVideoElement().pause) this.getRemoteVideoElement().pause(); + if (this.getLocalVideoElement() && this.getLocalVideoElement().pause) this.getLocalVideoElement().pause(); this.stopAllMedia(); if (this.peerConn) this.peerConn.close(); @@ -278,11 +278,18 @@ angular.module('MatrixCall', []) } if (this.state == 'ended') return; - if (this.localVideoElement && this.type == 'video') { + var videoEl = this.getLocalVideoElement(); + + if (videoEl && this.type == 'video') { var vidTrack = stream.getVideoTracks()[0]; - this.localVideoElement.src = URL.createObjectURL(stream); - this.localVideoElement.muted = true; - this.localVideoElement.play(); + videoEl.autoplay = true; + videoEl.src = URL.createObjectURL(stream); + videoEl.muted = true; + var self = this; + $timeout(function() { + var vel = self.getLocalVideoElement(); + if (vel.play) vel.play(); + }); } this.localAVStream = stream; @@ -306,11 +313,18 @@ angular.module('MatrixCall', []) MatrixCall.prototype.gotUserMediaForAnswer = function(stream) { if (this.state == 'ended') return; - if (this.localVideoElement && this.type == 'video') { + var localVidEl = this.getLocalVideoElement(); + + if (localVidEl && this.type == 'video') { + localVidEl.autoplay = true; var vidTrack = stream.getVideoTracks()[0]; - this.localVideoElement.src = URL.createObjectURL(stream); - this.localVideoElement.muted = true; - this.localVideoElement.play(); + localVidEl.src = URL.createObjectURL(stream); + localVidEl.muted = true; + var self = this; + $timeout(function() { + var vel = self.getLocalVideoElement(); + if (vel.play) vel.play(); + }); } this.localAVStream = stream; @@ -339,11 +353,11 @@ angular.module('MatrixCall', []) } MatrixCall.prototype.gotRemoteIceCandidate = function(cand) { - console.log("Got remote ICE "+cand.sdpMid+" candidate: "+cand.candidate); if (this.state == 'ended') { - console.log("Ignoring remote ICE candidate because call has ended"); + //console.log("Ignoring remote ICE candidate because call has ended"); return; } + console.log("Got remote ICE "+cand.sdpMid+" candidate: "+cand.candidate); this.peerConn.addIceCandidate(new RTCIceCandidate(cand), function() {}, function(e) {}); }; @@ -363,41 +377,46 @@ angular.module('MatrixCall', []) return; } - this.peerConn.setLocalDescription(description); - - var content = { - version: 0, - call_id: this.call_id, - offer: description, - lifetime: MatrixCall.CALL_TIMEOUT - }; - this.sendEventWithRetry('m.call.invite', content); - var self = this; - $timeout(function() { - if (self.state == 'invite_sent') { - self.hangup('invite_timeout'); - } - }, MatrixCall.CALL_TIMEOUT); + this.peerConn.setLocalDescription(description, function() { + var content = { + version: 0, + call_id: self.call_id, + // OpenWebRTC appears to add extra stuff (like the DTLS fingerprint) to the description + // when setting it on the peerconnection. According to the spec it should only add ICE + // candidates. Any ICE candidates that have already been generated at this point will + // probably be sent both in the offer and separately. Ho hum. + offer: self.peerConn.localDescription, + lifetime: MatrixCall.CALL_TIMEOUT + }; + self.sendEventWithRetry('m.call.invite', content); - $rootScope.$apply(function() { - self.state = 'invite_sent'; - }); + $timeout(function() { + if (self.state == 'invite_sent') { + self.hangup('invite_timeout'); + } + }, MatrixCall.CALL_TIMEOUT); + + $rootScope.$apply(function() { + self.state = 'invite_sent'; + }); + }, function() { console.log("Error setting local description!"); }); }; MatrixCall.prototype.createdAnswer = function(description) { console.log("Created answer: "+description); - this.peerConn.setLocalDescription(description); - var content = { - version: 0, - call_id: this.call_id, - answer: description - }; - this.sendEventWithRetry('m.call.answer', content); var self = this; - $rootScope.$apply(function() { - self.state = 'connecting'; - }); + this.peerConn.setLocalDescription(description, function() { + var content = { + version: 0, + call_id: self.call_id, + answer: self.peerConn.localDescription + }; + self.sendEventWithRetry('m.call.answer', content); + $rootScope.$apply(function() { + self.state = 'connecting'; + }); + }, function() { console.log("Error setting local description!"); } ); }; MatrixCall.prototype.getLocalOfferFailed = function(error) { @@ -465,10 +484,15 @@ angular.module('MatrixCall', []) }; MatrixCall.prototype.tryPlayRemoteStream = function(event) { - if (this.remoteVideoElement && this.remoteAVStream) { - var player = this.remoteVideoElement; + if (this.getRemoteVideoElement() && this.remoteAVStream) { + var player = this.getRemoteVideoElement(); + player.autoplay = true; player.src = URL.createObjectURL(this.remoteAVStream); - player.play(); + var self = this; + $timeout(function() { + var vel = self.getRemoteVideoElement(); + if (vel.play) vel.play(); + }); } }; @@ -500,8 +524,8 @@ angular.module('MatrixCall', []) MatrixCall.prototype.onHangupReceived = function(msg) { console.log("Hangup received"); - if (this.remoteVideoElement) this.remoteVideoElement.pause(); - if (this.localVideoElement) this.localVideoElement.pause(); + if (this.getRemoteVideoElement() && this.getRemoteVideoElement().pause) this.getRemoteVideoElement().pause(); + if (this.getLocalVideoElement() && this.getLocalVideoElement().pause) this.getLocalVideoElement().pause(); this.state = 'ended'; this.hangupParty = 'remote'; this.hangupReason = msg.reason; @@ -524,8 +548,8 @@ angular.module('MatrixCall', []) newCall.gotUserMediaForAnswer(this.localAVStream); delete(this.localAVStream); } - newCall.localVideoElement = this.localVideoElement; - newCall.remoteVideoElement = this.remoteVideoElement; + newCall.localVideoSelector = this.localVideoSelector; + newCall.remoteVideoSelector = this.remoteVideoSelector; this.successor = newCall; this.hangup(true); }; @@ -601,5 +625,21 @@ angular.module('MatrixCall', []) }, delayMs); }; + MatrixCall.prototype.getLocalVideoElement = function() { + if (this.localVideoSelector) { + var t = angular.element(this.localVideoSelector); + if (t.length) return t[0]; + } + return null; + }; + + MatrixCall.prototype.getRemoteVideoElement = function() { + if (this.remoteVideoSelector) { + var t = angular.element(this.remoteVideoSelector); + if (t.length) return t[0]; + } + return null; + }; + return MatrixCall; }]); diff --git a/syweb/webclient/index.html b/syweb/webclient/index.html index 992e8d337..45be6c274 100644 --- a/syweb/webclient/index.html +++ b/syweb/webclient/index.html @@ -53,8 +53,8 @@
- - +
+
diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index d3fb85b9d..b878a1f71 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -919,8 +919,8 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) var call = new MatrixCall($scope.room_id); call.onError = $rootScope.onCallError; call.onHangup = $rootScope.onCallHangup; - call.localVideoElement = angular.element('#localVideo')[0]; - call.remoteVideoElement = angular.element('#remoteVideo')[0]; + call.localVideoSelector = '#localVideo'; + call.remoteVideoSelector = '#remoteVideo'; call.placeVideoCall(); $rootScope.currentCall = call; }; From 1c06806f90a6368cdc3b9fa3b9053021b7c40e94 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 10:21:32 +0000 Subject: [PATCH 169/265] Finish redaction algorithm. --- synapse/api/events/__init__.py | 4 ++-- synapse/api/events/utils.py | 39 ++++++++++++++++++++++----------- synapse/crypto/event_signing.py | 7 ++---- synapse/federation/units.py | 6 ++--- synapse/storage/_base.py | 2 +- 5 files changed, 33 insertions(+), 25 deletions(-) diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 8d65c29ac..f1e53f23a 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -86,8 +86,8 @@ class SynapseEvent(JsonEncodedObject): def __init__(self, raises=True, **kwargs): super(SynapseEvent, self).__init__(**kwargs) - if "content" in kwargs: - self.check_json(self.content, raises=raises) + # if "content" in kwargs: + # self.check_json(self.content, raises=raises) def get_content_template(self): """ Retrieve the JSON template for this event as a dict. diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index 5fc79105b..802648f8f 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -18,24 +18,31 @@ from .room import ( RoomAliasesEvent, RoomCreateEvent, ) + def prune_event(event): - """ Prunes the given event of all keys we don't know about or think could - potentially be dodgy. + """ Returns a pruned version of the given event, which removes all keys we + don't know about or think could potentially be dodgy. This is used when we "redact" an event. We want to remove all fields that the user has specified, but we do want to keep necessary information like type, state_key etc. """ - return _prune_event_or_pdu(event.type, event) + event_type = event.type -def prune_pdu(pdu): - """Removes keys that contain unrestricted and non-essential data from a PDU - """ - return _prune_event_or_pdu(pdu.type, pdu) - -def _prune_event_or_pdu(event_type, event): - # Remove all extraneous fields. - event.unrecognized_keys = {} + allowed_keys = [ + "event_id", + "user_id", + "room_id", + "hashes", + "signatures", + "content", + "type", + "state_key", + "depth", + "prev_events", + "prev_state", + "auth_events", + ] new_content = {} @@ -65,6 +72,12 @@ def _prune_event_or_pdu(event_type, event): elif event_type == RoomAliasesEvent.TYPE: add_fields("aliases") - event.content = new_content + allowed_fields = { + k: v + for k, v in event.get_full_dict().items() + if k in allowed_keys + } - return event + allowed_fields["content"] = new_content + + return type(event)(**allowed_fields) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 7d800615f..056e8f6ca 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -55,9 +55,7 @@ def _compute_content_hash(event, hash_algorithm): def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): - # FIXME(erikj): GenericEvent! - tmp_event = GenericEvent(**event.get_full_dict()) - tmp_event = prune_event(tmp_event) + tmp_event = prune_event(event) event_json = tmp_event.get_dict() event_json.pop("signatures", None) event_json.pop("age_ts", None) @@ -68,8 +66,7 @@ def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): def compute_event_signature(event, signature_name, signing_key): - tmp_event = copy.deepcopy(event) - tmp_event = prune_event(tmp_event) + tmp_event = prune_event(event) redact_json = tmp_event.get_full_dict() redact_json.pop("signatures", None) redact_json.pop("age_ts", None) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 2070ffe1e..d98014cac 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -56,17 +56,15 @@ class Pdu(JsonEncodedObject): "origin_server_ts", "type", "destinations", - "transaction_id", "prev_events", "depth", "content", - "outlier", "hashes", + "user_id", + "auth_events", "signatures", # Below this are keys valid only for State Pdus. "state_key", "prev_state", - "required_power_level", - "user_id", ] internal_keys = [ diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9aa404695..3ab81a78d 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -509,7 +509,7 @@ class SQLBaseStore(object): ) if del_evs: - prune_event(ev) + ev = prune_event(ev) ev.redacted_because = del_evs[0] return events From 6cb6cb9e6908ad9b71ebd63ca535eb6c7c48be86 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 10:31:00 +0000 Subject: [PATCH 170/265] Tidy up some of the unused sql tables --- synapse/crypto/event_signing.py | 2 - synapse/storage/__init__.py | 21 ++-------- synapse/storage/room.py | 27 ------------- synapse/storage/schema/im.sql | 68 +++------------------------------ 4 files changed, 9 insertions(+), 109 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 056e8f6ca..baa93b0ee 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -19,9 +19,7 @@ from synapse.api.events.utils import prune_event from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json -from synapse.api.events.room import GenericEvent -import copy import hashlib import logging diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 96adf20c8..7d810e6a6 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -16,9 +16,7 @@ from twisted.internet import defer from synapse.api.events.room import ( - RoomMemberEvent, RoomTopicEvent, FeedbackEvent, - RoomNameEvent, - RoomJoinRulesEvent, + RoomMemberEvent, RoomTopicEvent, FeedbackEvent, RoomNameEvent, RoomRedactionEvent, ) @@ -95,8 +93,7 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks @log_function - def persist_event(self, event=None, backfilled=False, pdu=None, - is_new_state=True): + def persist_event(self, event, backfilled=False, is_new_state=True): stream_ordering = None if backfilled: if not self.min_token_deferred.called: @@ -107,8 +104,7 @@ class DataStore(RoomMemberStore, RoomStore, try: yield self.runInteraction( "persist_event", - self._persist_pdu_event_txn, - pdu=pdu, + self._persist_event_txn, event=event, backfilled=backfilled, stream_ordering=stream_ordering, @@ -139,15 +135,6 @@ class DataStore(RoomMemberStore, RoomStore, event = self._parse_event_from_row(events_dict) defer.returnValue(event) - def _persist_pdu_event_txn(self, txn, pdu=None, event=None, - backfilled=False, stream_ordering=None, - is_new_state=True): - if event is not None: - return self._persist_event_txn( - txn, event, backfilled, stream_ordering, - is_new_state=is_new_state, - ) - @log_function def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None, is_new_state=True): @@ -159,8 +146,6 @@ class DataStore(RoomMemberStore, RoomStore, self._store_room_name_txn(txn, event) elif event.type == RoomTopicEvent.TYPE: self._store_room_topic_txn(txn, event) - elif event.type == RoomJoinRulesEvent.TYPE: - self._store_join_rule(txn, event) elif event.type == RoomRedactionEvent.TYPE: self._store_redaction(txn, event) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 0c83c11ad..ca70506d2 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -132,22 +132,6 @@ class RoomStore(SQLBaseStore): defer.returnValue(ret) - @defer.inlineCallbacks - def get_room_join_rule(self, room_id): - sql = ( - "SELECT join_rule FROM room_join_rules as r " - "INNER JOIN current_state_events as c " - "ON r.event_id = c.event_id " - "WHERE c.room_id = ? " - ) - - rows = yield self._execute(None, sql, room_id) - - if len(rows) == 1: - defer.returnValue(rows[0][0]) - else: - defer.returnValue(None) - def _store_room_topic_txn(self, txn, event): self._simple_insert_txn( txn, @@ -170,17 +154,6 @@ class RoomStore(SQLBaseStore): } ) - def _store_join_rule(self, txn, event): - self._simple_insert_txn( - txn, - "room_join_rules", - { - "event_id": event.event_id, - "room_id": event.room_id, - "join_rule": event.content["join_rule"], - }, - ) - class RoomsTable(Table): table_name = "rooms" diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 8d6f65599..8ba732a23 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -85,80 +85,24 @@ CREATE TABLE IF NOT EXISTS topics( topic TEXT NOT NULL ); +CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); +CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); + CREATE TABLE IF NOT EXISTS room_names( event_id TEXT NOT NULL, room_id TEXT NOT NULL, name TEXT NOT NULL ); +CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); +CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); + CREATE TABLE IF NOT EXISTS rooms( room_id TEXT PRIMARY KEY NOT NULL, is_public INTEGER, creator TEXT ); -CREATE TABLE IF NOT EXISTS room_join_rules( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - join_rule TEXT NOT NULL -); -CREATE INDEX IF NOT EXISTS room_join_rules_event_id ON room_join_rules(event_id); -CREATE INDEX IF NOT EXISTS room_join_rules_room_id ON room_join_rules(room_id); - - -CREATE TABLE IF NOT EXISTS room_power_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - user_id TEXT NOT NULL, - level INTEGER NOT NULL -); -CREATE INDEX IF NOT EXISTS room_power_levels_event_id ON room_power_levels(event_id); -CREATE INDEX IF NOT EXISTS room_power_levels_room_id ON room_power_levels(room_id); -CREATE INDEX IF NOT EXISTS room_power_levels_room_user ON room_power_levels(room_id, user_id); - - -CREATE TABLE IF NOT EXISTS room_default_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_default_levels_event_id ON room_default_levels(event_id); -CREATE INDEX IF NOT EXISTS room_default_levels_room_id ON room_default_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_add_state_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_add_state_levels_event_id ON room_add_state_levels(event_id); -CREATE INDEX IF NOT EXISTS room_add_state_levels_room_id ON room_add_state_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_send_event_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - level INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS room_send_event_levels_event_id ON room_send_event_levels(event_id); -CREATE INDEX IF NOT EXISTS room_send_event_levels_room_id ON room_send_event_levels(room_id); - - -CREATE TABLE IF NOT EXISTS room_ops_levels( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - ban_level INTEGER, - kick_level INTEGER, - redact_level INTEGER -); - -CREATE INDEX IF NOT EXISTS room_ops_levels_event_id ON room_ops_levels(event_id); -CREATE INDEX IF NOT EXISTS room_ops_levels_room_id ON room_ops_levels(room_id); - - CREATE TABLE IF NOT EXISTS room_hosts( room_id TEXT NOT NULL, host TEXT NOT NULL, From 407d8a5019e8e352261886d6479f45e6144ae5b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 10:35:43 +0000 Subject: [PATCH 171/265] Fix invite auth --- synapse/api/auth.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d4f284bd6..077d1ab0b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -128,13 +128,14 @@ class Auth(object): key = (RoomMemberEvent.TYPE, event.user_id, ) caller = event.old_state_events.get(key) - caller_in_room = caller and caller.membership == "join" + caller_in_room = caller and caller.membership == Membership.JOIN + caller_invited = caller and caller.membership == Membership.INVITE # get info about the target key = (RoomMemberEvent.TYPE, target_user_id, ) target = event.old_state_events.get(key) - target_in_room = target and target.membership == "join" + target_in_room = target and target.membership == Membership.JOIN membership = event.content["membership"] @@ -162,6 +163,7 @@ class Auth(object): "is_membership_change_allowed: %s", { "caller_in_room": caller_in_room, + "caller_invited": caller_invited, "target_in_room": target_in_room, "membership": membership, "join_rule": join_rule, @@ -189,7 +191,7 @@ class Auth(object): elif join_rule == JoinRules.PUBLIC: pass elif join_rule == JoinRules.INVITE: - if not caller_in_room: + if not caller_in_room and not caller_invited: raise AuthError(403, "You are not invited to this room.") else: # TODO (erikj): may_join list From 65f846ade09feba129a14b2b9714c2ad51d00f0a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 11:15:02 +0000 Subject: [PATCH 172/265] Notify users about invites. --- synapse/api/auth.py | 16 +++++++++------- synapse/handlers/federation.py | 5 ++++- synapse/storage/stream.py | 3 +-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 077d1ab0b..3e5d878ee 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -337,15 +337,17 @@ class Auth(object): if join_rule_event: join_rule = join_rule_event.content.get("join_rule") is_public = join_rule == JoinRules.PUBLIC if join_rule else False + else: + is_public = False - if event.type == RoomMemberEvent.TYPE: - if event.content["membership"] == Membership.JOIN: - if is_public: - auth_events.append(join_rule_event.event_id) - elif member_event: + if event.type == RoomMemberEvent.TYPE: + e_type = event.content["membership"] + if e_type in [Membership.JOIN, Membership.INVITE]: + auth_events.append(join_rule_event.event_id) + + if member_event and not is_public: auth_events.append(member_event.event_id) - - if member_event: + elif member_event: if member_event.content["membership"] == Membership.JOIN: auth_events.append(member_event.event_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7e1058390..9a59fe94d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -431,7 +431,10 @@ class FederationHandler(BaseHandler): backfilled=False, ) - yield self.notifier.on_new_room_event(event) + target_user = self.hs.parse_userid(event.state_key) + yield self.notifier.on_new_room_event( + event, extra_users=[target_user], + ) defer.returnValue(self.pdu_codec.pdu_from_event(event)) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8f7f61d29..475e7f20a 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -177,10 +177,9 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT *, (%(redacted)s) AS redacted FROM events AS e WHERE " - "((room_id IN (%(current)s)) OR " + "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " - "AND e.outlier = 0 " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "redacted": del_sql, From 6447db063a0d01135582bdfb3392b419f16a19e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 11:59:51 +0000 Subject: [PATCH 173/265] Fix backfill to work. Add auth to backfill request --- synapse/api/auth.py | 6 +++++ synapse/federation/replication.py | 36 ++++++++++++++++++++++------- synapse/federation/transport.py | 6 ++--- synapse/handlers/federation.py | 10 ++++---- synapse/storage/_base.py | 12 ++++++++++ synapse/storage/event_federation.py | 4 ++-- 6 files changed, 56 insertions(+), 18 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3e5d878ee..48f9d460a 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -104,6 +104,12 @@ class Auth(object): pass defer.returnValue(None) + @defer.inlineCallbacks + def check_host_in_room(self, room_id, host): + joined_hosts = yield self.store.get_joined_hosts_for_room(room_id) + + defer.returnValue(host in joined_hosts) + def check_event_sender_in_room(self, event): key = (RoomMemberEvent.TYPE, event.user_id, ) member_event = event.state_events.get(key) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 719bfcc42..7837f1c25 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -205,7 +205,7 @@ class ReplicationLayer(object): pdus = [Pdu(outlier=False, **p) for p in transaction.pdus] for pdu in pdus: - yield self._handle_new_pdu(pdu, backfilled=True) + yield self._handle_new_pdu(dest, pdu, backfilled=True) defer.returnValue(pdus) @@ -274,9 +274,9 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_backfill_request(self, context, versions, limit): + def on_backfill_request(self, origin, context, versions, limit): pdus = yield self.handler.on_backfill_request( - context, versions, limit + origin, context, versions, limit ) defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @@ -408,13 +408,22 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_make_join_request(self, context, user_id): pdu = yield self.handler.on_make_join_request(context, user_id) - defer.returnValue(pdu.get_dict()) + defer.returnValue({ + "event": pdu.get_dict(), + }) @defer.inlineCallbacks def on_invite_request(self, origin, content): pdu = Pdu(**content) ret_pdu = yield self.handler.on_invite_request(origin, pdu) - defer.returnValue((200, ret_pdu.get_dict())) + defer.returnValue( + ( + 200, + { + "event": ret_pdu.get_dict(), + } + ) + ) @defer.inlineCallbacks def on_send_join_request(self, origin, content): @@ -429,16 +438,25 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_event_auth(self, origin, context, event_id): auth_pdus = yield self.handler.on_event_auth(event_id) - defer.returnValue((200, [a.get_dict() for a in auth_pdus])) + defer.returnValue( + ( + 200, + { + "auth_chain": [a.get_dict() for a in auth_pdus], + } + ) + ) @defer.inlineCallbacks def make_join(self, destination, context, user_id): - pdu_dict = yield self.transport_layer.make_join( + ret = yield self.transport_layer.make_join( destination=destination, context=context, user_id=user_id, ) + pdu_dict = ret["event"] + logger.debug("Got response to make_join: %s", pdu_dict) defer.returnValue(Pdu(**pdu_dict)) @@ -467,13 +485,15 @@ class ReplicationLayer(object): @defer.inlineCallbacks def send_invite(self, destination, context, event_id, pdu): - code, pdu_dict = yield self.transport_layer.send_invite( + code, content = yield self.transport_layer.send_invite( destination=destination, context=context, event_id=event_id, content=pdu.get_dict(), ) + pdu_dict = content["event"] + logger.debug("Got response to send_invite: %s", pdu_dict) defer.returnValue(Pdu(**pdu_dict)) diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index babe8447e..92a1f4ce1 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -413,7 +413,7 @@ class TransportLayer(object): self._with_authentication( lambda origin, content, query, context: self._on_backfill_request( - context, query["v"], query["limit"] + origin, context, query["v"], query["limit"] ) ) ) @@ -552,7 +552,7 @@ class TransportLayer(object): defer.returnValue(data) @log_function - def _on_backfill_request(self, context, v_list, limits): + def _on_backfill_request(self, origin, context, v_list, limits): if not limits: return defer.succeed( (400, {"error": "Did not include limit param"}) @@ -563,7 +563,7 @@ class TransportLayer(object): versions = v_list return self.request_handler.on_backfill_request( - context, versions, limit + origin, context, versions, limit ) @defer.inlineCallbacks diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9a59fe94d..00d10609b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -193,10 +193,7 @@ class FederationHandler(BaseHandler): dest, room_id, limit, - extremities=[ - self.pdu_codec.decode_event_id(e) - for e in extremities - ] + extremities=extremities, ) events = [] @@ -473,7 +470,10 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def on_backfill_request(self, context, pdu_list, limit): + def on_backfill_request(self, origin, context, pdu_list, limit): + in_room = yield self.auth.check_host_in_room(context, origin) + if not in_room: + raise AuthError(403, "Host not in room.") events = yield self.store.get_backfill_events( context, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3ab81a78d..a23f2b941 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -447,6 +447,18 @@ class SQLBaseStore(object): **d ) + def _get_events_txn(self, txn, event_ids): + # FIXME (erikj): This should be batched? + + sql = "SELECT * FROM events WHERE event_id = ?" + + event_rows = [] + for e_id in event_ids: + c = txn.execute(sql, (e_id,)) + event_rows.extend(self.cursor_to_dict(c)) + + return self._parse_events_txn(txn, event_rows) + def _parse_events(self, rows): return self.runInteraction( "_parse_events", self._parse_events_txn, rows diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 06e32d592..a70703014 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -371,10 +371,10 @@ class EventFederationStore(SQLBaseStore): "_backfill_interaction: got id=%s", *row ) - new_front.append(row) + new_front.append(row[0]) front = new_front event_results += new_front # We also want to update the `prev_pdus` attributes before returning. - return self._get_pdu_tuples(txn, event_results) + return self._get_events_txn(txn, event_results) From 003668cfaadc2e96fab0127d7c563eb4b17e0619 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 13:37:24 +0000 Subject: [PATCH 174/265] Add auth to the various server-server APIs --- synapse/federation/replication.py | 14 ++++++++------ synapse/federation/transport.py | 3 ++- synapse/handlers/federation.py | 15 +++++++++++++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 7837f1c25..dd8124dbb 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -347,10 +347,12 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_context_state_request(self, context, event_id): + def on_context_state_request(self, origin, context, event_id): if event_id: pdus = yield self.handler.get_state_for_pdu( - event_id + origin, + context, + event_id, ) else: raise NotImplementedError("Specify an event") @@ -365,8 +367,8 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_pdu_request(self, event_id): - pdu = yield self._get_persisted_pdu(event_id) + def on_pdu_request(self, origin, event_id): + pdu = yield self._get_persisted_pdu(origin, event_id) if pdu: defer.returnValue( @@ -499,13 +501,13 @@ class ReplicationLayer(object): defer.returnValue(Pdu(**pdu_dict)) @log_function - def _get_persisted_pdu(self, event_id): + def _get_persisted_pdu(self, origin, event_id): """ Get a PDU from the database with given origin and id. Returns: Deferred: Results in a `Pdu`. """ - return self.handler.get_persisted_pdu(event_id) + return self.handler.get_persisted_pdu(origin, event_id) def _transaction_from_pdus(self, pdu_list): """Returns a new Transaction containing the given PDUs suitable for diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 92a1f4ce1..d84a44c21 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -390,7 +390,7 @@ class TransportLayer(object): re.compile("^" + PREFIX + "/event/([^/]*)/$"), self._with_authentication( lambda origin, content, query, event_id: - handler.on_pdu_request(event_id) + handler.on_pdu_request(origin, event_id) ) ) @@ -401,6 +401,7 @@ class TransportLayer(object): self._with_authentication( lambda origin, content, query, context: handler.on_context_state_request( + origin, context, query.get("event_id", [None])[0], ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 00d10609b..587fa308c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -436,9 +436,13 @@ class FederationHandler(BaseHandler): defer.returnValue(self.pdu_codec.pdu_from_event(event)) @defer.inlineCallbacks - def get_state_for_pdu(self, event_id): + def get_state_for_pdu(self, origin, room_id, event_id): yield run_on_reactor() + in_room = yield self.auth.check_host_in_room(room_id, origin) + if not in_room: + raise AuthError(403, "Host not in room.") + state_groups = yield self.store.get_state_groups( [event_id] ) @@ -488,7 +492,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def get_persisted_pdu(self, event_id): + def get_persisted_pdu(self, origin, event_id): """ Get a PDU from the database with given origin and id. Returns: @@ -500,6 +504,13 @@ class FederationHandler(BaseHandler): ) if event: + in_room = yield self.auth.check_host_in_room( + event.room_id, + origin + ) + if not in_room: + raise AuthError(403, "Host not in room.") + defer.returnValue(self.pdu_codec.pdu_from_event(event)) else: defer.returnValue(None) From c46088405aa6baa41da0aef56073ae3b4fe9f850 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 13:39:33 +0000 Subject: [PATCH 175/265] Remove useless comments --- synapse/federation/replication.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index dd8124dbb..e79830435 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -356,12 +356,6 @@ class ReplicationLayer(object): ) else: raise NotImplementedError("Specify an event") - # results = yield self.store.get_current_state_for_context( - # context - # ) - # pdus = [Pdu.from_pdu_tuple(p) for p in results] - # - # logger.debug("Context returning %d results", len(pdus)) defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @@ -382,21 +376,6 @@ class ReplicationLayer(object): def on_pull_request(self, origin, versions): raise NotImplementedError("Pull transacions not implemented") - # transaction_id = max([int(v) for v in versions]) - # - # response = yield self.pdu_actions.after_transaction( - # transaction_id, - # origin, - # self.server_name - # ) - # - # if not response: - # response = [] - # - # defer.returnValue( - # (200, self._transaction_from_pdus(response).get_dict()) - # ) - @defer.inlineCallbacks def on_query_request(self, query_type, args): if query_type in self.query_handlers: @@ -570,8 +549,6 @@ class ReplicationLayer(object): origin, pdu.room_id, pdu.event_id, ) - # Persist the Pdu, but don't mark it as processed yet. - # yield self.store.persist_event(pdu=pdu) if not backfilled: ret = yield self.handler.on_receive_pdu( From 5d439b127ba34b951dfd09a7d3c684c2d50df702 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 13:46:44 +0000 Subject: [PATCH 176/265] PEP8 --- synapse/api/auth.py | 3 +-- synapse/api/events/room.py | 1 + synapse/federation/replication.py | 1 - synapse/federation/transport.py | 9 ++++++--- synapse/federation/units.py | 7 +++---- synapse/handlers/federation.py | 5 ++++- synapse/storage/__init__.py | 7 ++++--- synapse/storage/event_federation.py | 9 +++------ 8 files changed, 22 insertions(+), 20 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 48f9d460a..a5c696470 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -369,7 +369,6 @@ class Auth(object): ] event.auth_events = zip(auth_events, hashes) - @log_function def _can_send_event(self, event): key = (RoomPowerLevelsEvent.TYPE, "", ) @@ -452,7 +451,7 @@ class Auth(object): event.user_id, ) - # Check other levels: + # Check other levels: levels_to_check = [ ("users_default", []), ("events_default", []), diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py index 25bc88370..8c4ac45d0 100644 --- a/synapse/api/events/room.py +++ b/synapse/api/events/room.py @@ -153,6 +153,7 @@ class RoomPowerLevelsEvent(SynapseStateEvent): def get_content_template(self): return {} + class RoomAliasesEvent(SynapseStateEvent): TYPE = "m.room.aliases" diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index e79830435..bacba3675 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -549,7 +549,6 @@ class ReplicationLayer(object): origin, pdu.room_id, pdu.event_id, ) - if not backfilled: ret = yield self.handler.on_receive_pdu( pdu, diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index d84a44c21..95c40c6c1 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -284,7 +284,7 @@ class TransportLayer(object): origin = None if request.method == "PUT": - #TODO: Handle other method types? other content types? + # TODO: Handle other method types? other content types? try: content_bytes = request.content.read() content = json.loads(content_bytes) @@ -296,11 +296,13 @@ class TransportLayer(object): try: params = auth.split(" ")[1].split(",") param_dict = dict(kv.split("=") for kv in params) + def strip_quotes(value): if value.startswith("\""): return value[1:-1] else: return value + origin = strip_quotes(param_dict["origin"]) key = strip_quotes(param_dict["key"]) sig = strip_quotes(param_dict["sig"]) @@ -321,7 +323,7 @@ class TransportLayer(object): if auth.startswith("X-Matrix"): (origin, key, sig) = parse_auth_header(auth) json_request["origin"] = origin - json_request["signatures"].setdefault(origin,{})[key] = sig + json_request["signatures"].setdefault(origin, {})[key] = sig if not json_request["signatures"]: raise SynapseError( @@ -515,7 +517,8 @@ class TransportLayer(object): return try: - code, response = yield self.received_handler.on_incoming_transaction( + handler = self.received_handler + code, response = yield handler.on_incoming_transaction( transaction_data ) except: diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d98014cac..f4e7b62bd 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -192,7 +192,9 @@ class Transaction(JsonEncodedObject): transaction_id and origin_server_ts keys. """ if "origin_server_ts" not in kwargs: - raise KeyError("Require 'origin_server_ts' to construct a Transaction") + raise KeyError( + "Require 'origin_server_ts' to construct a Transaction" + ) if "transaction_id" not in kwargs: raise KeyError( "Require 'transaction_id' to construct a Transaction" @@ -204,6 +206,3 @@ class Transaction(JsonEncodedObject): kwargs["pdus"] = [p.get_dict() for p in pdus] return Transaction(**kwargs) - - - diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 587fa308c..e909af6bd 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -521,6 +521,9 @@ class FederationHandler(BaseHandler): @log_function def _on_user_joined(self, user, room_id): - waiters = self.waiting_for_join_list.get((user.to_string(), room_id), []) + waiters = self.waiting_for_join_list.get( + (user.to_string(), room_id), + [] + ) while waiters: waiters.pop().callback(None) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7d810e6a6..4034437f6 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -494,11 +494,13 @@ def prepare_database(db_conn): user_version = row[0] if user_version > SCHEMA_VERSION: - raise ValueError("Cannot use this database as it is too " + + raise ValueError( + "Cannot use this database as it is too " + "new for the server to understand" ) elif user_version < SCHEMA_VERSION: - logging.info("Upgrading database from version %d", + logging.info( + "Upgrading database from version %d", user_version ) @@ -520,4 +522,3 @@ def prepare_database(db_conn): c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION) c.close() - diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index a70703014..a027db386 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -215,7 +215,7 @@ class EventFederationStore(SQLBaseStore): min_depth = self._simple_select_one_onecol_txn( txn, table="room_depth", - keyvalues={"room_id": room_id,}, + keyvalues={"room_id": room_id}, retcol="min_depth", allow_none=True, ) @@ -267,10 +267,8 @@ class EventFederationStore(SQLBaseStore): } ) - - - # We only insert as a forward extremity the new pdu if there are no - # other pdus that reference it as a prev pdu + # We only insert as a forward extremity the new pdu if there are + # no other pdus that reference it as a prev pdu query = ( "INSERT OR IGNORE INTO %(table)s (event_id, room_id) " "SELECT ?, ? WHERE NOT EXISTS (" @@ -312,7 +310,6 @@ class EventFederationStore(SQLBaseStore): ) txn.execute(query) - def get_backfill_events(self, room_id, event_list, limit): """Get a list of Events for a given topic that occured before (and including) the pdus in pdu_list. Return a list of max size `limit`. From b01159f2348de61a085d7ea20cc0fc90db2fc080 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 14:58:33 +0000 Subject: [PATCH 177/265] Fix room handler test --- tests/handlers/test_room.py | 55 +++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 146ed21b6..55c9f6e14 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -58,9 +58,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "profile_handler", "federation_handler", ]), - auth=NonCallableMock(spec_set=["check"]), + auth=NonCallableMock(spec_set=["check", "add_auth_events"]), state_handler=NonCallableMock(spec_set=[ - "handle_new_event", "annotate_state_groups", ]), config=self.mock_config, @@ -68,6 +67,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.federation = NonCallableMock(spec_set=[ "handle_new_event", + "send_invite", "get_state_for_room", ]) @@ -110,7 +110,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): joined = ["red", "green"] - self.state_handler.handle_new_event.return_value = defer.succeed(True) self.datastore.get_joined_hosts_for_room.return_value = ( defer.succeed(joined) ) @@ -120,6 +119,18 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_room_member.return_value = defer.succeed(None) + event.state_events = { + (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( + user_id="@alice:green", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( + user_id="@bob:red", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, target_user_id): event, + } + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -128,7 +139,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ) self.assertEquals( - set(["blue", "red", "green"]), + set(["red", "green"]), set(event.destinations) ) @@ -147,22 +158,14 @@ class RoomMemberHandlerTestCase(unittest.TestCase): room_id = "!foo:red" user_id = "@bob:red" user = self.hs.parse_userid(user_id) - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, + event = self._create_member( user_id=user_id, - state_key=target_user_id, room_id=room_id, - membership=Membership.JOIN, - content=content, ) joined = ["red", "green"] - self.state_handler.handle_new_event.return_value = defer.succeed(True) - def get_joined(*args): return defer.succeed(joined) @@ -180,6 +183,14 @@ class RoomMemberHandlerTestCase(unittest.TestCase): join_signal_observer = Mock() self.distributor.observe("user_joined_room", join_signal_observer) + event.state_events = { + (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( + user_id="@alice:green", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, user_id): event, + } + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -203,6 +214,16 @@ class RoomMemberHandlerTestCase(unittest.TestCase): user=user, room_id=room_id ) + def _create_member(self, user_id, room_id): + return self.hs.get_event_factory().create_event( + etype=RoomMemberEvent.TYPE, + user_id=user_id, + state_key=user_id, + room_id=room_id, + membership=Membership.JOIN, + content={"membership": Membership.JOIN}, + ) + class RoomCreationTest(unittest.TestCase): @@ -228,9 +249,8 @@ class RoomCreationTest(unittest.TestCase): "room_member_handler", "federation_handler", ]), - auth=NonCallableMock(spec_set=["check"]), + auth=NonCallableMock(spec_set=["check", "add_auth_events"]), state_handler=NonCallableMock(spec_set=[ - "handle_new_event", "annotate_state_groups", ]), ratelimiter=NonCallableMock(spec_set=[ @@ -259,6 +279,11 @@ class RoomCreationTest(unittest.TestCase): ]) self.room_member_handler = self.handlers.room_member_handler + def annotate(event): + event.state_events = {} + return defer.succeed(None) + self.state_handler.annotate_state_groups.side_effect = annotate + def hosts(room): return defer.succeed([]) self.datastore.get_joined_hosts_for_room.side_effect = hosts From cdc1b5d629b49300de848647bd2a80a024fae8f7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:21:30 +0000 Subject: [PATCH 178/265] Fix regression where we did not return redacted events. --- synapse/storage/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a23f2b941..2df64bdfe 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -469,7 +469,7 @@ class SQLBaseStore(object): select_event_sql = "SELECT * FROM events WHERE event_id = ?" - for ev in events: + for i, ev in enumerate(events): signatures = self._get_event_origin_signatures_txn( txn, ev.event_id, ) @@ -522,6 +522,7 @@ class SQLBaseStore(object): if del_evs: ev = prune_event(ev) + events[i] = ev ev.redacted_because = del_evs[0] return events From c174d19d1e107c31fe5d2d2b919bbd49a5b86e56 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:21:41 +0000 Subject: [PATCH 179/265] Fix redaction storage test --- tests/storage/test_redaction.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index fb175a5c4..adfe64a98 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -71,6 +71,8 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = [] + event.auth_events = [] # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -94,6 +96,7 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] yield self.store.persist_event( event @@ -115,6 +118,7 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] yield self.store.persist_event( event From e2cebe26e8ba1d7c2a39594ea1f4be0dee73ace4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:24:15 +0000 Subject: [PATCH 180/265] Fix room_member storage test --- tests/storage/test_roommember.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 9e2b2e31c..8614e5ca9 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -64,6 +64,8 @@ class RoomMemberStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = {} + event.auth_events = {} yield self.store.persist_event( event From 3cd9c02f71f967918f494e61121a8ecb18e67611 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:29:19 +0000 Subject: [PATCH 181/265] Fix stream test. --- tests/storage/test_stream.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index b034fd8f1..5038546ae 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -48,7 +48,7 @@ class StreamStoreTestCase(unittest.TestCase): self.depth = 1 @defer.inlineCallbacks - def inject_room_member(self, room, user, membership, prev_state=None): + def inject_room_member(self, room, user, membership, replaces_state=None): self.depth += 1 event = self.event_factory.create_event( @@ -64,9 +64,12 @@ class StreamStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = [] + event.auth_events = [] - if prev_state: - event.prev_state = prev_state + if replaces_state: + event.prev_state = [(replaces_state, "hash")] + event.replaces_state = replaces_state # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -90,6 +93,7 @@ class StreamStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -216,7 +220,7 @@ class StreamStoreTestCase(unittest.TestCase): event2 = yield self.inject_room_member( self.room1, self.u_alice, Membership.JOIN, - prev_state=event1.event_id, + replaces_state=event1.event_id, ) end = yield self.store.get_room_events_max_id() @@ -233,4 +237,7 @@ class StreamStoreTestCase(unittest.TestCase): event = results[0] - self.assertTrue(hasattr(event, "prev_content"), msg="No prev_content key") + self.assertTrue( + hasattr(event, "prev_content"), + msg="No prev_content key" + ) From 4ebdb1968225243bab86ef5d8d9f8feb274dd9ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:32:35 +0000 Subject: [PATCH 182/265] Fix SQLBaseStoreTestCase --- tests/storage/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 3ad9a4b0c..fabd364be 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -74,7 +74,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_select_one_1col(self): self.mock_txn.rowcount = 1 - self.mock_txn.fetchone.return_value = ("Value",) + self.mock_txn.fetchall.return_value = [("Value",)] value = yield self.datastore._simple_select_one_onecol( table="tablename", From ec824927c1e1af76abad1ea0c5eea1098a5c9cfa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:37:53 +0000 Subject: [PATCH 183/265] Fix rest.test_events. Convert to use SQLiteMemoryDbPool --- tests/rest/test_events.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 82f5d277e..4a3234c33 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -25,7 +25,7 @@ import synapse.rest.room from synapse.server import HomeServer -from ..utils import MockHttpResource, MemoryDataStore, MockKey +from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -113,25 +113,20 @@ class EventStreamPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - state_handler = Mock(spec=[ - "handle_new_event", - "annotate_state_groups" - ]) - state_handler.handle_new_event.return_value = True - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "test", - db_pool=None, + db_pool=db_pool, http_client=None, replication_layer=Mock(), - state_handler=state_handler, - datastore=MemoryDataStore(), persistence_service=persistence_service, clock=Mock(spec=[ "call_later", From a8e565eca8cbfcedbdfd812c98a6545c2fc31afd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 18:24:43 +0000 Subject: [PATCH 184/265] Add an EventValidator. Fix bugs in auth ++ storage --- synapse/api/auth.py | 16 ++++++--- synapse/api/events/__init__.py | 61 --------------------------------- synapse/handlers/profile.py | 11 +++--- synapse/rest/base.py | 2 ++ synapse/rest/room.py | 13 +++++++ synapse/server.py | 5 +++ synapse/storage/_base.py | 2 +- synapse/storage/registration.py | 6 +++- synapse/storage/room.py | 38 ++++++++++---------- 9 files changed, 64 insertions(+), 90 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a5c696470..6c2d3db26 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -70,6 +70,7 @@ class Auth(object): logger.debug("Denying! %s", event) return allowed + self.check_event_sender_in_room(event) self._can_send_event(event) if event.type == RoomPowerLevelsEvent.TYPE: @@ -83,8 +84,10 @@ class Auth(object): else: raise AuthError(500, "Unknown event: %s" % event) except AuthError as e: - logger.info("Event auth check failed on event %s with msg: %s", - event, e.msg) + logger.info( + "Event auth check failed on event %s with msg: %s", + event, e.msg + ) logger.info("Denying! %s", event) if raises: raise e @@ -277,7 +280,7 @@ class Auth(object): default=[""] )[0] if user and access_token and ip_addr: - self.store.insert_client_ip( + yield self.store.insert_client_ip( user=user, access_token=access_token, device_id=user_info["device_id"], @@ -349,7 +352,8 @@ class Auth(object): if event.type == RoomMemberEvent.TYPE: e_type = event.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: - auth_events.append(join_rule_event.event_id) + if join_rule_event: + auth_events.append(join_rule_event.event_id) if member_event and not is_public: auth_events.append(member_event.event_id) @@ -405,7 +409,9 @@ class Auth(object): if user_level < send_level: raise AuthError( - 403, "You don't have permission to post that to the room" + 403, + "You don't have permission to post that to the room. " + + "user_level (%d) < send_level (%d)" % (user_level, send_level) ) return True diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index f1e53f23a..1d8bed290 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.api.errors import SynapseError, Codes from synapse.util.jsonobject import JsonEncodedObject @@ -118,66 +117,6 @@ class SynapseEvent(JsonEncodedObject): """ raise NotImplementedError("get_content_template not implemented.") - def check_json(self, content, raises=True): - """Checks the given JSON content abides by the rules of the template. - - Args: - content : A JSON object to check. - raises: True to raise a SynapseError if the check fails. - Returns: - True if the content passes the template. Returns False if the check - fails and raises=False. - Raises: - SynapseError if the check fails and raises=True. - """ - # recursively call to inspect each layer - err_msg = self._check_json(content, self.get_content_template()) - if err_msg: - if raises: - raise SynapseError(400, err_msg, Codes.BAD_JSON) - else: - return False - else: - return True - - def _check_json(self, content, template): - """Check content and template matches. - - If the template is a dict, each key in the dict will be validated with - the content, else it will just compare the types of content and - template. This basic type check is required because this function will - be recursively called and could be called with just strs or ints. - - Args: - content: The content to validate. - template: The validation template. - Returns: - str: An error message if the validation fails, else None. - """ - if type(content) != type(template): - return "Mismatched types: %s" % template - - if type(template) == dict: - for key in template: - if key not in content: - return "Missing %s key" % key - - if type(content[key]) != type(template[key]): - return "Key %s is of the wrong type (got %s, want %s)" % ( - key, type(content[key]), type(template[key])) - - if type(content[key]) == dict: - # we must go deeper - msg = self._check_json(content[key], template[key]) - if msg: - return msg - elif type(content[key]) == list: - # make sure each item type in content matches the template - for entry in content[key]: - msg = self._check_json(entry, template[key][0]) - if msg: - return msg - class SynapseStateEvent(SynapseEvent): diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index e47814483..834b37f5f 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -152,10 +152,13 @@ class ProfileHandler(BaseHandler): if not user.is_mine: defer.returnValue(None) - (displayname, avatar_url) = yield defer.gatherResults([ - self.store.get_profile_displayname(user.localpart), - self.store.get_profile_avatar_url(user.localpart), - ]) + (displayname, avatar_url) = yield defer.gatherResults( + [ + self.store.get_profile_displayname(user.localpart), + self.store.get_profile_avatar_url(user.localpart), + ], + consumeErrors=True + ) state["displayname"] = displayname state["avatar_url"] = avatar_url diff --git a/synapse/rest/base.py b/synapse/rest/base.py index dc784c152..79fc4dfb8 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -67,6 +67,8 @@ class RestServlet(object): self.auth = hs.get_auth() self.txns = HttpTransactionStore() + self.validator = hs.get_event_validator() + def register(self, http_server): """ Register this servlet with the given HTTP server. """ if hasattr(self, "PATTERN"): diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 5c9c9d3af..05da0be09 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -154,6 +154,9 @@ class RoomStateEventRestServlet(RestServlet): user_id=user.to_string(), state_key=urllib.unquote(state_key) ) + + self.validator.validate(event) + if event_type == RoomMemberEvent.TYPE: # membership events are special handler = self.handlers.room_member_handler @@ -188,6 +191,8 @@ class RoomSendEventRestServlet(RestServlet): content=content ) + self.validator.validate(event) + msg_handler = self.handlers.message_handler yield msg_handler.send_message(event) @@ -253,6 +258,9 @@ class JoinRoomAliasServlet(RestServlet): user_id=user.to_string(), state_key=user.to_string() ) + + self.validator.validate(event) + handler = self.handlers.room_member_handler yield handler.change_membership(event) defer.returnValue((200, {})) @@ -424,6 +432,9 @@ class RoomMembershipRestServlet(RestServlet): user_id=user.to_string(), state_key=state_key ) + + self.validator.validate(event) + handler = self.handlers.room_member_handler yield handler.change_membership(event) defer.returnValue((200, {})) @@ -461,6 +472,8 @@ class RoomRedactEventRestServlet(RestServlet): redacts=urllib.unquote(event_id), ) + self.validator.validate(event) + msg_handler = self.handlers.message_handler yield msg_handler.send_message(event) diff --git a/synapse/server.py b/synapse/server.py index d770b20b1..da0a44433 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -22,6 +22,7 @@ from synapse.federation import initialize_http_replication from synapse.api.events import serialize_event from synapse.api.events.factory import EventFactory +from synapse.api.events.validator import EventValidator from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -80,6 +81,7 @@ class BaseHomeServer(object): 'event_sources', 'ratelimiter', 'keyring', + 'event_validator', ] def __init__(self, hostname, **kwargs): @@ -223,6 +225,9 @@ class HomeServer(BaseHomeServer): def build_keyring(self): return Keyring(self) + def build_event_validator(self): + return EventValidator(self) + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2df64bdfe..a1ee0318f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -212,7 +212,7 @@ class SQLBaseStore(object): retcol : string giving the name of the column to return """ return self.runInteraction( - "_simple_select_one_onecol_txn", + "_simple_select_one_onecol", self._simple_select_one_onecol_txn, table, keyvalues, retcol, allow_none=allow_none, ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a2ca6f9a6..1f89d7734 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -107,13 +107,17 @@ class RegistrationStore(SQLBaseStore): token ) + @defer.inlineCallbacks def is_server_admin(self, user): - return self._simple_select_one_onecol( + res = yield self._simple_select_one_onecol( table="users", keyvalues={"name": user.to_string()}, retcol="admin", + allow_none=True, ) + defer.returnValue(res if res else False) + def _query_for_auth(self, txn, token): sql = ( "SELECT users.name, users.admin, access_tokens.device_id " diff --git a/synapse/storage/room.py b/synapse/storage/room.py index ca70506d2..cc0513b8d 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -133,26 +133,28 @@ class RoomStore(SQLBaseStore): defer.returnValue(ret) def _store_room_topic_txn(self, txn, event): - self._simple_insert_txn( - txn, - "topics", - { - "event_id": event.event_id, - "room_id": event.room_id, - "topic": event.topic, - } - ) + if hasattr(event, "topic"): + self._simple_insert_txn( + txn, + "topics", + { + "event_id": event.event_id, + "room_id": event.room_id, + "topic": event.topic, + } + ) def _store_room_name_txn(self, txn, event): - self._simple_insert_txn( - txn, - "room_names", - { - "event_id": event.event_id, - "room_id": event.room_id, - "name": event.name, - } - ) + if hasattr(event, "name"): + self._simple_insert_txn( + txn, + "room_names", + { + "event_id": event.event_id, + "room_id": event.room_id, + "name": event.name, + } + ) class RoomsTable(Table): From 0b51d970b4d53ef9c2c66e3537f56331f436ae76 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 10 Nov 2014 18:43:16 +0000 Subject: [PATCH 185/265] document up the current architecture a bit based on the workshop the other week --- docs/client-server/OLD_specification.rst | 2 +- docs/implementation-notes/architecture.rst | 65 +++++++++++++++++++ .../python_architecture.rst | 6 ++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 docs/implementation-notes/architecture.rst diff --git a/docs/client-server/OLD_specification.rst b/docs/client-server/OLD_specification.rst index 47fba5eea..425ae57d9 100644 --- a/docs/client-server/OLD_specification.rst +++ b/docs/client-server/OLD_specification.rst @@ -4,7 +4,7 @@ Matrix Client-Server API .. WARNING:: - This specification is old. Please see /docs/specification.rst instead. + This specification is old. Please see matrix-doc/specification instead. diff --git a/docs/implementation-notes/architecture.rst b/docs/implementation-notes/architecture.rst new file mode 100644 index 000000000..b44785820 --- /dev/null +++ b/docs/implementation-notes/architecture.rst @@ -0,0 +1,65 @@ +Synapse Architecture +==================== + +As of the end of Oct 2014, Synapse's overall architecture looks like:: + + Notifier + ^ | + | | + .------------|------. + | handlers/ | | + | v | + | Event*Handler<---------> rest/* <=> Client + | Rooms*Handler | + HSes <=> federation/* <==> FederationHandler | + | | PresenceHandler | + | | TypingHandler | + | '-------------------' + | | | + | state/* | + | | | + | v v + `--------------> storage/* + | + v + .----. + | DB | + '----' + +* Handlers: business logic of synapse itself. Follows a set contract of BaseHandler: + + * BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic): + + * handle_state(event) + * auth(event) + * persist_event(event) + * notify notifier or federation(event) + + * PresenceHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + * TypingHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + * EventsHandler: handles the events stream... + * FederationHandler: - gets PDU from Federation Layer; turns into an event; + follows basehandler functionality. + * RoomsHandler: does all the room logic, including members - lots of classes in + RoomsHandler. + * ProfileHandler: talks to the storage to store/retrieve profile info. + +* EventFactory: generates events of particular event types. +* Notifier: Backs the events handler +* REST: Interfaces handlers and events to the outside world via HTTP/JSON. + Converts events back and forth from JSON. +* Federation: holds the HTTP client & server to talk to other servers. Does + replication to make sure there's nothing missing in the graph. Handles + reliability. Handles txns. +* Distributor: generic event bus. used for presence & typing only currently. + Notifier could be implemented using Distributor - so far we are only using for + things which actually /require/ dynamic pluggability however as it can + obfuscate the actual flow of control. +* Auth: helper singleton to say whether a given event is allowed to do a given + thing (TODO: put this on the diagram) +* State: helper singleton: does state conflict resolution. You give it an event + and it tells you if it actually updates the state or not, and annotates the + event up properly and handles merge conflict resolution. +* Storage: abstracts the storage engine. diff --git a/docs/implementation-notes/python_architecture.rst b/docs/implementation-notes/python_architecture.rst index 8beaa615d..2a5a2613c 100644 --- a/docs/implementation-notes/python_architecture.rst +++ b/docs/implementation-notes/python_architecture.rst @@ -1,3 +1,9 @@ +.. WARNING:: + These architecture notes are spectacularly old, and date back to when Synapse + was just federation code in isolation. This should be merged into the main + spec. + + = Server to Server = == Server to Server Stack == From f987393b32212f4fea4efa88a8d79a0d677e6f30 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 10 Nov 2014 21:56:52 +0000 Subject: [PATCH 186/265] moar boxes. --- docs/implementation-notes/architecture.rst | 39 ++++++++++++---------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/docs/implementation-notes/architecture.rst b/docs/implementation-notes/architecture.rst index b44785820..5b6463d78 100644 --- a/docs/implementation-notes/architecture.rst +++ b/docs/implementation-notes/architecture.rst @@ -3,24 +3,27 @@ Synapse Architecture As of the end of Oct 2014, Synapse's overall architecture looks like:: - Notifier - ^ | - | | - .------------|------. - | handlers/ | | - | v | - | Event*Handler<---------> rest/* <=> Client - | Rooms*Handler | - HSes <=> federation/* <==> FederationHandler | - | | PresenceHandler | - | | TypingHandler | - | '-------------------' - | | | - | state/* | - | | | - | v v - `--------------> storage/* - | + synapse + .-----------------------------------------------------. + | Notifier | + | ^ | | + | | | | + | .------------|------. | + | | handlers/ | | | + | | v | | + | | Event*Handler<---------> rest/* <=> Client + | | Rooms*Handler | | + HSes <=> federation/* <==> FederationHandler | | + | | | PresenceHandler | | + | | | TypingHandler | | + | | '-------------------' | + | | | | | + | | state/* | | + | | | | | + | | v v | + | `--------------> storage/* | + | | | + '--------------------------|--------------------------' v .----. | DB | From f45a6a70041dfb38a8f3266d17d4b9269dca0bf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 22:07:08 +0000 Subject: [PATCH 187/265] Fix RST sublist formatting bug --- docs/implementation-notes/architecture.rst | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/implementation-notes/architecture.rst b/docs/implementation-notes/architecture.rst index 5b6463d78..b8476ddbf 100644 --- a/docs/implementation-notes/architecture.rst +++ b/docs/implementation-notes/architecture.rst @@ -31,23 +31,23 @@ As of the end of Oct 2014, Synapse's overall architecture looks like:: * Handlers: business logic of synapse itself. Follows a set contract of BaseHandler: - * BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic): + - BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic): - * handle_state(event) - * auth(event) - * persist_event(event) - * notify notifier or federation(event) + + handle_state(event) + + auth(event) + + persist_event(event) + + notify notifier or federation(event) - * PresenceHandler: use distributor to get EDUs out of Federation. Very - lightweight logic built on the distributor - * TypingHandler: use distributor to get EDUs out of Federation. Very - lightweight logic built on the distributor - * EventsHandler: handles the events stream... - * FederationHandler: - gets PDU from Federation Layer; turns into an event; - follows basehandler functionality. - * RoomsHandler: does all the room logic, including members - lots of classes in - RoomsHandler. - * ProfileHandler: talks to the storage to store/retrieve profile info. + - PresenceHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + - TypingHandler: use distributor to get EDUs out of Federation. Very + lightweight logic built on the distributor + - EventsHandler: handles the events stream... + - FederationHandler: - gets PDU from Federation Layer; turns into an event; + follows basehandler functionality. + - RoomsHandler: does all the room logic, including members - lots of classes in + RoomsHandler. + - ProfileHandler: talks to the storage to store/retrieve profile info. * EventFactory: generates events of particular event types. * Notifier: Backs the events handler From 303b455965b3a9a4045fefa4f329ac805fb034e8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 11 Nov 2014 04:35:16 +0000 Subject: [PATCH 188/265] trivial spacing fix --- docs/implementation-notes/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/implementation-notes/architecture.rst b/docs/implementation-notes/architecture.rst index b8476ddbf..98050428b 100644 --- a/docs/implementation-notes/architecture.rst +++ b/docs/implementation-notes/architecture.rst @@ -11,7 +11,7 @@ As of the end of Oct 2014, Synapse's overall architecture looks like:: | .------------|------. | | | handlers/ | | | | | v | | - | | Event*Handler<---------> rest/* <=> Client + | | Event*Handler <--------> rest/* <=> Client | | Rooms*Handler | | HSes <=> federation/* <==> FederationHandler | | | | | PresenceHandler | | From b765dc005ba22cb41e01439f1b07d567d2de68da Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 11 Nov 2014 04:39:16 +0000 Subject: [PATCH 189/265] major CSS overhaul to try to make things look a bit cleaner --- syweb/webclient/app.css | 154 +++++++++++++++++++----- syweb/webclient/img/attach.png | Bin 0 -> 473 bytes syweb/webclient/img/settings.png | Bin 0 -> 864 bytes syweb/webclient/img/video.png | Bin 0 -> 604 bytes syweb/webclient/img/voice.png | Bin 0 -> 659 bytes syweb/webclient/index.html | 2 + syweb/webclient/js/angular-peity.js | 69 +++++++++++ syweb/webclient/js/jquery.peity.min.js | 13 ++ syweb/webclient/mobile.css | 18 +++ syweb/webclient/room/room-controller.js | 19 ++- syweb/webclient/room/room.html | 131 +++++++++----------- 11 files changed, 302 insertions(+), 104 deletions(-) create mode 100644 syweb/webclient/img/attach.png create mode 100644 syweb/webclient/img/settings.png create mode 100644 syweb/webclient/img/video.png create mode 100644 syweb/webclient/img/voice.png create mode 100644 syweb/webclient/js/angular-peity.js create mode 100644 syweb/webclient/js/jquery.peity.min.js diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css index be2a73872..e2f99f397 100755 --- a/syweb/webclient/app.css +++ b/syweb/webclient/app.css @@ -318,7 +318,7 @@ textarea, input { position: absolute; bottom: 0px; width: 100%; - height: 100px; + height: 70px; background-color: #f8f8f8; border-top: #aaa 1px solid; } @@ -326,7 +326,9 @@ textarea, input { #controls { max-width: 1280px; padding: 12px; + padding-right: 42px; margin: auto; + position: relative; } #buttonsCell { @@ -343,7 +345,17 @@ textarea, input { #mainInput { width: 100%; - resize: none; + padding: 5px; +} + +#attachButton { + position: absolute; + margin-top: 3px; + right: 0px; + background: url('img/attach.png'); + width: 25px; + height: 25px; + border: 0px; } .blink { @@ -415,7 +427,8 @@ textarea, input { .roomHeaderInfo { text-align: right; float: right; - margin-top: 15px; + margin-top: 0px; + margin-right: 30px; } /*** Room Info Dialog ***/ @@ -449,15 +462,32 @@ textarea, input { resize: vertical; } +/*** Control Buttons ***/ +#controlButtons { + float: right; + margin-right: -4px; + padding-bottom: 6px; +} + +.controlButton { + border: 0px; + width: 30px; + height: 30px; + padding-left: 3px; + padding-right: 3px; +} + /*** Participant list ***/ #usersTableWrapper { float: right; - width: 120px; + clear: right; + width: 100px; height: 100%; overflow-y: auto; } +/* #usersTable { width: 100%; border-collapse: collapse; @@ -473,36 +503,66 @@ textarea, input { position: relative; background-color: #000; } +*/ -.userAvatar .userAvatarImage { - position: absolute; - top: 0px; - object-fit: cover; - width: 100%; +.userAvatar { } +.userAvatarFrame { + border-radius: 46px; + width: 80px; + margin: auto; + position: relative; + border: 3px solid #aaa; + background-color: #aaa; +} + +.userAvatarImage { + border-radius: 40px; + text-align: center; + object-fit: cover; + display: block; +} + +/* .userAvatar .userAvatarGradient { position: absolute; bottom: 20px; width: 100%; } +*/ -.userAvatar .userName { - position: absolute; - color: #fff; - margin: 2px; - bottom: 0px; +.userName { + margin-top: 3px; + margin-bottom: 6px; + text-align: center; font-size: 12px; word-break: break-all; } -.userAvatar .userPowerLevel { +.userPowerLevel { position: absolute; - bottom: 0px; - height: 2px; + bottom: -1px; + height: 1px; background-color: #f00; } +.userPowerLevelBar { + display: inline; + position: absolute; + width: 2px; + height: 10px; +/* border: 1px solid #000; +*/ background-color: #aaa; +} + +.userPowerLevelMeter { + position: relative; + bottom: 0px; + background-color: #f00; +} + +/* .userPresence { text-align: center; font-size: 12px; @@ -510,12 +570,15 @@ textarea, input { background-color: #aaa; border-bottom: 1px #ddd solid; } +*/ .online { + border-color: #38AF00; background-color: #38AF00; } .unavailable { + border-color: #FFCC00; background-color: #FFCC00; } @@ -538,18 +601,21 @@ textarea, input { #messageTable td { padding: 0px; +/* border: 1px solid #888; */ } .leftBlock { - width: 14em; + width: 6em; word-wrap: break-word; vertical-align: top; background-color: #fff; - color: #888; + color: #aaa; font-weight: medium; font-size: 12px; text-align: right; +/* border-top: 1px #ddd solid; +*/ } .rightBlock { @@ -560,13 +626,24 @@ textarea, input { } .sender, .timestamp { - padding-right: 1em; - padding-left: 1em; - padding-top: 3px; +/* padding-top: 3px; +*/} + +.timestamp { + font-size: 10px; + color: #ccc; + height: 13px; + margin-top: 4px; +*/ transition-property: opacity; + transition-duration: 0.3s; } .sender { - margin-bottom: -3px; + font-size: 12px; +/* + margin-top: 5px; + margin-bottom: -9px; +*/ } .avatar { @@ -577,7 +654,11 @@ textarea, input { } .avatarImage { + position: relative; + top: 5px; object-fit: cover; + border-radius: 32px; + margin-top: 4px; } .emote { @@ -591,6 +672,7 @@ textarea, input { } .image { + border: 1px solid #888; display: block; max-width:320px; max-height:320px; @@ -603,19 +685,23 @@ textarea, input { } .bubble { +/* background-color: #eee; border: 1px solid #d8d8d8; - display: inline-block; margin-bottom: -1px; - max-width: 90%; - font-size: 14px; - word-wrap: break-word; padding-top: 7px; padding-bottom: 5px; + -webkit-text-size-adjust:100% + vertical-align: middle; +*/ + display: inline-block; + max-width: 90%; padding-left: 1em; padding-right: 1em; - vertical-align: middle; - -webkit-text-size-adjust:100% + padding-top: 2px; + padding-bottom: 2px; + font-size: 14px; + word-wrap: break-word; } .bubble img { @@ -623,9 +709,11 @@ textarea, input { max-height: auto; } +/* .differentUser td { padding-bottom: 5px ! important; } +*/ .mine { text-align: right; @@ -635,13 +723,15 @@ textarea, input { .text.membership .bubble, .mine .text.emote .bubble, .mine .text.membership .bubble - { +{ background-color: transparent ! important; border: 0px ! important; } .mine .text .bubble { +/* background-color: #f8f8ff ! important; +*/ text-align: left ! important; } @@ -701,6 +791,8 @@ textarea, input { overflow: hidden; white-space: nowrap; text-overflow: ellipsis; + padding-left: 0.5em; + padding-right: 0.5em; } .recentsRoom { @@ -751,7 +843,7 @@ textarea, input { padding-right: 10px; margin-right: 10px; height: 100%; - border-right: 1px solid #ddd; +/* border-right: 1px solid #ddd; */ overflow-y: auto; } diff --git a/syweb/webclient/img/attach.png b/syweb/webclient/img/attach.png new file mode 100644 index 0000000000000000000000000000000000000000..d95eabaf00b81c675e9d8a7d1f963b62474db1b1 GIT binary patch literal 473 zcmV;~0Ve*5P)P000>X1^@s6#OZ}&0000PbVXQnQ*UN; zcVTj606}DLVr3vnZDD6+Qe|Oed2z{QJOBUzdr3q=RCwBAT(DpP1Cc-uh(&>z6NtY9 z@oOM{M2toj0`dkxoD9T)KrD?#{wWZz2jaOvd;ymsO#lD?$0iOka1jun2jUzY7Qocy zK-De9rH+Y_kr7?Y42aJFaV!IAAPTC_48syMzC92h24V>Y@_+g!b7j8)r18qPN%n~6W zcA}9bAajM7ph=FIMwWogH3LPw3ZY0QRvt%UPyzXbhk>?$A7sg28e8&<1y@Om&FduO z6J+oR7tJ10j;wcnb0Meg}uV(xURgeY5A4xR)3she& zdIo30<6R}FtKJbX{2f%CJg!vALO^pa6oV=tV`vqPoCnSV@oqw11PCwy$U{(wnBPLL P00000NkvXXu0mjfrxLb_ literal 0 HcmV?d00001 diff --git a/syweb/webclient/img/settings.png b/syweb/webclient/img/settings.png new file mode 100644 index 0000000000000000000000000000000000000000..ac99fe402be68535a6b0ea8f0d6a6131d3bbff8b GIT binary patch literal 864 zcmV-m1E2hfP){&EeYwdB|>SS^w6uv7HUh8KqXM=!IKo?MRV|^1eAaf z5DG$}|1E#E<2XCJruM^Qb!X?ldH?_UI~%N4tCml@PuLW zRm0;u6Jaiwvr?&))oeDc*=**Fff)ab#iD)1_+&D1a|MnQ!b~WBO4u+h7o@O!K5wPd zY3F`C9=A!cIH>kezQJ5}sL>3bxhaJsC=n`fFOm3L?9ph{NdcZ43~eqW1Mx~gus+jgiTk%$vAR?Spa5 z*F}fz3X1!>5rnVX?ambq4B=BNSq&-1kz+%I`awQH+|!N1q1p(rwXj!(8o7a}oo+lH zTm8OP*I9sjr`9l0j(}`f5TKHR2uej4bFmXBKIP-eLY@(7ea!E}v8wO+4hwOTEw zdua?@r%;U1W49_63gkU0D^Zn+Zl@NYWXJN`34bn3N<>bv5Ht}v1yG19$cDqA$qhK( z%I{;S-XnZ=n*ryYPRCIKB7`1w0M+PTKKA*ZIC;v)+b@Jyrc;O&$)CiS-LU0>?c7$B8xB$5;%Gr$xqMF~@A0 zrg6t3kq8?Oht^?a)q+%wxcjtFC@@);9l@Bq)oT5`Gl*9dg{4v{77B$}v)SbB^IZeX zl?c?B89te0GC6??C>u1DyoHp>WM&CZP)uRI8ZKoErT z7B6TFf)zPB9~ekS{)0bXb5si8XSW*8_otP5ClOG0ww7A9?u(B z+TEjxe&K`X`{Vigp7%cQbD?ZDtAv^-mWTyngqY#>XX2B1Anu7rNwCp?{xM>c*dt

IY@mFBO$8?!^Ax?ZuJ1NFd1hVX59_ag1ZB@de+qerbdo)2zj3B72-N50t{2B)Q`48q2T@ifmVqd z@2xh8v!Gd~X||a{XhWYsXYBQDyiH65&5A~&J@t7s!CtO0lEID|oZw{|oQ+}wG=|rr zHXgK=3{T4QRXiSdw~^l#!x*7vVQde<*oN`=5N`Fjp@t+F*N5O+H#mn5T8_aLM4)gG zgu`LgY&P5a(1t$2VWIU5<2%X7a=EN5%TmQ+v8@kn=))MH?r=IGX}?Lb7xHmBomQn% z$^DL1wpuNfOeR%6pI5b7O=61DXc8;Ny*ri6K;T?1=Ncvw33uP(8f`*T#a>N)9%I$7 zk`v0CP@qbs(#=ElPE4T))D;L;y(K9GkfDJ07{qEjJ#-zkr7KTw0zqYPBr(Qnaqt6u zEl@^RAHPc + + diff --git a/syweb/webclient/js/angular-peity.js b/syweb/webclient/js/angular-peity.js new file mode 100644 index 000000000..2acb647d9 --- /dev/null +++ b/syweb/webclient/js/angular-peity.js @@ -0,0 +1,69 @@ +var angularPeity = angular.module( 'angular-peity', [] ); + +$.fn.peity.defaults.pie = { + fill: ["#ff0000", "#aaaaaa"], + radius: 4, +} + +var buildChartDirective = function ( chartType ) { + return { + restrict: 'E', + scope: { + data: "=", + options: "=" + }, + link: function ( scope, element, attrs ) { + + var options = {}; + if ( scope.options ) { + options = scope.options; + } + + // N.B. live-binding to data by Matthew + scope.$watch('data', function () { + var span = document.createElement( 'span' ); + span.textContent = scope.data.join(); + + if ( !attrs.class ) { + span.className = ""; + } else { + span.className = attrs.class; + } + + if (element[0].nodeType === 8) { + element.replaceWith( span ); + } + else if (element[0].firstChild) { + element.empty(); + element[0].appendChild( span ); + } + else { + element[0].appendChild( span ); + } + + jQuery( span ).peity( chartType, options ); + }); + } + }; +}; + + +angularPeity.directive( 'pieChart', function () { + + return buildChartDirective( "pie" ); + +} ); + + +angularPeity.directive( 'barChart', function () { + + return buildChartDirective( "bar" ); + +} ); + + +angularPeity.directive( 'lineChart', function () { + + return buildChartDirective( "line" ); + +} ); diff --git a/syweb/webclient/js/jquery.peity.min.js b/syweb/webclient/js/jquery.peity.min.js new file mode 100644 index 000000000..054b83c5d --- /dev/null +++ b/syweb/webclient/js/jquery.peity.min.js @@ -0,0 +1,13 @@ +// Peity jQuery plugin version 3.0.2 +// (c) 2014 Ben Pickles +// +// http://benpickles.github.io/peity +// +// Released under MIT license. +(function(h,w,i,v){var p=function(a,b){var d=w.createElementNS("http://www.w3.org/2000/svg",a);h(d).attr(b);return d},y="createElementNS"in w&&p("svg",{}).createSVGRect,e=h.fn.peity=function(a,b){y&&this.each(function(){var d=h(this),c=d.data("peity");c?(a&&(c.type=a),h.extend(c.opts,b)):(c=new x(d,a,h.extend({},e.defaults[a],b)),d.change(function(){c.draw()}).data("peity",c));c.draw()});return this},x=function(a,b,d){this.$el=a;this.type=b;this.opts=d},r=x.prototype;r.draw=function(){e.graphers[this.type].call(this, +this.opts)};r.fill=function(){var a=this.opts.fill;return h.isFunction(a)?a:function(b,d){return a[d%a.length]}};r.prepare=function(a,b){this.svg||this.$el.hide().after(this.svg=p("svg",{"class":"peity"}));return h(this.svg).empty().data("peity",this).attr({height:b,width:a})};r.values=function(){return h.map(this.$el.text().split(this.opts.delimiter),function(a){return parseFloat(a)})};e.defaults={};e.graphers={};e.register=function(a,b,d){this.defaults[a]=b;this.graphers[a]=d};e.register("pie", +{fill:["#ff9900","#fff4dd","#ffc66e"],radius:8},function(a){if(!a.delimiter){var b=this.$el.text().match(/[^0-9\.]/);a.delimiter=b?b[0]:","}b=this.values();if("/"==a.delimiter)var d=b[0],b=[d,i.max(0,b[1]-d)];for(var c=0,d=b.length,n=0;cj?m=q(i.min(d,0)):o=q(i.max(c,0)):u=1;u=o-m;0==u&&(u=1,0

+ + Invite a user: + + + +
+
+ +
- -
@@ -57,6 +66,26 @@
[matrix] + +
+ + + +
+
@@ -91,32 +120,30 @@
- +
- - - - -
- {{ member.displayname || member.id.substr(0, member.id.indexOf(':')) }} - -
-
-
- {{ member.id | mUserDisplayName: room_id }} -
-
- {{ member.id.substr(0, member.id.indexOf(':')) }}
- {{ member.id.substr(member.id.indexOf(':')) }} -
-
- 		- {{ member.last_active_ago + (now - member.last_updated) | duration }}
ago -
+
+
+ {{ member.displayname || member.id.substr(0, member.id.indexOf(':')) }} + +
+
+ + + {{ member.id | mUserDisplayName: room_id }} + + + {{ member.id.substr(0, member.id.indexOf(':')) }}
+ {{ member.id.substr(member.id.indexOf(':')) }} + + ({{ member.last_active_ago + (now - member.last_updated) | duration }}) +
+
-
{{ msg.__room_member.cnt.displayname || msg.user_id | mUserDisplayName: room_id }}
+
{{ (msg.origin_server_ts) | date:'MMM d HH:mm' }}
+
{{ msg.__room_member.cnt.displayname || msg.user_id | mUserDisplayName: room_id }}
 +
{{ msg.content.displayname || members[msg.state_key].displayname || msg.state_key }} joined @@ -222,49 +250,10 @@
- - - - \n\n\n\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['param_list'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, stack2, options, self=this, helperMissing=helpers.helperMissing, functionType="function", escapeExpression=this.escapeExpression; - -function program1(depth0,data) { - - - return " multiple='multiple'"; - } - -function program3(depth0,data) { - - - return "\n "; - } - -function program5(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.defaultValue, {hash:{},inverse:self.program(8, program8, data),fn:self.program(6, program6, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program6(depth0,data) { - - - return "\n "; - } - -function program8(depth0,data) { - - var buffer = "", stack1, stack2, options; - buffer += "\n "; - options = {hash:{},inverse:self.program(11, program11, data),fn:self.program(9, program9, data),data:data}; - stack2 = ((stack1 = helpers.isArray || depth0.isArray),stack1 ? stack1.call(depth0, depth0, options) : helperMissing.call(depth0, "isArray", depth0, options)); - if(stack2 || stack2 === 0) { buffer += stack2; } - buffer += "\n "; - return buffer; - } -function program9(depth0,data) { - - - return "\n "; - } - -function program11(depth0,data) { - - - return "\n \n "; - } - -function program13(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.isDefault, {hash:{},inverse:self.program(16, program16, data),fn:self.program(14, program14, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program14(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program16(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - - buffer += "\n\n\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['param_readonly'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, functionType="function", escapeExpression=this.escapeExpression, self=this; - -function program1(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program3(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.defaultValue, {hash:{},inverse:self.program(6, program6, data),fn:self.program(4, program4, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program4(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - if (stack1 = helpers.defaultValue) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.defaultValue; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + "\n "; - return buffer; - } - -function program6(depth0,data) { - - - return "\n (empty)\n "; - } - - buffer += "\n\n\n\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['param_readonly_required'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, functionType="function", escapeExpression=this.escapeExpression, self=this; - -function program1(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program3(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.defaultValue, {hash:{},inverse:self.program(6, program6, data),fn:self.program(4, program4, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program4(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - if (stack1 = helpers.defaultValue) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.defaultValue; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + "\n "; - return buffer; - } - -function program6(depth0,data) { - - - return "\n (empty)\n "; - } - - buffer += "\n\n\n\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['param_required'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, functionType="function", escapeExpression=this.escapeExpression, self=this; - -function program1(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.isFile, {hash:{},inverse:self.program(4, program4, data),fn:self.program(2, program2, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program2(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program4(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.defaultValue, {hash:{},inverse:self.program(7, program7, data),fn:self.program(5, program5, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program5(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program7(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n
\n
\n "; - return buffer; - } - -function program9(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.isFile, {hash:{},inverse:self.program(12, program12, data),fn:self.program(10, program10, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program10(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program12(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.defaultValue, {hash:{},inverse:self.program(15, program15, data),fn:self.program(13, program13, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program13(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program15(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - - buffer += "
\n\n\n\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['parameter_content_type'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, functionType="function", self=this; - -function program1(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers.each.call(depth0, depth0.consumes, {hash:{},inverse:self.noop,fn:self.program(2, program2, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n"; - return buffer; - } -function program2(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program4(depth0,data) { - - - return "\n \n"; - } - - buffer += "\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['resource'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, options, functionType="function", escapeExpression=this.escapeExpression, self=this, blockHelperMissing=helpers.blockHelperMissing; - -function program1(depth0,data) { - - - return " : "; - } - - buffer += "
\n

\n "; - if (stack1 = helpers.name) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.name; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + " "; - options = {hash:{},inverse:self.noop,fn:self.program(1, program1, data),data:data}; - if (stack1 = helpers.description) { stack1 = stack1.call(depth0, options); } - else { stack1 = depth0.description; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if (!helpers.description) { stack1 = blockHelperMissing.call(depth0, stack1, options); } - if(stack1 || stack1 === 0) { buffer += stack1; } - if (stack1 = helpers.description) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.description; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n

\n \n
\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['response_content_type'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, functionType="function", self=this; - -function program1(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers.each.call(depth0, depth0.produces, {hash:{},inverse:self.noop,fn:self.program(2, program2, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n"; - return buffer; - } -function program2(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program4(depth0,data) { - - - return "\n \n"; - } - - buffer += "\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['signature'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, functionType="function", escapeExpression=this.escapeExpression; - - - buffer += "
\n\n
\n\n
\n
\n "; - if (stack1 = helpers.signature) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.signature; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n
\n\n
\n 
";
-  if (stack1 = helpers.sampleJSON) { stack1 = stack1.call(depth0, {hash:{},data:data}); }
-  else { stack1 = depth0.sampleJSON; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; }
-  buffer += escapeExpression(stack1)
-    + "
\n \n
\n
\n\n"; - return buffer; - }); -})(); - -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['status_code'] = template(function (Handlebars,depth0,helpers,partials,data) { - this.compilerInfo = [4,'>= 1.0.0']; -helpers = this.merge(helpers, Handlebars.helpers); data = data || {}; - var buffer = "", stack1, functionType="function", escapeExpression=this.escapeExpression; - - - buffer += "
\n\n"; - return buffer; - }); -})(); - - - -// Generated by CoffeeScript 1.6.3 -(function() { - var ContentTypeView, HeaderView, MainView, OperationView, ParameterContentTypeView, ParameterView, ResourceView, ResponseContentTypeView, SignatureView, StatusCodeView, SwaggerUi, _ref, _ref1, _ref10, _ref2, _ref3, _ref4, _ref5, _ref6, _ref7, _ref8, _ref9, - __hasProp = {}.hasOwnProperty, - __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; }; - - SwaggerUi = (function(_super) { - __extends(SwaggerUi, _super); - - function SwaggerUi() { - _ref = SwaggerUi.__super__.constructor.apply(this, arguments); - return _ref; - } - - SwaggerUi.prototype.dom_id = "swagger_ui"; - - SwaggerUi.prototype.options = null; - - SwaggerUi.prototype.api = null; - - SwaggerUi.prototype.headerView = null; - - SwaggerUi.prototype.mainView = null; - - SwaggerUi.prototype.initialize = function(options) { - var _this = this; - if (options == null) { - options = {}; - } - if (options.dom_id != null) { - this.dom_id = options.dom_id; - delete options.dom_id; - } - if ($('#' + this.dom_id) == null) { - $('body').append('
'); - } - this.options = options; - this.options.success = function() { - return _this.render(); - }; - this.options.progress = function(d) { - return _this.showMessage(d); - }; - this.options.failure = function(d) { - return _this.onLoadFailure(d); - }; - this.headerView = new HeaderView({ - el: $('#header') - }); - return this.headerView.on('update-swagger-ui', function(data) { - return _this.updateSwaggerUi(data); - }); - }; - - SwaggerUi.prototype.updateSwaggerUi = function(data) { - this.options.url = data.url; - return this.load(); - }; - - SwaggerUi.prototype.load = function() { - var url, _ref1; - if ((_ref1 = this.mainView) != null) { - _ref1.clear(); - } - url = this.options.url; - if (url.indexOf("http") !== 0) { - url = this.buildUrl(window.location.href.toString(), url); - } - this.options.url = url; - this.headerView.update(url); - this.api = new SwaggerApi(this.options); - this.api.build(); - return this.api; - }; - - SwaggerUi.prototype.render = function() { - var _this = this; - this.showMessage('Finished Loading Resource Information. Rendering Swagger UI...'); - this.mainView = new MainView({ - model: this.api, - el: $('#' + this.dom_id), - swaggerOptions: this.options - }).render(); - this.showMessage(); - switch (this.options.docExpansion) { - case "full": - Docs.expandOperationsForResource(''); - break; - case "list": - Docs.collapseOperationsForResource(''); - } - if (this.options.onComplete) { - this.options.onComplete(this.api, this); - } - return setTimeout(function() { - return Docs.shebang(); - }, 400); - }; - - SwaggerUi.prototype.buildUrl = function(base, url) { - var endOfPath, parts; - log("base is " + base); - if (url.indexOf("/") === 0) { - parts = base.split("/"); - base = parts[0] + "//" + parts[2]; - return base + url; - } else { - endOfPath = base.length; - if (base.indexOf("?") > -1) { - endOfPath = Math.min(endOfPath, base.indexOf("?")); - } - if (base.indexOf("#") > -1) { - endOfPath = Math.min(endOfPath, base.indexOf("#")); - } - base = base.substring(0, endOfPath); - if (base.indexOf("/", base.length - 1) !== -1) { - return base + url; - } - return base + "/" + url; - } - }; - - SwaggerUi.prototype.showMessage = function(data) { - if (data == null) { - data = ''; - } - $('#message-bar').removeClass('message-fail'); - $('#message-bar').addClass('message-success'); - return $('#message-bar').html(data); - }; - - SwaggerUi.prototype.onLoadFailure = function(data) { - var val; - if (data == null) { - data = ''; - } - $('#message-bar').removeClass('message-success'); - $('#message-bar').addClass('message-fail'); - val = $('#message-bar').html(data); - if (this.options.onFailure != null) { - this.options.onFailure(data); - } - return val; - }; - - return SwaggerUi; - - })(Backbone.Router); - - window.SwaggerUi = SwaggerUi; - - HeaderView = (function(_super) { - __extends(HeaderView, _super); - - function HeaderView() { - _ref1 = HeaderView.__super__.constructor.apply(this, arguments); - return _ref1; - } - - HeaderView.prototype.events = { - 'click #show-pet-store-icon': 'showPetStore', - 'click #show-wordnik-dev-icon': 'showWordnikDev', - 'click #explore': 'showCustom', - 'keyup #input_baseUrl': 'showCustomOnKeyup', - 'keyup #input_apiKey': 'showCustomOnKeyup' - }; - - HeaderView.prototype.initialize = function() {}; - - HeaderView.prototype.showPetStore = function(e) { - return this.trigger('update-swagger-ui', { - url: "http://petstore.swagger.wordnik.com/api/api-docs" - }); - }; - - HeaderView.prototype.showWordnikDev = function(e) { - return this.trigger('update-swagger-ui', { - url: "http://api.wordnik.com/v4/resources.json" - }); - }; - - HeaderView.prototype.showCustomOnKeyup = function(e) { - if (e.keyCode === 13) { - return this.showCustom(); - } - }; - - HeaderView.prototype.showCustom = function(e) { - if (e != null) { - e.preventDefault(); - } - return this.trigger('update-swagger-ui', { - url: $('#input_baseUrl').val(), - apiKey: $('#input_apiKey').val() - }); - }; - - HeaderView.prototype.update = function(url, apiKey, trigger) { - if (trigger == null) { - trigger = false; - } - $('#input_baseUrl').val(url); - if (trigger) { - return this.trigger('update-swagger-ui', { - url: url - }); - } - }; - - return HeaderView; - - })(Backbone.View); - - MainView = (function(_super) { - var sorters; - - __extends(MainView, _super); - - function MainView() { - _ref2 = MainView.__super__.constructor.apply(this, arguments); - return _ref2; - } - - sorters = { - 'alpha': function(a, b) { - return a.path.localeCompare(b.path); - }, - 'method': function(a, b) { - return a.method.localeCompare(b.method); - } - }; - - MainView.prototype.initialize = function(opts) { - var route, sorter, sorterName, _i, _len, _ref3; - if (opts == null) { - opts = {}; - } - if (opts.swaggerOptions.sorter) { - sorterName = opts.swaggerOptions.sorter; - sorter = sorters[sorterName]; - _ref3 = this.model.apisArray; - for (_i = 0, _len = _ref3.length; _i < _len; _i++) { - route = _ref3[_i]; - route.operationsArray.sort(sorter); - } - if (sorterName === "alpha") { - return this.model.apisArray.sort(sorter); - } - } - }; - - MainView.prototype.render = function() { - var counter, id, resource, resources, _i, _len, _ref3; - $(this.el).html(Handlebars.templates.main(this.model)); - resources = {}; - counter = 0; - _ref3 = this.model.apisArray; - for (_i = 0, _len = _ref3.length; _i < _len; _i++) { - resource = _ref3[_i]; - id = resource.name; - while (typeof resources[id] !== 'undefined') { - id = id + "_" + counter; - counter += 1; - } - resource.id = id; - resources[id] = resource; - this.addResource(resource); - } - return this; - }; - - MainView.prototype.addResource = function(resource) { - var resourceView; - resourceView = new ResourceView({ - model: resource, - tagName: 'li', - id: 'resource_' + resource.id, - className: 'resource', - swaggerOptions: this.options.swaggerOptions - }); - return $('#resources').append(resourceView.render().el); - }; - - MainView.prototype.clear = function() { - return $(this.el).html(''); - }; - - return MainView; - - })(Backbone.View); - - ResourceView = (function(_super) { - __extends(ResourceView, _super); - - function ResourceView() { - _ref3 = ResourceView.__super__.constructor.apply(this, arguments); - return _ref3; - } - - ResourceView.prototype.initialize = function() {}; - - ResourceView.prototype.render = function() { - var counter, id, methods, operation, _i, _len, _ref4; - $(this.el).html(Handlebars.templates.resource(this.model)); - methods = {}; - _ref4 = this.model.operationsArray; - for (_i = 0, _len = _ref4.length; _i < _len; _i++) { - operation = _ref4[_i]; - counter = 0; - id = operation.nickname; - while (typeof methods[id] !== 'undefined') { - id = id + "_" + counter; - counter += 1; - } - methods[id] = operation; - operation.nickname = id; - operation.parentId = this.model.id; - this.addOperation(operation); - } - return this; - }; - - ResourceView.prototype.addOperation = function(operation) { - var operationView; - operation.number = this.number; - operationView = new OperationView({ - model: operation, - tagName: 'li', - className: 'endpoint', - swaggerOptions: this.options.swaggerOptions - }); - $('.endpoints', $(this.el)).append(operationView.render().el); - return this.number++; - }; - - return ResourceView; - - })(Backbone.View); - - OperationView = (function(_super) { - __extends(OperationView, _super); - - function OperationView() { - _ref4 = OperationView.__super__.constructor.apply(this, arguments); - return _ref4; - } - - OperationView.prototype.invocationUrl = null; - - OperationView.prototype.events = { - 'submit .sandbox': 'submitOperation', - 'click .submit': 'submitOperation', - 'click .response_hider': 'hideResponse', - 'click .toggleOperation': 'toggleOperationContent', - 'mouseenter .api-ic': 'mouseEnter', - 'mouseout .api-ic': 'mouseExit' - }; - - OperationView.prototype.initialize = function() {}; - - OperationView.prototype.mouseEnter = function(e) { - var elem, hgh, pos, scMaxX, scMaxY, scX, scY, wd, x, y; - elem = $(e.currentTarget.parentNode).find('#api_information_panel'); - x = e.pageX; - y = e.pageY; - scX = $(window).scrollLeft(); - scY = $(window).scrollTop(); - scMaxX = scX + $(window).width(); - scMaxY = scY + $(window).height(); - wd = elem.width(); - hgh = elem.height(); - if (x + wd > scMaxX) { - x = scMaxX - wd; - } - if (x < scX) { - x = scX; - } - if (y + hgh > scMaxY) { - y = scMaxY - hgh; - } - if (y < scY) { - y = scY; - } - pos = {}; - pos.top = y; - pos.left = x; - elem.css(pos); - return $(e.currentTarget.parentNode).find('#api_information_panel').show(); - }; - - OperationView.prototype.mouseExit = function(e) { - return $(e.currentTarget.parentNode).find('#api_information_panel').hide(); - }; - - OperationView.prototype.render = function() { - var contentTypeModel, isMethodSubmissionSupported, k, o, param, responseContentTypeView, responseSignatureView, signatureModel, statusCode, type, v, _i, _j, _k, _l, _len, _len1, _len2, _len3, _ref5, _ref6, _ref7, _ref8; - isMethodSubmissionSupported = true; - if (!isMethodSubmissionSupported) { - this.model.isReadOnly = true; - } - this.model.oauth = null; - if (this.model.authorizations) { - _ref5 = this.model.authorizations; - for (k in _ref5) { - v = _ref5[k]; - if (k === "oauth2") { - if (this.model.oauth === null) { - this.model.oauth = {}; - } - if (this.model.oauth.scopes === void 0) { - this.model.oauth.scopes = []; - } - for (_i = 0, _len = v.length; _i < _len; _i++) { - o = v[_i]; - this.model.oauth.scopes.push(o); - } - } - } - } - $(this.el).html(Handlebars.templates.operation(this.model)); - if (this.model.responseClassSignature && this.model.responseClassSignature !== 'string') { - signatureModel = { - sampleJSON: this.model.responseSampleJSON, - isParam: false, - signature: this.model.responseClassSignature - }; - responseSignatureView = new SignatureView({ - model: signatureModel, - tagName: 'div' - }); - $('.model-signature', $(this.el)).append(responseSignatureView.render().el); - } else { - $('.model-signature', $(this.el)).html(this.model.type); - } - contentTypeModel = { - isParam: false - }; - contentTypeModel.consumes = this.model.consumes; - contentTypeModel.produces = this.model.produces; - _ref6 = this.model.parameters; - for (_j = 0, _len1 = _ref6.length; _j < _len1; _j++) { - param = _ref6[_j]; - type = param.type || param.dataType; - if (type.toLowerCase() === 'file') { - if (!contentTypeModel.consumes) { - log("set content type "); - contentTypeModel.consumes = 'multipart/form-data'; - } - } - } - responseContentTypeView = new ResponseContentTypeView({ - model: contentTypeModel - }); - $('.response-content-type', $(this.el)).append(responseContentTypeView.render().el); - _ref7 = this.model.parameters; - for (_k = 0, _len2 = _ref7.length; _k < _len2; _k++) { - param = _ref7[_k]; - this.addParameter(param, contentTypeModel.consumes); - } - _ref8 = this.model.responseMessages; - for (_l = 0, _len3 = _ref8.length; _l < _len3; _l++) { - statusCode = _ref8[_l]; - this.addStatusCode(statusCode); - } - return this; - }; - - OperationView.prototype.addParameter = function(param, consumes) { - var paramView; - param.consumes = consumes; - paramView = new ParameterView({ - model: param, - tagName: 'tr', - readOnly: this.model.isReadOnly - }); - return $('.operation-params', $(this.el)).append(paramView.render().el); - }; - - OperationView.prototype.addStatusCode = function(statusCode) { - var statusCodeView; - statusCodeView = new StatusCodeView({ - model: statusCode, - tagName: 'tr' - }); - return $('.operation-status', $(this.el)).append(statusCodeView.render().el); - }; - - OperationView.prototype.submitOperation = function(e) { - var error_free, form, isFileUpload, map, o, opts, val, _i, _j, _k, _len, _len1, _len2, _ref5, _ref6, _ref7; - if (e != null) { - e.preventDefault(); - } - form = $('.sandbox', $(this.el)); - error_free = true; - form.find("input.required").each(function() { - var _this = this; - $(this).removeClass("error"); - if (jQuery.trim($(this).val()) === "") { - $(this).addClass("error"); - $(this).wiggle({ - callback: function() { - return $(_this).focus(); - } - }); - return error_free = false; - } - }); - if (error_free) { - map = {}; - opts = { - parent: this - }; - isFileUpload = false; - _ref5 = form.find("input"); - for (_i = 0, _len = _ref5.length; _i < _len; _i++) { - o = _ref5[_i]; - if ((o.value != null) && jQuery.trim(o.value).length > 0) { - map[o.name] = o.value; - } - if (o.type === "file") { - isFileUpload = true; - } - } - _ref6 = form.find("textarea"); - for (_j = 0, _len1 = _ref6.length; _j < _len1; _j++) { - o = _ref6[_j]; - if ((o.value != null) && jQuery.trim(o.value).length > 0) { - map["body"] = o.value; - } - } - _ref7 = form.find("select"); - for (_k = 0, _len2 = _ref7.length; _k < _len2; _k++) { - o = _ref7[_k]; - val = this.getSelectedValue(o); - if ((val != null) && jQuery.trim(val).length > 0) { - map[o.name] = val; - } - } - opts.responseContentType = $("div select[name=responseContentType]", $(this.el)).val(); - opts.requestContentType = $("div select[name=parameterContentType]", $(this.el)).val(); - $(".response_throbber", $(this.el)).show(); - if (isFileUpload) { - return this.handleFileUpload(map, form); - } else { - return this.model["do"](map, opts, this.showCompleteStatus, this.showErrorStatus, this); - } - } - }; - - OperationView.prototype.success = function(response, parent) { - return parent.showCompleteStatus(response); - }; - - OperationView.prototype.handleFileUpload = function(map, form) { - var bodyParam, el, headerParams, o, obj, param, params, _i, _j, _k, _l, _len, _len1, _len2, _len3, _ref5, _ref6, _ref7, _ref8, - _this = this; - _ref5 = form.serializeArray(); - for (_i = 0, _len = _ref5.length; _i < _len; _i++) { - o = _ref5[_i]; - if ((o.value != null) && jQuery.trim(o.value).length > 0) { - map[o.name] = o.value; - } - } - bodyParam = new FormData(); - params = 0; - _ref6 = this.model.parameters; - for (_j = 0, _len1 = _ref6.length; _j < _len1; _j++) { - param = _ref6[_j]; - if (param.paramType === 'form') { - if (param.type.toLowerCase() !== 'file' && map[param.name] !== void 0) { - bodyParam.append(param.name, map[param.name]); - } - } - } - headerParams = {}; - _ref7 = this.model.parameters; - for (_k = 0, _len2 = _ref7.length; _k < _len2; _k++) { - param = _ref7[_k]; - if (param.paramType === 'header') { - headerParams[param.name] = map[param.name]; - } - } - log(headerParams); - _ref8 = form.find('input[type~="file"]'); - for (_l = 0, _len3 = _ref8.length; _l < _len3; _l++) { - el = _ref8[_l]; - if (typeof el.files[0] !== 'undefined') { - bodyParam.append($(el).attr('name'), el.files[0]); - params += 1; - } - } - this.invocationUrl = this.model.supportHeaderParams() ? (headerParams = this.model.getHeaderParams(map), this.model.urlify(map, false)) : this.model.urlify(map, true); - $(".request_url", $(this.el)).html("
" + this.invocationUrl + "
"); - obj = { - type: this.model.method, - url: this.invocationUrl, - headers: headerParams, - data: bodyParam, - dataType: 'json', - contentType: false, - processData: false, - error: function(data, textStatus, error) { - return _this.showErrorStatus(_this.wrap(data), _this); - }, - success: function(data) { - return _this.showResponse(data, _this); - }, - complete: function(data) { - return _this.showCompleteStatus(_this.wrap(data), _this); - } - }; - if (window.authorizations) { - window.authorizations.apply(obj); - } - if (params === 0) { - obj.data.append("fake", "true"); - } - jQuery.ajax(obj); - return false; - }; - - OperationView.prototype.wrap = function(data) { - var h, headerArray, headers, i, o, _i, _len; - headers = {}; - headerArray = data.getAllResponseHeaders().split("\r"); - for (_i = 0, _len = headerArray.length; _i < _len; _i++) { - i = headerArray[_i]; - h = i.split(':'); - if (h[0] !== void 0 && h[1] !== void 0) { - headers[h[0].trim()] = h[1].trim(); - } - } - o = {}; - o.content = {}; - o.content.data = data.responseText; - o.headers = headers; - o.request = {}; - o.request.url = this.invocationUrl; - o.status = data.status; - return o; - }; - - OperationView.prototype.getSelectedValue = function(select) { - var opt, options, _i, _len, _ref5; - if (!select.multiple) { - return select.value; - } else { - options = []; - _ref5 = select.options; - for (_i = 0, _len = _ref5.length; _i < _len; _i++) { - opt = _ref5[_i]; - if (opt.selected) { - options.push(opt.value); - } - } - if (options.length > 0) { - return options.join(","); - } else { - return null; - } - } - }; - - OperationView.prototype.hideResponse = function(e) { - if (e != null) { - e.preventDefault(); - } - $(".response", $(this.el)).slideUp(); - return $(".response_hider", $(this.el)).fadeOut(); - }; - - OperationView.prototype.showResponse = function(response) { - var prettyJson; - prettyJson = JSON.stringify(response, null, "\t").replace(/\n/g, "
"); - return $(".response_body", $(this.el)).html(escape(prettyJson)); - }; - - OperationView.prototype.showErrorStatus = function(data, parent) { - return parent.showStatus(data); - }; - - OperationView.prototype.showCompleteStatus = function(data, parent) { - return parent.showStatus(data); - }; - - OperationView.prototype.formatXml = function(xml) { - var contexp, formatted, indent, lastType, lines, ln, pad, reg, transitions, wsexp, _fn, _i, _len; - reg = /(>)(<)(\/*)/g; - wsexp = /[ ]*(.*)[ ]+\n/g; - contexp = /(<.+>)(.+\n)/g; - xml = xml.replace(reg, '$1\n$2$3').replace(wsexp, '$1\n').replace(contexp, '$1\n$2'); - pad = 0; - formatted = ''; - lines = xml.split('\n'); - indent = 0; - lastType = 'other'; - transitions = { - 'single->single': 0, - 'single->closing': -1, - 'single->opening': 0, - 'single->other': 0, - 'closing->single': 0, - 'closing->closing': -1, - 'closing->opening': 0, - 'closing->other': 0, - 'opening->single': 1, - 'opening->closing': 0, - 'opening->opening': 1, - 'opening->other': 1, - 'other->single': 0, - 'other->closing': -1, - 'other->opening': 0, - 'other->other': 0 - }; - _fn = function(ln) { - var fromTo, j, key, padding, type, types, value; - types = { - single: Boolean(ln.match(/<.+\/>/)), - closing: Boolean(ln.match(/<\/.+>/)), - opening: Boolean(ln.match(/<[^!?].*>/)) - }; - type = ((function() { - var _results; - _results = []; - for (key in types) { - value = types[key]; - if (value) { - _results.push(key); - } - } - return _results; - })())[0]; - type = type === void 0 ? 'other' : type; - fromTo = lastType + '->' + type; - lastType = type; - padding = ''; - indent += transitions[fromTo]; - padding = ((function() { - var _j, _ref5, _results; - _results = []; - for (j = _j = 0, _ref5 = indent; 0 <= _ref5 ? _j < _ref5 : _j > _ref5; j = 0 <= _ref5 ? ++_j : --_j) { - _results.push(' '); - } - return _results; - })()).join(''); - if (fromTo === 'opening->closing') { - return formatted = formatted.substr(0, formatted.length - 1) + ln + '\n'; - } else { - return formatted += padding + ln + '\n'; - } - }; - for (_i = 0, _len = lines.length; _i < _len; _i++) { - ln = lines[_i]; - _fn(ln); - } - return formatted; - }; - - OperationView.prototype.showStatus = function(response) { - var code, content, contentType, headers, opts, pre, response_body, response_body_el, url; - if (response.content === void 0) { - content = response.data; - url = response.url; - } else { - content = response.content.data; - url = response.request.url; - } - headers = response.headers; - contentType = headers && headers["Content-Type"] ? headers["Content-Type"].split(";")[0].trim() : null; - if (!content) { - code = $('').text("no content"); - pre = $('
').append(code);
-      } else if (contentType === "application/json" || /\+json$/.test(contentType)) {
-        code = $('').text(JSON.stringify(JSON.parse(content), null, "  "));
-        pre = $('
').append(code);
-      } else if (contentType === "application/xml" || /\+xml$/.test(contentType)) {
-        code = $('').text(this.formatXml(content));
-        pre = $('
').append(code);
-      } else if (contentType === "text/html") {
-        code = $('').html(content);
-        pre = $('
').append(code);
-      } else if (/^image\//.test(contentType)) {
-        pre = $('').attr('src', url);
-      } else {
-        code = $('').text(content);
-        pre = $('
').append(code);
-      }
-      response_body = pre;
-      $(".request_url", $(this.el)).html("
" + url + "
");
-      $(".response_code", $(this.el)).html("
" + response.status + "
");
-      $(".response_body", $(this.el)).html(response_body);
-      $(".response_headers", $(this.el)).html("
" + _.escape(JSON.stringify(response.headers, null, "  ")).replace(/\n/g, "
") + "
");
-      $(".response", $(this.el)).slideDown();
-      $(".response_hider", $(this.el)).show();
-      $(".response_throbber", $(this.el)).hide();
-      response_body_el = $('.response_body', $(this.el))[0];
-      opts = this.options.swaggerOptions;
-      if (opts.highlightSizeThreshold && response.data.length > opts.highlightSizeThreshold) {
-        return response_body_el;
-      } else {
-        return hljs.highlightBlock(response_body_el);
-      }
-    };
-
-    OperationView.prototype.toggleOperationContent = function() {
-      var elem;
-      elem = $('#' + Docs.escapeResourceName(this.model.parentId) + "_" + this.model.nickname + "_content");
-      if (elem.is(':visible')) {
-        return Docs.collapseOperation(elem);
-      } else {
-        return Docs.expandOperation(elem);
-      }
-    };
-
-    return OperationView;
-
-  })(Backbone.View);
-
-  StatusCodeView = (function(_super) {
-    __extends(StatusCodeView, _super);
-
-    function StatusCodeView() {
-      _ref5 = StatusCodeView.__super__.constructor.apply(this, arguments);
-      return _ref5;
-    }
-
-    StatusCodeView.prototype.initialize = function() {};
-
-    StatusCodeView.prototype.render = function() {
-      var responseModel, responseModelView, template;
-      template = this.template();
-      $(this.el).html(template(this.model));
-      if (swaggerUi.api.models.hasOwnProperty(this.model.responseModel)) {
-        responseModel = {
-          sampleJSON: JSON.stringify(swaggerUi.api.models[this.model.responseModel].createJSONSample(), null, 2),
-          isParam: false,
-          signature: swaggerUi.api.models[this.model.responseModel].getMockSignature()
-        };
-        responseModelView = new SignatureView({
-          model: responseModel,
-          tagName: 'div'
-        });
-        $('.model-signature', this.$el).append(responseModelView.render().el);
-      } else {
-        $('.model-signature', this.$el).html('');
-      }
-      return this;
-    };
-
-    StatusCodeView.prototype.template = function() {
-      return Handlebars.templates.status_code;
-    };
-
-    return StatusCodeView;
-
-  })(Backbone.View);
-
-  ParameterView = (function(_super) {
-    __extends(ParameterView, _super);
-
-    function ParameterView() {
-      _ref6 = ParameterView.__super__.constructor.apply(this, arguments);
-      return _ref6;
-    }
-
-    ParameterView.prototype.initialize = function() {
-      return Handlebars.registerHelper('isArray', function(param, opts) {
-        if (param.type.toLowerCase() === 'array' || param.allowMultiple) {
-          return opts.fn(this);
-        } else {
-          return opts.inverse(this);
-        }
-      });
-    };
-
-    ParameterView.prototype.render = function() {
-      var contentTypeModel, isParam, parameterContentTypeView, responseContentTypeView, signatureModel, signatureView, template, type;
-      type = this.model.type || this.model.dataType;
-      if (this.model.paramType === 'body') {
-        this.model.isBody = true;
-      }
-      if (type.toLowerCase() === 'file') {
-        this.model.isFile = true;
-      }
-      template = this.template();
-      $(this.el).html(template(this.model));
-      signatureModel = {
-        sampleJSON: this.model.sampleJSON,
-        isParam: true,
-        signature: this.model.signature
-      };
-      if (this.model.sampleJSON) {
-        signatureView = new SignatureView({
-          model: signatureModel,
-          tagName: 'div'
-        });
-        $('.model-signature', $(this.el)).append(signatureView.render().el);
-      } else {
-        $('.model-signature', $(this.el)).html(this.model.signature);
-      }
-      isParam = false;
-      if (this.model.isBody) {
-        isParam = true;
-      }
-      contentTypeModel = {
-        isParam: isParam
-      };
-      contentTypeModel.consumes = this.model.consumes;
-      if (isParam) {
-        parameterContentTypeView = new ParameterContentTypeView({
-          model: contentTypeModel
-        });
-        $('.parameter-content-type', $(this.el)).append(parameterContentTypeView.render().el);
-      } else {
-        responseContentTypeView = new ResponseContentTypeView({
-          model: contentTypeModel
-        });
-        $('.response-content-type', $(this.el)).append(responseContentTypeView.render().el);
-      }
-      return this;
-    };
-
-    ParameterView.prototype.template = function() {
-      if (this.model.isList) {
-        return Handlebars.templates.param_list;
-      } else {
-        if (this.options.readOnly) {
-          if (this.model.required) {
-            return Handlebars.templates.param_readonly_required;
-          } else {
-            return Handlebars.templates.param_readonly;
-          }
-        } else {
-          if (this.model.required) {
-            return Handlebars.templates.param_required;
-          } else {
-            return Handlebars.templates.param;
-          }
-        }
-      }
-    };
-
-    return ParameterView;
-
-  })(Backbone.View);
-
-  SignatureView = (function(_super) {
-    __extends(SignatureView, _super);
-
-    function SignatureView() {
-      _ref7 = SignatureView.__super__.constructor.apply(this, arguments);
-      return _ref7;
-    }
-
-    SignatureView.prototype.events = {
-      'click a.description-link': 'switchToDescription',
-      'click a.snippet-link': 'switchToSnippet',
-      'mousedown .snippet': 'snippetToTextArea'
-    };
-
-    SignatureView.prototype.initialize = function() {};
-
-    SignatureView.prototype.render = function() {
-      var template;
-      template = this.template();
-      $(this.el).html(template(this.model));
-      this.switchToSnippet();
-      this.isParam = this.model.isParam;
-      if (this.isParam) {
-        $('.notice', $(this.el)).text('Click to set as parameter value');
-      }
-      return this;
-    };
-
-    SignatureView.prototype.template = function() {
-      return Handlebars.templates.signature;
-    };
-
-    SignatureView.prototype.switchToDescription = function(e) {
-      if (e != null) {
-        e.preventDefault();
-      }
-      $(".snippet", $(this.el)).hide();
-      $(".description", $(this.el)).show();
-      $('.description-link', $(this.el)).addClass('selected');
-      return $('.snippet-link', $(this.el)).removeClass('selected');
-    };
-
-    SignatureView.prototype.switchToSnippet = function(e) {
-      if (e != null) {
-        e.preventDefault();
-      }
-      $(".description", $(this.el)).hide();
-      $(".snippet", $(this.el)).show();
-      $('.snippet-link', $(this.el)).addClass('selected');
-      return $('.description-link', $(this.el)).removeClass('selected');
-    };
-
-    SignatureView.prototype.snippetToTextArea = function(e) {
-      var textArea;
-      if (this.isParam) {
-        if (e != null) {
-          e.preventDefault();
-        }
-        textArea = $('textarea', $(this.el.parentNode.parentNode.parentNode));
-        if ($.trim(textArea.val()) === '') {
-          return textArea.val(this.model.sampleJSON);
-        }
-      }
-    };
-
-    return SignatureView;
-
-  })(Backbone.View);
-
-  ContentTypeView = (function(_super) {
-    __extends(ContentTypeView, _super);
-
-    function ContentTypeView() {
-      _ref8 = ContentTypeView.__super__.constructor.apply(this, arguments);
-      return _ref8;
-    }
-
-    ContentTypeView.prototype.initialize = function() {};
-
-    ContentTypeView.prototype.render = function() {
-      var template;
-      template = this.template();
-      $(this.el).html(template(this.model));
-      $('label[for=contentType]', $(this.el)).text('Response Content Type');
-      return this;
-    };
-
-    ContentTypeView.prototype.template = function() {
-      return Handlebars.templates.content_type;
-    };
-
-    return ContentTypeView;
-
-  })(Backbone.View);
-
-  ResponseContentTypeView = (function(_super) {
-    __extends(ResponseContentTypeView, _super);
-
-    function ResponseContentTypeView() {
-      _ref9 = ResponseContentTypeView.__super__.constructor.apply(this, arguments);
-      return _ref9;
-    }
-
-    ResponseContentTypeView.prototype.initialize = function() {};
-
-    ResponseContentTypeView.prototype.render = function() {
-      var template;
-      template = this.template();
-      $(this.el).html(template(this.model));
-      $('label[for=responseContentType]', $(this.el)).text('Response Content Type');
-      return this;
-    };
-
-    ResponseContentTypeView.prototype.template = function() {
-      return Handlebars.templates.response_content_type;
-    };
-
-    return ResponseContentTypeView;
-
-  })(Backbone.View);
-
-  ParameterContentTypeView = (function(_super) {
-    __extends(ParameterContentTypeView, _super);
-
-    function ParameterContentTypeView() {
-      _ref10 = ParameterContentTypeView.__super__.constructor.apply(this, arguments);
-      return _ref10;
-    }
-
-    ParameterContentTypeView.prototype.initialize = function() {};
-
-    ParameterContentTypeView.prototype.render = function() {
-      var template;
-      template = this.template();
-      $(this.el).html(template(this.model));
-      $('label[for=parameterContentType]', $(this.el)).text('Parameter content type:');
-      return this;
-    };
-
-    ParameterContentTypeView.prototype.template = function() {
-      return Handlebars.templates.parameter_content_type;
-    };
-
-    return ParameterContentTypeView;
-
-  })(Backbone.View);
-
-}).call(this);
-
diff --git a/docs/client-server/web/files/swagger.js b/docs/client-server/web/files/swagger.js
deleted file mode 100644
index c126d21be..000000000
--- a/docs/client-server/web/files/swagger.js
+++ /dev/null
@@ -1,1604 +0,0 @@
-// swagger.js
-// version 2.0.37
-
-var __bind = function(fn, me){
-  return function(){
-    return fn.apply(me, arguments);
-  };
-};
-
-log = function(){
-  log.history = log.history || [];
-  log.history.push(arguments);
-  if(this.console){
-    console.log( Array.prototype.slice.call(arguments)[0] );
-  }
-};
-
-if (!Array.prototype.indexOf) {
-  Array.prototype.indexOf = function(obj, start) {
-    for (var i = (start || 0), j = this.length; i < j; i++) {
-      if (this[i] === obj) { return i; }
-    }
-    return -1;
-  }
-}
-
-if (!('filter' in Array.prototype)) {
-  Array.prototype.filter= function(filter, that /*opt*/) {
-    var other= [], v;
-    for (var i=0, n= this.length; i 0)
-    this.basePath = this.url.substring(0, this.url.lastIndexOf('?'));
-  else
-    this.basePath = this.url;
-
-  if (isApi) {
-    var newName = response.resourcePath.replace(/\//g, '');
-    this.resourcePath = response.resourcePath;
-    res = new SwaggerResource(response, this);
-    this.apis[newName] = res;
-    this.apisArray.push(res);
-  } else {
-    var k;
-    for (k = 0; k < response.apis.length; k++) {
-      var resource = response.apis[k];
-      res = new SwaggerResource(resource, this);
-      this.apis[res.name] = res;
-      this.apisArray.push(res);
-    }
-  }
-  if (this.success) {
-    this.success();
-  }
-  return this;
-};
-
-SwaggerApi.prototype.buildFrom1_1Spec = function(response) {
-  log("This API is using a deprecated version of Swagger!  Please see http://github.com/wordnik/swagger-core/wiki for more info");
-  if (response.apiVersion != null)
-    this.apiVersion = response.apiVersion;
-  this.apis = {};
-  this.apisArray = [];
-  this.produces = response.produces;
-  if (response.info != null) {
-    this.info = response.info;
-  }
-  var isApi = false;
-  for (var i = 0; i < response.apis.length; i++) {
-    var api = response.apis[i];
-    if (api.operations) {
-      for (var j = 0; j < api.operations.length; j++) {
-        operation = api.operations[j];
-        isApi = true;
-      }
-    }
-  }
-  if (response.basePath) {
-    this.basePath = response.basePath;
-  } else if (this.url.indexOf('?') > 0) {
-    this.basePath = this.url.substring(0, this.url.lastIndexOf('?'));
-  } else {
-    this.basePath = this.url;
-  }
-  if (isApi) {
-    var newName = response.resourcePath.replace(/\//g, '');
-    this.resourcePath = response.resourcePath;
-    var res = new SwaggerResource(response, this);
-    this.apis[newName] = res;
-    this.apisArray.push(res);
-  } else {
-    for (k = 0; k < response.apis.length; k++) {
-      resource = response.apis[k];
-      res = new SwaggerResource(resource, this);
-      this.apis[res.name] = res;
-      this.apisArray.push(res);
-    }
-  }
-  if (this.success) {
-    this.success();
-  }
-  return this;
-};
-
-SwaggerApi.prototype.selfReflect = function() {
-  var resource, resource_name, _ref;
-  if (this.apis == null) {
-    return false;
-  }
-  _ref = this.apis;
-  for (resource_name in _ref) {
-    resource = _ref[resource_name];
-    if (resource.ready == null) {
-      return false;
-    }
-  }
-  this.setConsolidatedModels();
-  this.ready = true;
-  if (this.success != null) {
-    return this.success();
-  }
-};
-
-SwaggerApi.prototype.fail = function(message) {
-  this.failure(message);
-  throw message;
-};
-
-SwaggerApi.prototype.setConsolidatedModels = function() {
-  var model, modelName, resource, resource_name, _i, _len, _ref, _ref1, _results;
-  this.modelsArray = [];
-  this.models = {};
-  _ref = this.apis;
-  for (resource_name in _ref) {
-    resource = _ref[resource_name];
-    for (modelName in resource.models) {
-      if (this.models[modelName] == null) {
-        this.models[modelName] = resource.models[modelName];
-        this.modelsArray.push(resource.models[modelName]);
-      }
-    }
-  }
-  _ref1 = this.modelsArray;
-  _results = [];
-  for (_i = 0, _len = _ref1.length; _i < _len; _i++) {
-    model = _ref1[_i];
-    _results.push(model.setReferencedModels(this.models));
-  }
-  return _results;
-};
-
-SwaggerApi.prototype.help = function() {
-  var operation, operation_name, parameter, resource, resource_name, _i, _len, _ref, _ref1, _ref2;
-  _ref = this.apis;
-  for (resource_name in _ref) {
-    resource = _ref[resource_name];
-    log(resource_name);
-    _ref1 = resource.operations;
-    for (operation_name in _ref1) {
-      operation = _ref1[operation_name];
-      log("  " + operation.nickname);
-      _ref2 = operation.parameters;
-      for (_i = 0, _len = _ref2.length; _i < _len; _i++) {
-        parameter = _ref2[_i];
-        log("    " + parameter.name + (parameter.required ? ' (required)' : '') + " - " + parameter.description);
-      }
-    }
-  }
-  return this;
-};
-
-var SwaggerResource = function(resourceObj, api) {
-  var _this = this;
-  this.api = api;
-  this.api = this.api;
-  consumes = (this.consumes | []);
-  produces = (this.produces | []);
-  this.path = this.api.resourcePath != null ? this.api.resourcePath : resourceObj.path;
-  this.description = resourceObj.description;
-
-  var parts = this.path.split("/");
-  this.name = parts[parts.length - 1].replace('.{format}', '');
-  this.basePath = this.api.basePath;
-  this.operations = {};
-  this.operationsArray = [];
-  this.modelsArray = [];
-  this.models = {};
-  this.rawModels = {};
-  this.useJQuery = (typeof api.useJQuery !== 'undefined' ? api.useJQuery : null);
-
-  if ((resourceObj.apis != null) && (this.api.resourcePath != null)) {
-    this.addApiDeclaration(resourceObj);
-  } else {
-    if (this.path == null) {
-      this.api.fail("SwaggerResources must have a path.");
-    }
-    if (this.path.substring(0, 4) === 'http') {
-      this.url = this.path.replace('{format}', 'json');
-    } else {
-      this.url = this.api.basePath + this.path.replace('{format}', 'json');
-    }
-    this.api.progress('fetching resource ' + this.name + ': ' + this.url);
-    obj = {
-      url: this.url,
-      method: "get",
-      useJQuery: this.useJQuery,
-      headers: {
-        accept: "application/json"
-      },
-      on: {
-        response: function(resp) {
-          var responseObj = resp.obj || JSON.parse(resp.data);
-          return _this.addApiDeclaration(responseObj);
-        },
-        error: function(response) {
-          return _this.api.fail("Unable to read api '" +
-            _this.name + "' from path " + _this.url + " (server returned " + response.statusText + ")");
-        }
-      }
-    };
-    var e = typeof window !== 'undefined' ? window : exports;
-    e.authorizations.apply(obj);
-    new SwaggerHttp().execute(obj);
-  }
-}
-
-SwaggerResource.prototype.getAbsoluteBasePath = function (relativeBasePath) {
-  var pos, url;
-  url = this.api.basePath;
-  pos = url.lastIndexOf(relativeBasePath);
-  var parts = url.split("/");
-  var rootUrl = parts[0] + "//" + parts[2];
-
-  if(relativeBasePath.indexOf("http") === 0)
-    return relativeBasePath;
-  if(relativeBasePath === "/")
-    return rootUrl;
-  if(relativeBasePath.substring(0, 1) == "/") {
-    // use root + relative
-    return rootUrl + relativeBasePath;
-  }
-  else {
-    var pos = this.basePath.lastIndexOf("/");
-    var base = this.basePath.substring(0, pos);
-    if(base.substring(base.length - 1) == "/")
-      return base + relativeBasePath;
-    else
-      return base + "/" + relativeBasePath;
-  }
-};
-
-SwaggerResource.prototype.addApiDeclaration = function(response) {
-  if (response.produces != null)
-    this.produces = response.produces;
-  if (response.consumes != null)
-    this.consumes = response.consumes;
-  if ((response.basePath != null) && response.basePath.replace(/\s/g, '').length > 0)
-    this.basePath = response.basePath.indexOf("http") === -1 ? this.getAbsoluteBasePath(response.basePath) : response.basePath;
-
-  this.addModels(response.models);
-  if (response.apis) {
-    for (var i = 0 ; i < response.apis.length; i++) {
-      var endpoint = response.apis[i];
-      this.addOperations(endpoint.path, endpoint.operations, response.consumes, response.produces);
-    }
-  }
-  this.api[this.name] = this;
-  this.ready = true;
-  return this.api.selfReflect();
-};
-
-SwaggerResource.prototype.addModels = function(models) {
-  if (models != null) {
-    var modelName;
-    for (modelName in models) {
-      if (this.models[modelName] == null) {
-        var swaggerModel = new SwaggerModel(modelName, models[modelName]);
-        this.modelsArray.push(swaggerModel);
-        this.models[modelName] = swaggerModel;
-        this.rawModels[modelName] = models[modelName];
-      }
-    }
-    var output = [];
-    for (var i = 0; i < this.modelsArray.length; i++) {
-      model = this.modelsArray[i];
-      output.push(model.setReferencedModels(this.models));
-    }
-    return output;
-  }
-};
-
-SwaggerResource.prototype.addOperations = function(resource_path, ops, consumes, produces) {
-  if (ops) {
-    output = [];
-    for (var i = 0; i < ops.length; i++) {
-      o = ops[i];
-      consumes = this.consumes;
-      produces = this.produces;
-      if (o.consumes != null)
-        consumes = o.consumes;
-      else
-        consumes = this.consumes;
-
-      if (o.produces != null)
-        produces = o.produces;
-      else
-        produces = this.produces;
-      type = (o.type||o.responseClass);
-
-      if (type === "array") {
-        ref = null;
-        if (o.items)
-          ref = o.items["type"] || o.items["$ref"];
-        type = "array[" + ref + "]";
-      }
-      responseMessages = o.responseMessages;
-      method = o.method;
-      if (o.httpMethod) {
-        method = o.httpMethod;
-      }
-      if (o.supportedContentTypes) {
-        consumes = o.supportedContentTypes;
-      }
-      if (o.errorResponses) {
-        responseMessages = o.errorResponses;
-        for (var j = 0; j < responseMessages.length; j++) {
-          r = responseMessages[j];
-          r.message = r.reason;
-          r.reason = null;
-        }
-      }
-      o.nickname = this.sanitize(o.nickname);
-      op = new SwaggerOperation(o.nickname, resource_path, method, o.parameters, o.summary, o.notes, type, responseMessages, this, consumes, produces, o.authorizations);
-      this.operations[op.nickname] = op;
-      output.push(this.operationsArray.push(op));
-    }
-    return output;
-  }
-};
-
-SwaggerResource.prototype.sanitize = function(nickname) {
-  var op;
-  op = nickname.replace(/[\s!@#$%^&*()_+=\[{\]};:<>|.\/?,\\'""-]/g, '_');
-  op = op.replace(/((_){2,})/g, '_');
-  op = op.replace(/^(_)*/g, '');
-  op = op.replace(/([_])*$/g, '');
-  return op;
-};
-
-SwaggerResource.prototype.help = function() {
-  var op = this.operations;
-  var output = [];
-  var operation_name;
-  for (operation_name in op) {
-    operation = op[operation_name];
-    var msg = "  " + operation.nickname;
-    for (var i = 0; i < operation.parameters; i++) {
-      parameter = operation.parameters[i];
-      msg.concat("    " + parameter.name + (parameter.required ? ' (required)' : '') + " - " + parameter.description);
-    }
-    output.push(msg);
-  }
-  return output;
-};
-
-var SwaggerModel = function(modelName, obj) {
-  this.name = obj.id != null ? obj.id : modelName;
-  this.properties = [];
-  var propertyName;
-  for (propertyName in obj.properties) {
-    if (obj.required != null) {
-      var value;
-      for (value in obj.required) {
-        if (propertyName === obj.required[value]) {
-          obj.properties[propertyName].required = true;
-        }
-      }
-    }
-    prop = new SwaggerModelProperty(propertyName, obj.properties[propertyName]);
-    this.properties.push(prop);
-  }
-}
-
-SwaggerModel.prototype.setReferencedModels = function(allModels) {
-  var results = [];
-  for (var i = 0; i < this.properties.length; i++) {
-    var property = this.properties[i];
-    var type = property.type || property.dataType;
-    if (allModels[type] != null)
-      results.push(property.refModel = allModels[type]);
-    else if ((property.refDataType != null) && (allModels[property.refDataType] != null))
-      results.push(property.refModel = allModels[property.refDataType]);
-    else
-      results.push(void 0);
-  }
-  return results;
-};
-
-SwaggerModel.prototype.getMockSignature = function(modelsToIgnore) {
-  var propertiesStr = [];
-  for (var i = 0; i < this.properties.length; i++) {
-    prop = this.properties[i];
-    propertiesStr.push(prop.toString());
-  }
-
-  var strong = '';
-  var stronger = '';
-  var strongClose = '';
-  var classOpen = strong + this.name + ' {' + strongClose;
-  var classClose = strong + '}' + strongClose;
-  var returnVal = classOpen + '
' + propertiesStr.join(',
') + '
' + classClose;
-  if (!modelsToIgnore)
-    modelsToIgnore = [];
-  modelsToIgnore.push(this.name);
-
-  for (var i = 0; i < this.properties.length; i++) {
-    prop = this.properties[i];
-    if ((prop.refModel != null) && modelsToIgnore.indexOf(prop.refModel.name) === -1) {
-      returnVal = returnVal + ('
' + prop.refModel.getMockSignature(modelsToIgnore));
-    }
-  }
-  return returnVal;
-};
-
-SwaggerModel.prototype.createJSONSample = function(modelsToIgnore) {
-  if(sampleModels[this.name]) {
-    return sampleModels[this.name];
-  }
-  else {
-    var result = {};
-    var modelsToIgnore = (modelsToIgnore||[])
-    modelsToIgnore.push(this.name);
-    for (var i = 0; i < this.properties.length; i++) {
-      prop = this.properties[i];
-      result[prop.name] = prop.getSampleValue(modelsToIgnore);
-    }
-    modelsToIgnore.pop(this.name);
-    return result;
-  }
-};
-
-var SwaggerModelProperty = function(name, obj) {
-  this.name = name;
-  this.dataType = obj.type || obj.dataType || obj["$ref"];
-  this.isCollection = this.dataType && (this.dataType.toLowerCase() === 'array' || this.dataType.toLowerCase() === 'list' || this.dataType.toLowerCase() === 'set');
-  this.descr = obj.description;
-  this.required = obj.required;
-  if (obj.items != null) {
-    if (obj.items.type != null) {
-      this.refDataType = obj.items.type;
-    }
-    if (obj.items.$ref != null) {
-      this.refDataType = obj.items.$ref;
-    }
-  }
-  this.dataTypeWithRef = this.refDataType != null ? (this.dataType + '[' + this.refDataType + ']') : this.dataType;
-  if (obj.allowableValues != null) {
-    this.valueType = obj.allowableValues.valueType;
-    this.values = obj.allowableValues.values;
-    if (this.values != null) {
-      this.valuesString = "'" + this.values.join("' or '") + "'";
-    }
-  }
-  if (obj["enum"] != null) {
-    this.valueType = "string";
-    this.values = obj["enum"];
-    if (this.values != null) {
-      this.valueString = "'" + this.values.join("' or '") + "'";
-    }
-  }
-}
-
-SwaggerModelProperty.prototype.getSampleValue = function(modelsToIgnore) {
-  var result;
-  if ((this.refModel != null) && (modelsToIgnore.indexOf(prop.refModel.name) === -1)) {
-    result = this.refModel.createJSONSample(modelsToIgnore);
-  } else {
-    if (this.isCollection) {
-      result = this.toSampleValue(this.refDataType);
-    } else {
-      result = this.toSampleValue(this.dataType);
-    }
-  }
-  if (this.isCollection) {
-    return [result];
-  } else {
-    return result;
-  }
-};
-
-SwaggerModelProperty.prototype.toSampleValue = function(value) {
-  var result;
-  if (value === "integer") {
-    result = 0;
-  } else if (value === "boolean") {
-    result = false;
-  } else if (value === "double" || value === "number") {
-    result = 0.0;
-  } else if (value === "string") {
-    result = "";
-  } else {
-    result = value;
-  }
-  return result;
-};
-
-SwaggerModelProperty.prototype.toString = function() {
-  var req = this.required ? 'propReq' : 'propOpt';
-  var str = '' + this.name + ' (' + this.dataTypeWithRef + '';
-  if (!this.required) {
-    str += ', optional';
-  }
-  str += ')';
-  if (this.values != null) {
-    str += " = ['" + this.values.join("' or '") + "']";
-  }
-  if (this.descr != null) {
-    str += ': ' + this.descr + '';
-  }
-  return str;
-};
-
-var SwaggerOperation = function(nickname, path, method, parameters, summary, notes, type, responseMessages, resource, consumes, produces, authorizations) {
-  var _this = this;
-
-  var errors = [];
-  this.nickname = (nickname||errors.push("SwaggerOperations must have a nickname."));
-  this.path = (path||errors.push("SwaggerOperation " + nickname + " is missing path."));
-  this.method = (method||errors.push("SwaggerOperation " + nickname + " is missing method."));
-  this.parameters = parameters != null ? parameters : [];
-  this.summary = summary;
-  this.notes = notes;
-  this.type = type;
-  this.responseMessages = (responseMessages||[]);
-  this.resource = (resource||errors.push("Resource is required"));
-  this.consumes = consumes;
-  this.produces = produces;
-  this.authorizations = authorizations;
-  this["do"] = __bind(this["do"], this);
-
-  if (errors.length > 0)
-    this.resource.api.fail(errors);
-
-  this.path = this.path.replace('{format}', 'json');
-  this.method = this.method.toLowerCase();
-  this.isGetMethod = this.method === "get";
-
-  this.resourceName = this.resource.name;
-  if(typeof this.type !== 'undefined' && this.type === 'void')
-    this.type = null;
-  else {
-    this.responseClassSignature = this.getSignature(this.type, this.resource.models);
-    this.responseSampleJSON = this.getSampleJSON(this.type, this.resource.models);
-  }
-
-  for(var i = 0; i < this.parameters.length; i ++) {
-    var param = this.parameters[i];
-    // might take this away
-    param.name = param.name || param.type || param.dataType;
-
-    // for 1.1 compatibility
-    var type = param.type || param.dataType;
-    if(type === 'array') {
-      type = 'array[' + (param.items.$ref ? param.items.$ref : param.items.type) + ']';
-    }
-    param.type = type;
-
-    if(type.toLowerCase() === 'boolean') {
-      param.allowableValues = {};
-      param.allowableValues.values = ["true", "false"];
-    }
-    param.signature = this.getSignature(type, this.resource.models);
-    param.sampleJSON = this.getSampleJSON(type, this.resource.models);
-
-    var enumValue = param["enum"];
-    if(enumValue != null) {
-      param.isList = true;
-      param.allowableValues = {};
-      param.allowableValues.descriptiveValues = [];
-
-      for(var j = 0; j < enumValue.length; j++) {
-        var v = enumValue[j];
-        if(param.defaultValue != null) {
-          param.allowableValues.descriptiveValues.push ({
-            value: String(v),
-            isDefault: (v === param.defaultValue)
-          });
-        }
-        else {
-          param.allowableValues.descriptiveValues.push ({
-            value: String(v),
-            isDefault: false
-          });
-        }
-      }
-    }
-    else if(param.allowableValues != null) {
-      if(param.allowableValues.valueType === "RANGE")
-        param.isRange = true;
-      else
-        param.isList = true;
-      if(param.allowableValues != null) {
-        param.allowableValues.descriptiveValues = [];
-        if(param.allowableValues.values) {
-          for(var j = 0; j < param.allowableValues.values.length; j++){
-            var v = param.allowableValues.values[j];
-            if(param.defaultValue != null) {
-              param.allowableValues.descriptiveValues.push ({
-                value: String(v),
-                isDefault: (v === param.defaultValue)
-              });
-            }
-            else {
-              param.allowableValues.descriptiveValues.push ({
-                value: String(v),
-                isDefault: false
-              });
-            }
-          }
-        }
-      }
-    }
-  }
-  this.resource[this.nickname] = function(args, callback, error) {
-    return _this["do"](args, callback, error);
-  };
-  this.resource[this.nickname].help = function() {
-    return _this.help();
-  };
-}
-
-SwaggerOperation.prototype.isListType = function(type) {
-  if (type && type.indexOf('[') >= 0) {
-    return type.substring(type.indexOf('[') + 1, type.indexOf(']'));
-  } else {
-    return void 0;
-  }
-};
-
-SwaggerOperation.prototype.getSignature = function(type, models) {
-  var isPrimitive, listType;
-  listType = this.isListType(type);
-  isPrimitive = ((listType != null) && models[listType]) || (models[type] != null) ? false : true;
-  if (isPrimitive) {
-    return type;
-  } else {
-    if (listType != null) {
-      return models[listType].getMockSignature();
-    } else {
-      return models[type].getMockSignature();
-    }
-  }
-};
-
-SwaggerOperation.prototype.getSampleJSON = function(type, models) {
-  var isPrimitive, listType, val;
-  listType = this.isListType(type);
-  isPrimitive = ((listType != null) && models[listType]) || (models[type] != null) ? false : true;
-  val = isPrimitive ? void 0 : (listType != null ? models[listType].createJSONSample() : models[type].createJSONSample());
-  if (val) {
-    val = listType ? [val] : val;
-    if(typeof val == "string")
-      return val;
-    else if(typeof val === "object") {
-      var t = val;
-      if(val instanceof Array && val.length > 0) {
-        t = val[0];
-      }
-      if(t.nodeName) {
-        var xmlString = new XMLSerializer().serializeToString(t);
-        return this.formatXml(xmlString);
-      }
-      else
-        return JSON.stringify(val, null, 2);
-    }
-    else
-      return val;
-  }
-};
-
-SwaggerOperation.prototype["do"] = function(args, opts, callback, error) {
-  var key, param, params, possibleParams, req, requestContentType, responseContentType, value, _i, _len, _ref;
-  if (args == null) {
-    args = {};
-  }
-  if (opts == null) {
-    opts = {};
-  }
-  requestContentType = null;
-  responseContentType = null;
-  if ((typeof args) === "function") {
-    error = opts;
-    callback = args;
-    args = {};
-  }
-  if ((typeof opts) === "function") {
-    error = callback;
-    callback = opts;
-  }
-  if (error == null) {
-    error = function(xhr, textStatus, error) {
-      return log(xhr, textStatus, error);
-    };
-  }
-  if (callback == null) {
-    callback = function(response) {
-      var content;
-      content = null;
-      if (response != null) {
-        content = response.data;
-      } else {
-        content = "no data";
-      }
-      return log("default callback: " + content);
-    };
-  }
-  params = {};
-  params.headers = [];
-  if (args.headers != null) {
-    params.headers = args.headers;
-    delete args.headers;
-  }
-
-  var possibleParams = [];
-  for(var i = 0; i < this.parameters.length; i++) {
-    var param = this.parameters[i];
-    if(param.paramType === 'header') {
-      if(args[param.name])
-        params.headers[param.name] = args[param.name];
-    }
-    else if(param.paramType === 'form' || param.paramType.toLowerCase() === 'file')
-      possibleParams.push(param);
-  }
-
-  if (args.body != null) {
-    params.body = args.body;
-    delete args.body;
-  }
-
-  if (possibleParams) {
-    var key;
-    for (key in possibleParams) {
-      value = possibleParams[key];
-      if (args[value.name]) {
-        params[value.name] = args[value.name];
-      }
-    }
-  }
-
-  req = new SwaggerRequest(this.method, this.urlify(args), params, opts, callback, error, this);
-  if (opts.mock != null) {
-    return req;
-  } else {
-    return true;
-  }
-};
-
-SwaggerOperation.prototype.pathJson = function() {
-  return this.path.replace("{format}", "json");
-};
-
-SwaggerOperation.prototype.pathXml = function() {
-  return this.path.replace("{format}", "xml");
-};
-
-SwaggerOperation.prototype.encodePathParam = function(pathParam) {
-  var encParts, part, parts, _i, _len;
-  pathParam = pathParam.toString();
-  if (pathParam.indexOf("/") === -1) {
-    return encodeURIComponent(pathParam);
-  } else {
-    parts = pathParam.split("/");
-    encParts = [];
-    for (_i = 0, _len = parts.length; _i < _len; _i++) {
-      part = parts[_i];
-      encParts.push(encodeURIComponent(part));
-    }
-    return encParts.join("/");
-  }
-};
-
-SwaggerOperation.prototype.urlify = function(args) {
-  var url = this.resource.basePath + this.pathJson();
-  var params = this.parameters;
-  for(var i = 0; i < params.length; i ++){
-    var param = params[i];
-    if (param.paramType === 'path') {
-      if(args[param.name]) {
-        // apply path params and remove from args
-        var reg = new RegExp('\{' + param.name + '[^\}]*\}', 'gi');
-        url = url.replace(reg, this.encodePathParam(args[param.name]));
-        delete args[param.name];
-      }
-      else
-        throw "" + param.name + " is a required path param.";
-    }
-  }
-
-  var queryParams = "";
-  for(var i = 0; i < params.length; i ++){
-    var param = params[i];
-    if(param.paramType === 'query') {
-      if (args[param.name] !== undefined) {
-        if (queryParams !== '')
-          queryParams += "&";
-        queryParams += encodeURIComponent(param.name) + '=' + encodeURIComponent(args[param.name]);
-      }
-    }
-  }
-  if ((queryParams != null) && queryParams.length > 0)
-    url += '?' + queryParams;
-  return url;
-};
-
-SwaggerOperation.prototype.supportHeaderParams = function() {
-  return this.resource.api.supportHeaderParams;
-};
-
-SwaggerOperation.prototype.supportedSubmitMethods = function() {
-  return this.resource.api.supportedSubmitMethods;
-};
-
-SwaggerOperation.prototype.getQueryParams = function(args) {
-  return this.getMatchingParams(['query'], args);
-};
-
-SwaggerOperation.prototype.getHeaderParams = function(args) {
-  return this.getMatchingParams(['header'], args);
-};
-
-SwaggerOperation.prototype.getMatchingParams = function(paramTypes, args) {
-  var matchingParams = {};
-  var params = this.parameters;
-  for (var i = 0; i < params.length; i++) {
-    param = params[i];
-    if (args && args[param.name])
-      matchingParams[param.name] = args[param.name];
-  }
-  var headers = this.resource.api.headers;
-  var name;
-  for (name in headers) {
-    var value = headers[name];
-    matchingParams[name] = value;
-  }
-  return matchingParams;
-};
-
-SwaggerOperation.prototype.help = function() {
-  var msg = "";
-  var params = this.parameters;
-  for (var i = 0; i < params.length; i++) {
-    var param = params[i];
-    if (msg !== "")
-      msg += "\n";
-    msg += "* " + param.name + (param.required ? ' (required)' : '') + " - " + param.description;
-  }
-  return msg;
-};
-
-
-SwaggerOperation.prototype.formatXml = function(xml) {
-  var contexp, formatted, indent, lastType, lines, ln, pad, reg, transitions, wsexp, _fn, _i, _len;
-  reg = /(>)(<)(\/*)/g;
-  wsexp = /[ ]*(.*)[ ]+\n/g;
-  contexp = /(<.+>)(.+\n)/g;
-  xml = xml.replace(reg, '$1\n$2$3').replace(wsexp, '$1\n').replace(contexp, '$1\n$2');
-  pad = 0;
-  formatted = '';
-  lines = xml.split('\n');
-  indent = 0;
-  lastType = 'other';
-  transitions = {
-    'single->single': 0,
-    'single->closing': -1,
-    'single->opening': 0,
-    'single->other': 0,
-    'closing->single': 0,
-    'closing->closing': -1,
-    'closing->opening': 0,
-    'closing->other': 0,
-    'opening->single': 1,
-    'opening->closing': 0,
-    'opening->opening': 1,
-    'opening->other': 1,
-    'other->single': 0,
-    'other->closing': -1,
-    'other->opening': 0,
-    'other->other': 0
-  };
-  _fn = function(ln) {
-    var fromTo, j, key, padding, type, types, value;
-    types = {
-      single: Boolean(ln.match(/<.+\/>/)),
-      closing: Boolean(ln.match(/<\/.+>/)),
-      opening: Boolean(ln.match(/<[^!?].*>/))
-    };
-    type = ((function() {
-      var _results;
-      _results = [];
-      for (key in types) {
-        value = types[key];
-        if (value) {
-          _results.push(key);
-        }
-      }
-      return _results;
-    })())[0];
-    type = type === void 0 ? 'other' : type;
-    fromTo = lastType + '->' + type;
-    lastType = type;
-    padding = '';
-    indent += transitions[fromTo];
-    padding = ((function() {
-      var _j, _ref5, _results;
-      _results = [];
-      for (j = _j = 0, _ref5 = indent; 0 <= _ref5 ? _j < _ref5 : _j > _ref5; j = 0 <= _ref5 ? ++_j : --_j) {
-        _results.push('  ');
-      }
-      return _results;
-    })()).join('');
-    if (fromTo === 'opening->closing') {
-      return formatted = formatted.substr(0, formatted.length - 1) + ln + '\n';
-    } else {
-      return formatted += padding + ln + '\n';
-    }
-  };
-  for (_i = 0, _len = lines.length; _i < _len; _i++) {
-    ln = lines[_i];
-    _fn(ln);
-  }
-  return formatted;
-};
-
-var SwaggerRequest = function(type, url, params, opts, successCallback, errorCallback, operation, execution) {
-  var _this = this;
-  var errors = [];
-  this.useJQuery = (typeof operation.resource.useJQuery !== 'undefined' ? operation.resource.useJQuery : null);
-  this.type = (type||errors.push("SwaggerRequest type is required (get/post/put/delete/patch/options)."));
-  this.url = (url||errors.push("SwaggerRequest url is required."));
-  this.params = params;
-  this.opts = opts;
-  this.successCallback = (successCallback||errors.push("SwaggerRequest successCallback is required."));
-  this.errorCallback = (errorCallback||errors.push("SwaggerRequest error callback is required."));
-  this.operation = (operation||errors.push("SwaggerRequest operation is required."));
-  this.execution = execution;
-  this.headers = (params.headers||{});
-
-  if(errors.length > 0) {
-    throw errors;
-  }
-
-  this.type = this.type.toUpperCase();
-
-  // set request, response content type headers
-  var headers = this.setHeaders(params, this.operation);
-  var body = params.body;
-
-  // encode the body for form submits
-  if (headers["Content-Type"]) {
-    var values = {};
-    var i;
-    var operationParams = this.operation.parameters;
-    for(i = 0; i < operationParams.length; i++) {
-      var param = operationParams[i];
-      if(param.paramType === "form")
-        values[param.name] = param;
-    }
-
-    if(headers["Content-Type"].indexOf("application/x-www-form-urlencoded") === 0) {
-      var encoded = "";
-      var key;
-      for(key in values) {
-        value = this.params[key];
-        if(typeof value !== 'undefined'){
-          if(encoded !== "")
-            encoded += "&";
-          encoded += encodeURIComponent(key) + '=' + encodeURIComponent(value);
-        }
-      }
-      body = encoded;
-    }
-    else if (headers["Content-Type"].indexOf("multipart/form-data") === 0) {
-      // encode the body for form submits
-      var data = "";
-      var boundary = "----SwaggerFormBoundary" + Date.now();
-      var key;
-      for(key in values) {
-        value = this.params[key];
-        if(typeof value !== 'undefined') {
-          data += '--' + boundary + '\n';
-          data += 'Content-Disposition: form-data; name="' + key + '"';
-          data += '\n\n';
-          data += value + "\n";
-        }
-      }
-      data += "--" + boundary + "--\n";
-      headers["Content-Type"] = "multipart/form-data; boundary=" + boundary;
-      body = data;
-    }
-  }
-
-  if (!((this.headers != null) && (this.headers.mock != null))) {
-    obj = {
-      url: this.url,
-      method: this.type,
-      headers: headers,
-      body: body,
-      useJQuery: this.useJQuery,
-      on: {
-        error: function(response) {
-          return _this.errorCallback(response, _this.opts.parent);
-        },
-        redirect: function(response) {
-          return _this.successCallback(response, _this.opts.parent);
-        },
-        307: function(response) {
-          return _this.successCallback(response, _this.opts.parent);
-        },
-        response: function(response) {
-          return _this.successCallback(response, _this.opts.parent);
-        }
-      }
-    };
-    var e;
-    if (typeof window !== 'undefined') {
-      e = window;
-    } else {
-      e = exports;
-    }
-    status = e.authorizations.apply(obj, this.operation.authorizations);
-    if (opts.mock == null) {
-      if (status !== false) {
-        new SwaggerHttp().execute(obj);
-      } else {
-        obj.canceled = true;
-      }
-    } else {
-      return obj;
-    }
-  }
-};
-
-SwaggerRequest.prototype.setHeaders = function(params, operation) {
-  // default type
-  var accepts = "application/json";
-  var consumes = "application/json";
-
-  var allDefinedParams = this.operation.parameters;
-  var definedFormParams = [];
-  var definedFileParams = [];
-  var body = params.body;
-  var headers = {};
-
-  // get params from the operation and set them in definedFileParams, definedFormParams, headers
-  var i;
-  for(i = 0; i < allDefinedParams.length; i++) {
-    var param = allDefinedParams[i];
-    if(param.paramType === "form")
-      definedFormParams.push(param);
-    else if(param.paramType === "file")
-      definedFileParams.push(param);
-    else if(param.paramType === "header" && this.params.headers) {
-      var key = param.name;
-      var headerValue = this.params.headers[param.name];
-      if(typeof this.params.headers[param.name] !== 'undefined')
-        headers[key] = headerValue;
-    }
-  }
-
-  // if there's a body, need to set the accepts header via requestContentType
-  if (body && (this.type === "POST" || this.type === "PUT" || this.type === "PATCH" || this.type === "DELETE")) {
-    if (this.opts.requestContentType)
-      accepts = this.opts.requestContentType;
-  } else {
-    // if any form params, content type must be set
-    if(definedFormParams.length > 0) {
-      if(definedFileParams.length > 0)
-        consumes = "multipart/form-data";
-      else
-        consumes = "application/x-www-form-urlencoded";
-    }
-    else if (this.type == "DELETE")
-      body = "{}";
-    else if (this.type != "DELETE")
-      accepts = null;
-  }
-
-  if (consumes && this.operation.consumes) {
-    if (this.operation.consumes.indexOf(consumes) === -1) {
-      log("server doesn't consume " + consumes + ", try " + JSON.stringify(this.operation.consumes));
-      consumes = this.operation.consumes[0];
-    }
-  }
-
-  if (this.opts.responseContentType) {
-    accepts = this.opts.responseContentType;
-  } else {
-    accepts = "application/json";
-  }
-  if (accepts && this.operation.produces) {
-    if (this.operation.produces.indexOf(accepts) === -1) {
-      log("server can't produce " + accepts);
-      accepts = this.operation.produces[0];
-    }
-  }
-
-  if ((consumes && body !== "") || (consumes === "application/x-www-form-urlencoded"))
-    headers["Content-Type"] = consumes;
-  if (accepts)
-    headers["Accept"] = accepts;
-  return headers;
-}
-
-SwaggerRequest.prototype.asCurl = function() {
-  var results = [];
-  if(this.headers) {
-    var key;
-    for(key in this.headers) {
-      results.push("--header \"" + key + ": " + this.headers[v] + "\"");
-    }
-  }
-  return "curl " + (results.join(" ")) + " " + this.url;
-};
-
-/**
- * SwaggerHttp is a wrapper for executing requests
- */
-var SwaggerHttp = function() {};
-
-SwaggerHttp.prototype.execute = function(obj) {
-  if(obj && (typeof obj.useJQuery === 'boolean'))
-    this.useJQuery = obj.useJQuery;
-  else
-    this.useJQuery = this.isIE8();
-
-  if(this.useJQuery)
-    return new JQueryHttpClient().execute(obj);
-  else
-    return new ShredHttpClient().execute(obj);
-}
-
-SwaggerHttp.prototype.isIE8 = function() {
-  var detectedIE = false;
-  if (typeof navigator !== 'undefined' && navigator.userAgent) {
-    nav = navigator.userAgent.toLowerCase();
-    if (nav.indexOf('msie') !== -1) {
-      var version = parseInt(nav.split('msie')[1]);
-      if (version <= 8) {
-        detectedIE = true;
-      }
-    }
-  }
-  return detectedIE;
-};
-
-/*
- * JQueryHttpClient lets a browser take advantage of JQuery's cross-browser magic.
- * NOTE: when jQuery is available it will export both '$' and 'jQuery' to the global space.
- *       Since we are using closures here we need to alias it for internal use.
- */
-var JQueryHttpClient = function(options) {
-  "use strict";
-  if(!jQuery){
-    var jQuery = window.jQuery;
-  }
-}
-
-JQueryHttpClient.prototype.execute = function(obj) {
-  var cb = obj.on;
-  var request = obj;
-
-  obj.type = obj.method;
-  obj.cache = false;
-
-  obj.beforeSend = function(xhr) {
-    var key, results;
-    if (obj.headers) {
-      results = [];
-      var key;
-      for (key in obj.headers) {
-        if (key.toLowerCase() === "content-type") {
-          results.push(obj.contentType = obj.headers[key]);
-        } else if (key.toLowerCase() === "accept") {
-          results.push(obj.accepts = obj.headers[key]);
-        } else {
-          results.push(xhr.setRequestHeader(key, obj.headers[key]));
-        }
-      }
-      return results;
-    }
-  };
-
-  obj.data = obj.body;
-  obj.complete = function(response, textStatus, opts) {
-    var headers = {},
-        headerArray = response.getAllResponseHeaders().split("\n");
-
-    for(var i = 0; i < headerArray.length; i++) {
-      var toSplit = headerArray[i].trim();
-      if(toSplit.length === 0)
-        continue;
-      var separator = toSplit.indexOf(":");
-      if(separator === -1) {
-        // Name but no value in the header
-        headers[toSplit] = null;
-        continue;
-      }
-      var name = toSplit.substring(0, separator).trim(),
-          value = toSplit.substring(separator + 1).trim();
-      headers[name] = value;
-    }
-
-    var out = {
-      url: request.url,
-      method: request.method,
-      status: response.status,
-      data: response.responseText,
-      headers: headers
-    };
-
-    var contentType = (headers["content-type"]||headers["Content-Type"]||null)
-
-    if(contentType != null) {
-      if(contentType.indexOf("application/json") == 0 || contentType.indexOf("+json") > 0) {
-        if(response.responseText && response.responseText !== "")
-          out.obj = JSON.parse(response.responseText);
-        else
-          out.obj = {}
-      }
-    }
-
-    if(response.status >= 200 && response.status < 300)
-      cb.response(out);
-    else if(response.status === 0 || (response.status >= 400 && response.status < 599))
-      cb.error(out);
-    else
-      return cb.response(out);
-  };
-
-  jQuery.support.cors = true;
-  return jQuery.ajax(obj);
-}
-
-/*
- * ShredHttpClient is a light-weight, node or browser HTTP client
- */
-var ShredHttpClient = function(options) {
-  this.options = (options||{});
-  this.isInitialized = false;
-
-  var identity, toString;
-
-  if (typeof window !== 'undefined') {
-    this.Shred = require("./shred");
-    this.content = require("./shred/content");
-  }
-  else
-    this.Shred = require("shred");
-  this.shred = new this.Shred();
-}
-
-ShredHttpClient.prototype.initShred = function () {
-  this.isInitialized = true;
-  this.registerProcessors(this.shred);
-}
-
-ShredHttpClient.prototype.registerProcessors = function(shred) {
-  var identity = function(x) {
-    return x;
-  };
-  var toString = function(x) {
-    return x.toString();
-  };
-
-  if (typeof window !== 'undefined') {
-    this.content.registerProcessor(["application/json; charset=utf-8", "application/json", "json"], {
-      parser: identity,
-      stringify: toString
-    });
-  } else {
-    this.Shred.registerProcessor(["application/json; charset=utf-8", "application/json", "json"], {
-      parser: identity,
-      stringify: toString
-    });
-  }
-}
-
-ShredHttpClient.prototype.execute = function(obj) {
-  if(!this.isInitialized)
-    this.initShred();
-
-  var cb = obj.on, res;
-
-  var transform = function(response) {
-    var out = {
-      headers: response._headers,
-      url: response.request.url,
-      method: response.request.method,
-      status: response.status,
-      data: response.content.data
-    };
-
-    var contentType = (response._headers["content-type"]||response._headers["Content-Type"]||null)
-
-    if(contentType != null) {
-      if(contentType.indexOf("application/json") == 0 || contentType.indexOf("+json") > 0) {
-        if(response.content.data && response.content.data !== "")
-          out.obj = JSON.parse(response.content.data);
-        else
-          out.obj = {}
-      }
-    }
-    return out;
-  };
-
-  res = {
-    error: function(response) {
-      if (obj)
-        return cb.error(transform(response));
-    },
-    redirect: function(response) {
-      if (obj)
-        return cb.redirect(transform(response));
-    },
-    307: function(response) {
-      if (obj)
-        return cb.redirect(transform(response));
-    },
-    response: function(response) {
-      if (obj)
-        return cb.response(transform(response));
-    }
-  };
-  if (obj) {
-    obj.on = res;
-  }
-  return this.shred.request(obj);
-};
-
-/**
- * SwaggerAuthorizations applys the correct authorization to an operation being executed
- */
-var SwaggerAuthorizations = function() {
-  this.authz = {};
-};
-
-SwaggerAuthorizations.prototype.add = function(name, auth) {
-  this.authz[name] = auth;
-  return auth;
-};
-
-SwaggerAuthorizations.prototype.remove = function(name) {
-  return delete this.authz[name];
-};
-
-SwaggerAuthorizations.prototype.apply = function(obj, authorizations) {
-  var status = null;
-  var key;
-
-  // if the "authorizations" key is undefined, or has an empty array, add all keys
-  if(typeof authorizations === 'undefined' || Object.keys(authorizations).length == 0) {
-    for (key in this.authz) {
-      value = this.authz[key];
-      result = value.apply(obj, authorizations);
-      if (result === true)
-        status = true;
-    }
-  }
-  else {
-    for(name in authorizations) {
-      for (key in this.authz) {
-        if(key == name) {
-          value = this.authz[key];
-          result = value.apply(obj, authorizations);
-          if (result === true)
-            status = true;
-        }
-      }      
-    }
-  }
-
-  return status;
-};
-
-/**
- * ApiKeyAuthorization allows a query param or header to be injected
- */
-var ApiKeyAuthorization = function(name, value, type) {
-  this.name = name;
-  this.value = value;
-  this.type = type;
-};
-
-ApiKeyAuthorization.prototype.apply = function(obj, authorizations) {
-  if (this.type === "query") {
-    if (obj.url.indexOf('?') > 0)
-      obj.url = obj.url + "&" + this.name + "=" + this.value;
-    else
-      obj.url = obj.url + "?" + this.name + "=" + this.value;
-    return true;
-  } else if (this.type === "header") {
-    obj.headers[this.name] = this.value;
-    return true;
-  }
-};
-
-var CookieAuthorization = function(cookie) {
-  this.cookie = cookie;
-}
-
-CookieAuthorization.prototype.apply = function(obj, authorizations) {
-  obj.cookieJar = obj.cookieJar || CookieJar();
-  obj.cookieJar.setCookie(this.cookie);
-  return true;
-}
-
-/**
- * Password Authorization is a basic auth implementation
- */
-var PasswordAuthorization = function(name, username, password) {
-  this.name = name;
-  this.username = username;
-  this.password = password;
-  this._btoa = null;
-  if (typeof window !== 'undefined')
-    this._btoa = btoa;
-  else
-    this._btoa = require("btoa");
-};
-
-PasswordAuthorization.prototype.apply = function(obj, authorizations) {
-  var base64encoder = this._btoa;
-  obj.headers["Authorization"] = "Basic " + base64encoder(this.username + ":" + this.password);
-  return true;
-};
-
-var e = (typeof window !== 'undefined' ? window : exports);
-
-var sampleModels = {};
-var cookies = {};
-
-e.SampleModels = sampleModels;
-e.SwaggerHttp = SwaggerHttp;
-e.SwaggerRequest = SwaggerRequest;
-e.authorizations = new SwaggerAuthorizations();
-e.ApiKeyAuthorization = ApiKeyAuthorization;
-e.PasswordAuthorization = PasswordAuthorization;
-e.CookieAuthorization = CookieAuthorization;
-e.JQueryHttpClient = JQueryHttpClient;
-e.ShredHttpClient = ShredHttpClient;
-e.SwaggerOperation = SwaggerOperation;
-e.SwaggerModel = SwaggerModel;
-e.SwaggerModelProperty = SwaggerModelProperty;
-e.SwaggerResource = SwaggerResource;
-e.SwaggerApi = SwaggerApi;
-
diff --git a/docs/client-server/web/files/underscore-min.js b/docs/client-server/web/files/underscore-min.js
deleted file mode 100644
index 5a0cb3b00..000000000
--- a/docs/client-server/web/files/underscore-min.js
+++ /dev/null
@@ -1,32 +0,0 @@
-// Underscore.js 1.3.3
-// (c) 2009-2012 Jeremy Ashkenas, DocumentCloud Inc.
-// Underscore is freely distributable under the MIT license.
-// Portions of Underscore are inspired or borrowed from Prototype,
-// Oliver Steele's Functional, and John Resig's Micro-Templating.
-// For all details and documentation:
-// http://documentcloud.github.com/underscore
-(function(){function r(a,c,d){if(a===c)return 0!==a||1/a==1/c;if(null==a||null==c)return a===c;a._chain&&(a=a._wrapped);c._chain&&(c=c._wrapped);if(a.isEqual&&b.isFunction(a.isEqual))return a.isEqual(c);if(c.isEqual&&b.isFunction(c.isEqual))return c.isEqual(a);var e=l.call(a);if(e!=l.call(c))return!1;switch(e){case "[object String]":return a==""+c;case "[object Number]":return a!=+a?c!=+c:0==a?1/a==1/c:a==+c;case "[object Date]":case "[object Boolean]":return+a==+c;case "[object RegExp]":return a.source==
-c.source&&a.global==c.global&&a.multiline==c.multiline&&a.ignoreCase==c.ignoreCase}if("object"!=typeof a||"object"!=typeof c)return!1;for(var f=d.length;f--;)if(d[f]==a)return!0;d.push(a);var f=0,g=!0;if("[object Array]"==e){if(f=a.length,g=f==c.length)for(;f--&&(g=f in a==f in c&&r(a[f],c[f],d)););}else{if("constructor"in a!="constructor"in c||a.constructor!=c.constructor)return!1;for(var h in a)if(b.has(a,h)&&(f++,!(g=b.has(c,h)&&r(a[h],c[h],d))))break;if(g){for(h in c)if(b.has(c,h)&&!f--)break;
-g=!f}}d.pop();return g}var s=this,I=s._,o={},k=Array.prototype,p=Object.prototype,i=k.slice,J=k.unshift,l=p.toString,K=p.hasOwnProperty,y=k.forEach,z=k.map,A=k.reduce,B=k.reduceRight,C=k.filter,D=k.every,E=k.some,q=k.indexOf,F=k.lastIndexOf,p=Array.isArray,L=Object.keys,t=Function.prototype.bind,b=function(a){return new m(a)};"undefined"!==typeof exports?("undefined"!==typeof module&&module.exports&&(exports=module.exports=b),exports._=b):s._=b;b.VERSION="1.3.3";var j=b.each=b.forEach=function(a,
-c,d){if(a!=null)if(y&&a.forEach===y)a.forEach(c,d);else if(a.length===+a.length)for(var e=0,f=a.length;e2;a==null&&(a=[]);if(A&&
-a.reduce===A){e&&(c=b.bind(c,e));return f?a.reduce(c,d):a.reduce(c)}j(a,function(a,b,i){if(f)d=c.call(e,d,a,b,i);else{d=a;f=true}});if(!f)throw new TypeError("Reduce of empty array with no initial value");return d};b.reduceRight=b.foldr=function(a,c,d,e){var f=arguments.length>2;a==null&&(a=[]);if(B&&a.reduceRight===B){e&&(c=b.bind(c,e));return f?a.reduceRight(c,d):a.reduceRight(c)}var g=b.toArray(a).reverse();e&&!f&&(c=b.bind(c,e));return f?b.reduce(g,c,d,e):b.reduce(g,c)};b.find=b.detect=function(a,
-c,b){var e;G(a,function(a,g,h){if(c.call(b,a,g,h)){e=a;return true}});return e};b.filter=b.select=function(a,c,b){var e=[];if(a==null)return e;if(C&&a.filter===C)return a.filter(c,b);j(a,function(a,g,h){c.call(b,a,g,h)&&(e[e.length]=a)});return e};b.reject=function(a,c,b){var e=[];if(a==null)return e;j(a,function(a,g,h){c.call(b,a,g,h)||(e[e.length]=a)});return e};b.every=b.all=function(a,c,b){var e=true;if(a==null)return e;if(D&&a.every===D)return a.every(c,b);j(a,function(a,g,h){if(!(e=e&&c.call(b,
-a,g,h)))return o});return!!e};var G=b.some=b.any=function(a,c,d){c||(c=b.identity);var e=false;if(a==null)return e;if(E&&a.some===E)return a.some(c,d);j(a,function(a,b,h){if(e||(e=c.call(d,a,b,h)))return o});return!!e};b.include=b.contains=function(a,c){var b=false;if(a==null)return b;if(q&&a.indexOf===q)return a.indexOf(c)!=-1;return b=G(a,function(a){return a===c})};b.invoke=function(a,c){var d=i.call(arguments,2);return b.map(a,function(a){return(b.isFunction(c)?c||a:a[c]).apply(a,d)})};b.pluck=
-function(a,c){return b.map(a,function(a){return a[c]})};b.max=function(a,c,d){if(!c&&b.isArray(a)&&a[0]===+a[0])return Math.max.apply(Math,a);if(!c&&b.isEmpty(a))return-Infinity;var e={computed:-Infinity};j(a,function(a,b,h){b=c?c.call(d,a,b,h):a;b>=e.computed&&(e={value:a,computed:b})});return e.value};b.min=function(a,c,d){if(!c&&b.isArray(a)&&a[0]===+a[0])return Math.min.apply(Math,a);if(!c&&b.isEmpty(a))return Infinity;var e={computed:Infinity};j(a,function(a,b,h){b=c?c.call(d,a,b,h):a;bd?1:0}),"value")};b.groupBy=function(a,c){var d={},e=b.isFunction(c)?c:function(a){return a[c]};
-j(a,function(a,b){var c=e(a,b);(d[c]||(d[c]=[])).push(a)});return d};b.sortedIndex=function(a,c,d){d||(d=b.identity);for(var e=0,f=a.length;e>1;d(a[g])=0})})};b.difference=function(a){var c=b.flatten(i.call(arguments,1),true);return b.filter(a,function(a){return!b.include(c,a)})};b.zip=function(){for(var a=
-i.call(arguments),c=b.max(b.pluck(a,"length")),d=Array(c),e=0;e=0;d--)b=[a[d].apply(this,b)];return b[0]}};b.after=function(a,b){return a<=0?b():function(){if(--a<1)return b.apply(this,arguments)}};b.keys=L||function(a){if(a!==Object(a))throw new TypeError("Invalid object");var c=[],d;for(d in a)b.has(a,d)&&(c[c.length]=d);return c};b.values=function(a){return b.map(a,b.identity)};b.functions=b.methods=function(a){var c=[],d;for(d in a)b.isFunction(a[d])&&
-c.push(d);return c.sort()};b.extend=function(a){j(i.call(arguments,1),function(b){for(var d in b)a[d]=b[d]});return a};b.pick=function(a){var c={};j(b.flatten(i.call(arguments,1)),function(b){b in a&&(c[b]=a[b])});return c};b.defaults=function(a){j(i.call(arguments,1),function(b){for(var d in b)a[d]==null&&(a[d]=b[d])});return a};b.clone=function(a){return!b.isObject(a)?a:b.isArray(a)?a.slice():b.extend({},a)};b.tap=function(a,b){b(a);return a};b.isEqual=function(a,b){return r(a,b,[])};b.isEmpty=
-function(a){if(a==null)return true;if(b.isArray(a)||b.isString(a))return a.length===0;for(var c in a)if(b.has(a,c))return false;return true};b.isElement=function(a){return!!(a&&a.nodeType==1)};b.isArray=p||function(a){return l.call(a)=="[object Array]"};b.isObject=function(a){return a===Object(a)};b.isArguments=function(a){return l.call(a)=="[object Arguments]"};b.isArguments(arguments)||(b.isArguments=function(a){return!(!a||!b.has(a,"callee"))});b.isFunction=function(a){return l.call(a)=="[object Function]"};
-b.isString=function(a){return l.call(a)=="[object String]"};b.isNumber=function(a){return l.call(a)=="[object Number]"};b.isFinite=function(a){return b.isNumber(a)&&isFinite(a)};b.isNaN=function(a){return a!==a};b.isBoolean=function(a){return a===true||a===false||l.call(a)=="[object Boolean]"};b.isDate=function(a){return l.call(a)=="[object Date]"};b.isRegExp=function(a){return l.call(a)=="[object RegExp]"};b.isNull=function(a){return a===null};b.isUndefined=function(a){return a===void 0};b.has=function(a,
-b){return K.call(a,b)};b.noConflict=function(){s._=I;return this};b.identity=function(a){return a};b.times=function(a,b,d){for(var e=0;e/g,">").replace(/"/g,""").replace(/'/g,"'").replace(/\//g,"/")};b.result=function(a,c){if(a==null)return null;var d=a[c];return b.isFunction(d)?d.call(a):d};b.mixin=function(a){j(b.functions(a),function(c){M(c,b[c]=a[c])})};var N=0;b.uniqueId=
-function(a){var b=N++;return a?a+b:b};b.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g};var u=/.^/,n={"\\":"\\","'":"'",r:"\r",n:"\n",t:"\t",u2028:"\u2028",u2029:"\u2029"},v;for(v in n)n[n[v]]=v;var O=/\\|'|\r|\n|\t|\u2028|\u2029/g,P=/\\(\\|'|r|n|t|u2028|u2029)/g,w=function(a){return a.replace(P,function(a,b){return n[b]})};b.template=function(a,c,d){d=b.defaults(d||{},b.templateSettings);a="__p+='"+a.replace(O,function(a){return"\\"+n[a]}).replace(d.escape||
-u,function(a,b){return"'+\n_.escape("+w(b)+")+\n'"}).replace(d.interpolate||u,function(a,b){return"'+\n("+w(b)+")+\n'"}).replace(d.evaluate||u,function(a,b){return"';\n"+w(b)+"\n;__p+='"})+"';\n";d.variable||(a="with(obj||{}){\n"+a+"}\n");var a="var __p='';var print=function(){__p+=Array.prototype.join.call(arguments, '')};\n"+a+"return __p;\n",e=new Function(d.variable||"obj","_",a);if(c)return e(c,b);c=function(a){return e.call(this,a,b)};c.source="function("+(d.variable||"obj")+"){\n"+a+"}";return c};
-b.chain=function(a){return b(a).chain()};var m=function(a){this._wrapped=a};b.prototype=m.prototype;var x=function(a,c){return c?b(a).chain():a},M=function(a,c){m.prototype[a]=function(){var a=i.call(arguments);J.call(a,this._wrapped);return x(c.apply(b,a),this._chain)}};b.mixin(b);j("pop,push,reverse,shift,sort,splice,unshift".split(","),function(a){var b=k[a];m.prototype[a]=function(){var d=this._wrapped;b.apply(d,arguments);var e=d.length;(a=="shift"||a=="splice")&&e===0&&delete d[0];return x(d,
-this._chain)}});j(["concat","join","slice"],function(a){var b=k[a];m.prototype[a]=function(){return x(b.apply(this._wrapped,arguments),this._chain)}});m.prototype.chain=function(){this._chain=true;return this};m.prototype.value=function(){return this._wrapped}}).call(this);
diff --git a/docs/client-server/web/swagger.html b/docs/client-server/web/swagger.html
deleted file mode 100644
index c4ee64270..000000000
--- a/docs/client-server/web/swagger.html
+++ /dev/null
@@ -1,78 +0,0 @@
-
-
-  Matrix Client-Server API Documentation
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-  
-
-  
-  
-
-  
-
-
-
-
-
-
Can't read from server.  It may not have the appropriate access-control-origin settings.

-

-
-
-

From 192fce51d743a99700c0686ed03ba272c9053467 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson 
Date: Tue, 11 Nov 2014 21:18:33 +0200
Subject: [PATCH 218/265] hide crap from gitignore

---
 .gitignore | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.gitignore b/.gitignore
index bd37f92dc..3d14ac8c0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 *.pyc
 .*.swp
 
+.DS_Store
 _trial_temp/
 logs/
 dbs/
@@ -11,6 +12,14 @@ docs/build/
 
 cmdclient_config.json
 homeserver*.db
+homeserver*.log
+homeserver*.pid
+homeserver*.yaml
+
+*.signing.key
+*.tls.crt
+*.tls.dh
+*.tls.key
 
 .coverage
 htmlcov

From a0bc0fdf2155afbb90a758d78c909dd69aca09fd Mon Sep 17 00:00:00 2001
From: Matthew Hodgson 
Date: Tue, 11 Nov 2014 21:52:30 +0200
Subject: [PATCH 219/265] vestigial readme for sphinx

---
 docs/sphinx/README.rst | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 docs/sphinx/README.rst

diff --git a/docs/sphinx/README.rst b/docs/sphinx/README.rst
new file mode 100644
index 000000000..a7ab7c550
--- /dev/null
+++ b/docs/sphinx/README.rst
@@ -0,0 +1 @@
+TODO: how (if at all) is this actually maintained?

From 0ac2dc388eacf90aed7fe553781c3a33ea90fe0e Mon Sep 17 00:00:00 2001
From: Matthew Hodgson 
Date: Wed, 12 Nov 2014 01:04:32 +0200
Subject: [PATCH 220/265] move OLD_specification into matrix-doc/drafts

---
 docs/client-server/OLD_specification.rst | 1283 ----------------------
 1 file changed, 1283 deletions(-)
 delete mode 100644 docs/client-server/OLD_specification.rst

diff --git a/docs/client-server/OLD_specification.rst b/docs/client-server/OLD_specification.rst
deleted file mode 100644
index 425ae57d9..000000000
--- a/docs/client-server/OLD_specification.rst
+++ /dev/null
@@ -1,1283 +0,0 @@
-========================
-Matrix Client-Server API
-========================
-
-
-.. WARNING::
-  This specification is old. Please see matrix-doc/specification instead.
-
-
-
-
-
-
-
-
-
-
-
-The following specification outlines how a client can send and receive data from 
-a home server.
-
-[[TODO(kegan): 4/7/14 Grilling
-- Mechanism for getting historical state changes (e.g. topic updates) - add 
-  query param flag?
-- Generic mechanism for linking first class events (e.g. feedback) with other s
-  first class events (e.g. messages)?
-- Generic mechanism for updating 'stuff about the room' (e.g. favourite coffee) 
-  AND specifying clobbering rules (clobber/add to list/etc)?
-- How to ensure a consistent view for clients paginating through room lists? 
-  They aren't really ordered in any way, and if you're paginating
-  through them, how can you show them a consistent result set? Temporary 'room 
-  list versions' akin to event version? How does that work?
-]]
-
-[[TODO(kegan):
-Outstanding problems / missing spec:
-- Push
-- Typing notifications
-]]
-
-Terminology
------------
-Stream Tokens: 
-An opaque token used to make further streaming requests. When using any 
-pagination streaming API, responses will contain a start and end stream token. 
-When reconnecting to the stream, these tokens can be used to tell the server 
-where the client got up to in the stream.
-
-Event ID:
-Every event that comes down the event stream or that is returned from the REST
-API has an associated event ID (event_id). This ID will be the same between the 
-REST API and the event stream, so any duplicate events can be clobbered 
-correctly without knowing anything else about the event.
-
-Message ID:
-The ID of a message sent by a client in a room. Clients send IMs to each other 
-in rooms. Each IM sent by a client must have a unique message ID which is unique
-for that particular client.
-
-User ID:
-The @username:host style ID of the client. When registering for an account, the 
-client specifies their username. The user_id is this username along with the 
-home server's unique hostname. When federating between home servers, the user_id
-is used to uniquely identify users across multiple home servers.
-
-Room ID:
-The room_id@host style ID for the room. When rooms are created, the client either
-specifies or is allocated a room ID. This room ID must be used to send messages 
-in that room. Like with clients, there may be multiple rooms with the same ID 
-across multiple home servers. The room_id is used to uniquely identify a room 
-when federating.
-
-Global message ID:
-The globally unique ID for a message. This ID is formed from the msg_id, the 
-client's user_id and the room_id. This uniquely identifies any 
-message. It is represented with '-' as the delimeter between IDs. The 
-global_msg_id is of the form: room_id-user_id-msg_id
-
-
-REST API and the Event Stream
------------------------------
-Clients send data to the server via a RESTful API. They can receive data via 
-this API or from an event stream. An event stream is a special path which 
-streams all events the client may be interested in. This makes it easy to 
-immediately receive updates from the REST API. All data is represented as JSON.
-
-Pagination streaming API
-========================
-Clients are often interested in very large datasets. The data itself could
-be 1000s of messages in a given room, 1000s of rooms in a public room list, or
-1000s of events (presence, typing, messages, etc) in the system. It is not
-practical to send vast quantities of data to the client every time they
-request a list of public rooms for example. There needs to be a way to show a
-subset of this data, and apply various filters to it. This is what the pagination
-streaming API is. This API defines standard request/response parameters which 
-can be used when navigating this stream of data.
-
-Pagination Request Query Parameters
------------------------------------
-Clients may wish to paginate results from the event stream, or other sources of 
-information where the amount of information may be a problem,
-e.g. in a room with 10,000s messages. The pagination query parameters provide a 
-way to navigate a 'window' around a large set of data. These
-parameters are only valid for GET requests.
-       
-        S e r v e r - s i d e   d a t a
- |-------------------------------------------------|
-START      ^               ^                      END
-           |_______________|
-                   |
-            Client-extraction
-
-'START' and 'END' are magic token values which specify the start and end of the 
-dataset respectively.
-
-Query parameters:
-  from : $streamtoken - The opaque token to start streaming from.
-  to : $streamtoken - The opaque token to end streaming at. Typically,
-       clients will not know the item of data to end at, so this will usually be 
-       START or END.
-  limit : integer - An integer representing the maximum number of items to 
-          return.
-
-For example, the event stream has events E1 -> E15. The client wants the last 5 
-events and doesn't know any previous events:
-
-S                                                    E
-|-E1-E2-E3-E4-E5-E6-E7-E8-E9-E10-E11-E12-E13-E14-E15-|
-|                               |                    |
-|                          _____|                    |
-|__________________       |       ___________________|
-                   |      |      |
- GET /events?to=START&limit=5&from=END
- Returns:
-   E15,E14,E13,E12,E11
-
-
-Another example: a public room list has rooms R1 -> R17. The client is showing 5 
-rooms at a time on screen, and is on page 2. They want to
-now show page 3 (rooms R11 -> 15):
-
-S                                                           E
-|  0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16 | stream token
-|-R1-R2-R3-R4-R5-R6-R7-R8-R9-R10-R11-R12-R13-R14-R15-R16-R17| room
-                  |____________| |________________|
-                        |                |
-                    Currently            |
-                    viewing              |
-                                         |
-                         GET /rooms/list?from=9&to=END&limit=5
-                         Returns: R11,R12,R13,R14,R15
-                         
-Note that tokens are treated in an *exclusive*, not inclusive, manner. The end 
-token from the intial request was '9' which corresponded to R10. When the 2nd
-request was made, R10 did not appear again, even though from=9 was specified. If
-you know the token, you already have the data.
-
-Pagination Response
--------------------
-Responses to pagination requests MUST follow the format:
-{
-  "chunk": [ ... , Responses , ... ],
-  "start" : $streamtoken,
-  "end" : $streamtoken
-}
-Where $streamtoken is an opaque token which can be used in another query to
-get the next set of results. The "start" and "end" keys can only be omitted if
-the complete dataset is provided in "chunk".
-
-If the client wants earlier results, they should use from=$start_streamtoken,
-to=START. Likewise, if the client wants later results, they should use
-from=$end_streamtoken, to=END.
-
-Unless specified, the default pagination parameters are from=START, to=END, 
-without a limit set. This allows you to hit an API like
-/events without any query parameters to get everything.
-
-The Event Stream
-----------------
-The event stream returns events using the pagination streaming API. When the 
-client disconnects for a while and wants to reconnect to the event stream, they 
-should specify from=$end_streamtoken. This lets the server know where in the 
-event stream the client is. These tokens are completely opaque, and the client 
-cannot infer anything from them.
-
-  GET /events?from=$LAST_STREAM_TOKEN
-  REST Path: /events
-  Returns (success): A JSON array of Event Data.
-  Returns (failure): An Error Response
-
-LAST_STREAM_TOKEN is the last stream token obtained from the event stream. If the 
-client is connecting for the first time and does not know any stream tokens,
-they can use "START" to request all events from the start. For more information 
-on this, see "Pagination Request Query Parameters".
-
-The event stream supports shortpoll and longpoll with the "timeout" query
-parameter. This parameter specifies the number of milliseconds the server should
-hold onto the connection waiting for incoming events. If no events occur in this
-period, the connection will be closed and an empty chunk will be returned. To
-use shortpoll, specify "timeout=0".
-
-Event Data
-----------
-This is a JSON object which looks like:
-{
-  "event_id" : $EVENT_ID,
-  "type" : $EVENT_TYPE,
-  $URL_ARGS,
-  "content" : {
-    $EVENT_CONTENT
-  }
-}
-
-EVENT_ID
-  An ID identifying this event. This is so duplicate events can be suppressed on
-  the client.
-
-EVENT_TYPE
-  The namespaced event type (m.*)
-
-URL_ARGS
-  Path specific data from the REST API.
-
-EVENT_CONTENT
-  The event content, matching the REST content PUT previously.
-
-Events are differentiated via the event type "type" key. This is the type of 
-event being received. This can be expanded upon by using different namespaces. 
-Every event MUST have a 'type' key.
-
-Most events will have a corresponding REST URL. This URL will generally have 
-data in it to represent the resource being modified,
-e.g. /rooms/$room_id. The event data will contain extra top-level keys to expose 
-this information to clients listening on an event
-stream. The event content maps directly to the contents submitted via the REST 
-API.
-
-For example:
-  Event Type: m.example.room.members
-  REST Path: /examples/room/$room_id/members/$user_id
-  REST Content: { "membership" : "invited" }
-  
-is represented in the event stream as:
-
-{
-  "event_id" : "e_some_event_id",
-  "type" : "m.example.room.members",
-  "room_id" : $room_id,
-  "user_id" : $user_id,
-  "content" : {
-    "membership" : "invited"
-  }
-}
-
-As convention, the URL variable "$varname" will map directly onto the name 
-of the JSON key "varname".
-
-Error Responses
----------------
-If the client sends an invalid request, the server MAY respond with an error 
-response. This is of the form:
-{
-  "error" : "string",
-  "errcode" : "string"
-}
-The 'error' string will be a human-readable error message, usually a sentence
-explaining what went wrong. 
-
-The 'errcode' string will be a unique string which can be used to handle an 
-error message e.g. "M_FORBIDDEN". These error codes should have their namespace 
-first in ALL CAPS, followed by a single _. For example, if there was a custom
-namespace com.mydomain.here, and a "FORBIDDEN" code, the error code should look
-like "COM.MYDOMAIN.HERE_FORBIDDEN". There may be additional keys depending on 
-the error, but the keys 'error' and 'errcode' will always be present. 
-
-Some standard error codes are below:
-
-M_FORBIDDEN:
-Forbidden access, e.g. joining a room without permission, failed login.
-
-M_UNKNOWN_TOKEN:
-The access token specified was not recognised.
-
-M_BAD_JSON:
-Request contained valid JSON, but it was malformed in some way, e.g. missing
-required keys, invalid values for keys.
-
-M_NOT_JSON:
-Request did not contain valid JSON.
-
-M_NOT_FOUND:
-No resource was found for this request.
-
-Some requests have unique error codes:
-
-M_USER_IN_USE:
-Encountered when trying to register a user ID which has been taken.
-
-M_ROOM_IN_USE:
-Encountered when trying to create a room which has been taken.
-
-M_BAD_PAGINATION:
-Encountered when specifying bad pagination values to a Pagination Streaming API.
-
-
-========
-REST API
-========
-
-All content must be application/json. Some keys are required, while others are 
-optional. Unless otherwise specified,
-all HTTP PUT/POST/DELETEs will return a 200 OK with an empty response body on 
-success, and a 4xx/5xx with an optional Error Response on failure. When sending 
-data, if there are no keys to send, an empty JSON object should be sent.
-
-All POST/PUT/GET/DELETE requests MUST have an 'access_token' query parameter to 
-allow the server to authenticate the client. All
-POST requests MUST be submitted as application/json. 
-
-All paths MUST be namespaced by the version of the API being used. This should
-be:
-
-/_matrix/client/api/v1
-
-All REST paths in this section MUST be prefixed with this. E.g.
-  REST Path: /rooms/$room_id
-  Absolute Path: /_matrix/client/api/v1/rooms/$room_id
-
-Registration
-============
-Clients must register with the server in order to use the service. After 
-registering, the client will be given an
-access token which must be used in ALL requests as a query parameter 
-'access_token'.
-
-Registering for an account
---------------------------
-  POST /register
-  With: A JSON object containing the key "user_id" which contains the desired 
-        user_id, or an empty JSON object to have the server allocate a user_id 
-        automatically.
-  Returns (success): 200 OK with a JSON object:
-                     {
-                       "user_id" : "string [user_id]",
-                       "access_token" : "string"
-                     }
-  Returns (failure): An Error Response. M_USER_IN_USE if the user ID is taken.
-                     
-
-Unregistering an account
-------------------------
-  POST /unregister
-  With query parameters: access_token=$ACCESS_TOKEN
-  Returns (success): 200 OK
-  Returns (failure): An Error Response.
-  
-  
-Logging in to an existing account
-=================================
-If the client has already registered, they need to be able to login to their
-account. The home server may provide many different ways of logging in, such
-as user/password auth, login via a social network (OAuth), login by confirming 
-a token sent to their email address, etc. This section does NOT define how home 
-servers should authorise their users who want to login to their existing 
-accounts. This section defines the standard interface which implementations 
-should follow so that ANY client can login to ANY home server.
-
-The login process breaks down into the following:
-  1: Get login process info.
-  2: Submit the login stage credentials.
-  3: Get access token or be told the next stage in the login process and repeat 
-     step 2.
-     
-Getting login process info:
-  GET /login
-  Returns (success): 200 OK with LoginInfo.
-  Returns (failure): An Error Response.
-  
-Submitting the login stage credentials:
-  POST /login
-  With: LoginSubmission
-  Returns (success): 200 OK with LoginResult
-  Returns (failure): An Error Response
-  
-Where LoginInfo is a JSON object which MUST have a "type" key which denotes the 
-login type. If there are multiple login stages, this object MUST also contain a 
-"stages" key, which has a JSON array of login types denoting all the steps in 
-order to login, including the first stage which is in "type". This allows the 
-client to make an informed decision as to whether or not they can natively
-handle the entire login process, or whether they should fallback (see below).
-
-Where LoginSubmission is a JSON object which MUST have a "type" key.
-
-Where LoginResult is a JSON object which MUST have either a "next" key OR an
-"access_token" key, depending if the login process is over or not. This object
-MUST have a "session" key if multiple POSTs need to be sent to /login.
-
-Fallback
---------
-If the client does NOT know how to handle the given type, they should:
-  GET /login/fallback
-This MUST return an HTML page which can perform the entire login process.
-
-Password-based
---------------
-Type: "m.login.password"
-LoginSubmission:
-{
-  "type": "m.login.password",
-  "user": ,
-  "password": 
-}
-
-Example:
-Assume you are @bob:matrix.org and you wish to login on another mobile device.
-First, you GET /login which returns:
-{
-  "type": "m.login.password"
-}
-Your client knows how to handle this, so your client prompts the user to enter
-their username and password. This is then submitted:
-{
-  "type": "m.login.password",
-  "user": "@bob:matrix.org",
-  "password": "monkey"
-}
-The server checks this, finds it is valid, and returns:
-{
-  "access_token": "abcdef0123456789"
-}
-The server may optionally return "user_id" to confirm or change the user's ID.
-This is particularly useful if the home server wishes to support localpart entry
-of usernames (e.g. "bob" rather than "@bob:matrix.org").
-
-OAuth2-based
-------------
-Type: "m.login.oauth2"
-This is a multi-stage login.
-
-LoginSubmission:
-{
-  "type": "m.login.oauth2",
-  "user": 
-}
-Returns:
-{
-  "uri": 
-}
-
-The home server acts as a 'confidential' Client for the purposes of OAuth2.
-
-If the uri is a "sevice selection uri", it is a simple page which prompts the 
-user to choose which service to authorize with. On selection of a service, they
-link through to Authorization Request URIs. If there is only 1 service which the
-home server accepts when logging in, this indirection can be skipped and the
-"uri" key can be the Authorization Request URI. 
-
-The client visits the Authorization Request URI, which then shows the OAuth2 
-Allow/Deny prompt. Hitting 'Allow' returns the redirect URI with the auth code. 
-Home servers can choose any path for the redirect URI. The client should visit 
-the redirect URI, which will then finish the OAuth2 login process, granting the 
-home server an access token for the chosen service. When the home server gets 
-this access token, it knows that the cilent has authed with the 3rd party, and 
-so can return a LoginResult.
-
-The OAuth redirect URI (with auth code) MUST return a LoginResult.
-    
-Example:
-Assume you are @bob:matrix.org and you wish to login on another mobile device.
-First, you GET /login which returns:
-{
-  "type": "m.login.oauth2"
-}
-Your client knows how to handle this, so your client prompts the user to enter
-their username. This is then submitted:
-{
-  "type": "m.login.oauth2",
-  "user": "@bob:matrix.org"
-}
-The server only accepts auth from Google, so returns the Authorization Request
-URI for Google:
-{
-  "uri": "https://accounts.google.com/o/oauth2/auth?response_type=code&
-  client_id=CLIENT_ID&redirect_uri=REDIRECT_URI&scope=photos"
-}
-The client then visits this URI and authorizes the home server. The client then
-visits the REDIRECT_URI with the auth code= query parameter which returns:
-{
-  "access_token": "0123456789abcdef"
-}
-
-Email-based (code)
-------------------
-Type: "m.login.email.code"
-This is a multi-stage login.
-
-First LoginSubmission:
-{
-  "type": "m.login.email.code",
-  "user": 
-  "email": 
-}
-Returns:
-{
-  "session": 
-}
-
-The email contains a code which must be sent in the next LoginSubmission:
-{
-  "type": "m.login.email.code",
-  "session": ,
-  "code": 
-}
-Returns:
-{
-  "access_token": 
-}
-
-Example:
-Assume you are @bob:matrix.org and you wish to login on another mobile device.
-First, you GET /login which returns:
-{
-  "type": "m.login.email.code"
-}
-Your client knows how to handle this, so your client prompts the user to enter
-their email address. This is then submitted:
-{
-  "type": "m.login.email.code",
-  "user": "@bob:matrix.org",
-  "email": "bob@mydomain.com"
-}
-The server confirms that bob@mydomain.com is linked to @bob:matrix.org, then 
-sends an email to this address and returns:
-{
-  "session": "ewuigf7462"
-}
-The client's screen changes to a code submission page. The email arrives and it 
-says something to the effect of "please enter 2348623 into the app". This is
-the submitted along with the session:
-{
-  "type": "m.login.email.code",
-  "session": "ewuigf7462",
-  "code": "2348623"
-}
-The server accepts this and returns:
-{
-  "access_token": "abcdef0123456789"
-}
-
-Email-based (url)
------------------
-Type: "m.login.email.url"
-This is a multi-stage login.
-
-First LoginSubmission:
-{
-  "type": "m.login.email.url",
-  "user": 
-  "email": 
-}
-Returns:
-{
-  "session": 
-}
-
-The email contains a URL which must be clicked. After it has been clicked, the
-client should perform a request:
-{
-  "type": "m.login.email.code",
-  "session": 
-}
-Returns:
-{
-  "access_token": 
-}
-
-Example:
-Assume you are @bob:matrix.org and you wish to login on another mobile device.
-First, you GET /login which returns:
-{
-  "type": "m.login.email.url"
-}
-Your client knows how to handle this, so your client prompts the user to enter
-their email address. This is then submitted:
-{
-  "type": "m.login.email.url",
-  "user": "@bob:matrix.org",
-  "email": "bob@mydomain.com"
-}
-The server confirms that bob@mydomain.com is linked to @bob:matrix.org, then 
-sends an email to this address and returns:
-{
-  "session": "ewuigf7462"
-}
-The client then starts polling the server with the following:
-{
-  "type": "m.login.email.url",
-  "session": "ewuigf7462"
-}
-(Alternatively, the server could send the device a push notification when the
-email has been validated). The email arrives and it contains a URL to click on.
-The user clicks on the which completes the login process with the server. The
-next time the client polls, it returns:
-{
-  "access_token": "abcdef0123456789"
-}
-
-N-Factor auth
--------------
-Multiple login stages can be combined with the "next" key in the LoginResult.
-
-Example:
-A server demands an email.code then password auth before logging in. First, the
-client performs a GET /login which returns:
-{
-  "type": "m.login.email.code",
-  "stages": ["m.login.email.code", "m.login.password"]
-}
-The client performs the email login (See "Email-based (code)"), but instead of
-returning an access_token, it returns:
-{
-  "next": "m.login.password"
-}
-The client then presents a user/password screen and the login continues until
-this is complete (See "Password-based"), which then returns the "access_token".
-
-Rooms
-=====
-A room is a conceptual place where users can send and receive messages. Rooms 
-can be created, joined and left. Messages are sent
-to a room, and all participants in that room will receive the message. Rooms are 
-uniquely identified via the room_id.
-
-Creating a room (with a room ID)
---------------------------------
-  Event Type: m.room.create [TODO(kegan): Do we generate events for this?]
-  REST Path: /rooms/$room_id
-  Valid methods: PUT
-  Required keys: None.
-  Optional keys:
-    visibility : [public|private] - Set whether this room shows up in the public 
-    room list.
-  Returns:
-    On Failure: MAY return a suggested alternative room ID if this room ID is 
-    taken.
-    {
-      suggested_room_id : $new_room_id
-      error : "Room already in use."
-      errcode : "M_ROOM_IN_USE"
-    }
-    
-
-Creating a room (without a room ID)
------------------------------------
-  Event Type: m.room.create [TODO(kegan): Do we generate events for this?]
-  REST Path: /rooms
-  Valid methods: POST
-  Required keys: None.
-  Optional keys:
-    visibility : [public|private] - Set whether this room shows up in the public 
-    room list.
-  Returns:
-    On Success: The allocated room ID. Additional information about the room
-    such as the visibility MAY be included as extra keys in this response.
-    {
-      room_id : $room_id
-    }
-
-Setting the topic for a room
-----------------------------
-  Event Type: m.room.topic
-  REST Path: /rooms/$room_id/topic
-  Valid methods: GET/PUT
-  Required keys: 
-    topic : $topicname - Set the topic to $topicname in room $room_id.
-
-
-See a list of public rooms
---------------------------
-  REST Path: /public/rooms?pagination_query_parameters
-  Valid methods: GET
-  This API can use pagination query parameters.
-  Returns:
-    {
-      "chunk" : JSON array of RoomInfo JSON objects - Required.
-      "start" : "string (start token)" - See Pagination Response.
-      "end" : "string (end token)" - See Pagination Response.
-      "total" : integer - Optional. The total number of rooms.
-    }
-
-RoomInfo: Information about a single room.
-  Servers MUST send the key: room_id
-  Servers MAY send the keys: topic, num_members
-  {
-    "room_id" : "string",
-    "topic" : "string",
-    "num_members" : integer
-  }
-
-Room Members
-============
-
-Invite/Joining/Leaving a room
------------------------------
-  Event Type: m.room.member
-  REST Path: /rooms/$room_id/members/$user_id/state
-  Valid methods: PUT/GET/DELETE
-  Required keys:
-    membership : [join|invite] - The membership state of $user_id in room 
-                                 $room_id.
-  Optional keys:
-    displayname,
-    avatar_url : String fields from the member user's profile
-    state,
-    status_msg,
-    mtime_age : Presence information
-
-  These optional keys provide extra information that the client is likely to
-  be interested in so it doesn't have to perform an additional profile or
-  presence information fetch.
-
-Where:
-  join - Indicate you ($user_id) are joining the room $room_id.
-  invite - Indicate that $user_id has been invited to room $room_id.
-
-User $user_id can leave room $room_id by DELETEing this path.
-
-Checking the user list of a room
---------------------------------
-  REST Path: /rooms/$room_id/members/list
-  This API can use pagination query parameters.
-  Valid methods: GET
-  Returns:
-    A pagination response with chunk data as m.room.member events.
-
-Messages
-========
-Users send messages to other users in rooms. These messages may be text, images, 
-video, etc. Clients may also want to acknowledge messages by sending feedback, 
-in the form of delivery/read receipts.
-
-Server-attached keys
---------------------
-The server MAY attach additional keys to messages and feedback. If a client 
-submits keys with the same name, they will be clobbered by
-the server.
-
-Required keys:
-from : "string [user_id]"
-  The user_id of the user who sent the message/feedback.
-
-Optional keys:
-hsob_ts : integer
-  A timestamp (ms resolution) representing when the message/feedback got to the 
-  sender's home server ("home server outbound timestamp").
-
-hsib_ts : integer
-  A timestamp (ms resolution) representing when the 
-  message/feedback got to the receiver's home server ("home server inbound 
-  timestamp"). This may be the same as hsob_ts if the sender/receiver are on the 
-  same home server.
-
-Sending messages
-----------------
-  Event Type: m.room.message
-  REST Path: /rooms/$room_id/messages/$from/$msg_id
-  Valid methods: GET/PUT
-  URL parameters:
-    $from : user_id - The sender's user_id. This value will be clobbered by the 
-    server before sending.
-  Required keys: 
-    msgtype: [m.text|m.emote|m.image|m.audio|m.video|m.location|m.file] - 
-             The type of message. Not to be confused with the Event 'type'.
-  Optional keys:
-    sender_ts : integer - A timestamp (ms resolution) representing the 
-                wall-clock time when the message was sent from the client.
-  Reserved keys:
-    body : "string" - The human readable string for compatibility with clients 
-           which cannot process a given msgtype. This key is optional, but
-           if it is included, it MUST be human readable text 
-           describing the message. See individual msgtypes for more 
-           info on what this means in practice.
-
-Each msgtype may have required fields of their own.
-
-msgtype: m.text
-----------------
-Required keys:
-  body : "string" - The body of the message.
-Optional keys:
-  None.
-
-msgtype: m.emote
------------------
-Required keys:
-  body : "string" - *tries to come up with a witty explanation*.
-Optional keys:
-  None.
-
-msgtype: m.image
------------------
-Required keys:
-  url : "string" - The URL to the image.
-Optional keys:
-  body : "string" - info : JSON object (ImageInfo) - The image info for image 
-         referred to in 'url'.
-  thumbnail_url : "string" - The URL to the thumbnail.
-  thumbnail_info : JSON object (ImageInfo) - The image info for the image 
-                   referred to in 'thumbnail_url'.
-
-ImageInfo: Information about an image.
-{
-  "size" : integer (size of image in bytes),
-  "w" : integer (width of image in pixels),
-  "h" : integer (height of image in pixels),
-  "mimetype" : "string (e.g. image/jpeg)"
-}
-
-Interpretation of 'body' key: The alt text of the image, or some kind of content 
-description for accessibility e.g. "image attachment".
-
-msgtype: m.audio
------------------
-Required keys:
-  url : "string" - The URL to the audio.
-Optional keys:
-  info : JSON object (AudioInfo) - The audio info for the audio referred to in 
-         'url'.
-
-AudioInfo: Information about a piece of audio. 
-{
-  "mimetype" : "string (e.g. audio/aac)",
-  "size" : integer (size of audio in bytes),
-  "duration" : integer (duration of audio in milliseconds)
-}
-
-Interpretation of 'body' key: A description of the audio e.g. "Bee Gees - 
-Stayin' Alive", or some kind of content description for accessibility e.g. 
-"audio attachment".
-
-msgtype: m.video
------------------
-Required keys:
-  url : "string" - The URL to the video.
-Optional keys:
-  info : JSON object (VideoInfo) - The video info for the video referred to in 
-         'url'.
-
-VideoInfo: Information about a video.
-{
-  "mimetype" : "string (e.g. video/mp4)",
-  "size" : integer (size of video in bytes),
-  "duration" : integer (duration of video in milliseconds),
-  "w" : integer (width of video in pixels),
-  "h" : integer (height of video in pixels),
-  "thumbnail_url" : "string (URL to image)",
-  "thumbanil_info" : JSON object (ImageInfo)
-}
-
-Interpretation of 'body' key: A description of the video e.g. "Gangnam style", 
-or some kind of content description for accessibility e.g. "video attachment".
-
-msgtype: m.location
---------------------
-Required keys:
-  geo_uri : "string" - The geo URI representing the location.
-Optional keys:
-  thumbnail_url : "string" - The URL to a thumnail of the location being 
-                  represented.
-  thumbnail_info : JSON object (ImageInfo) - The image info for the image 
-                   referred to in 'thumbnail_url'.
-
-Interpretation of 'body' key: A description of the location e.g. "Big Ben, 
-London, UK", or some kind of content description for accessibility e.g. 
-"location attachment".
-
-
-Sending feedback
-----------------
-When you receive a message, you may want to send delivery receipt to let the 
-sender know that the message arrived. You may also want to send a read receipt 
-when the user has read the message. These receipts are collectively known as 
-'feedback'.
-
-  Event Type: m.room.message.feedback
-  REST Path: /rooms/$room_id/messages/$msgfrom/$msg_id/feedback/$from/$feedback
-  Valid methods: GET/PUT
-  URL parameters:
-    $msgfrom - The sender of the message's user_id.
-    $from : user_id - The sender of the feedback's user_id. This value will be 
-    clobbered by the server before sending.
-    $feedback : [d|r] - Specify if this is a [d]elivery or [r]ead receipt.
-  Required keys:
-    None.
-  Optional keys:
-    sender_ts : integer - A timestamp (ms resolution) representing the 
-    wall-clock time when the receipt was sent from the client.
-
-Receiving messages (bulk/pagination)
-------------------------------------
-  Event Type: m.room.message
-  REST Path: /rooms/$room_id/messages/list
-  Valid methods: GET
-  Query Parameters:
-    feedback : [true|false] - Specify if feedback should be bundled with each 
-    message.
-  This API can use pagination query parameters.
-  Returns:
-    A JSON array of Event Data in "chunk" (see Pagination Response). If the 
-    "feedback" parameter was set, the Event Data will also contain a "feedback" 
-    key which contains a JSON array of feedback, with each element as Event Data 
-    with compressed feedback for this message.
-
-Event Data with compressed feedback is a special type of feedback with 
-contextual keys removed. It is designed to limit the amount of redundant data 
-being sent for feedback. This removes the type, event_id, room ID, 
-message sender ID and message ID keys.
-
-     ORIGINAL (via event streaming)
-{
-  "event_id":"e1247632487",
-  "type":"m.room.message.feedback",
-  "from":"string [user_id]",
-  "feedback":"string [d|r]",
-  "room_id":"$room_id",
-  "msg_id":"$msg_id",
-  "msgfrom":"$msgfromid",
-  "content":{
-    "sender_ts":139880943
-  }
-}
-
-     COMPRESSED (via /messages/list)
-{
-  "from":"string [user_id]",
-  "feedback":"string [d|r]",
-  "content":{
-    "sender_ts":139880943
-  }
-}
-
-When you join a room $room_id, you may want the last 10 messages with feedback. 
-This is represented as:
-  GET 
-  /rooms/$room_id/messages/list?from=END&to=START&limit=10&feedback=true
-
-You may want to get 10 messages even earlier than that without feedback. If the 
-start stream token from the previous request was stok_019173, this request would 
-be:
-  GET 
-  /rooms/$room_id/messages/list?from=stok_019173&to=START&limit=10&
-                               feedback=false
-  
-NOTE: Care must be taken when using this API in conjunction with event 
-      streaming. It is possible that this will return a message which will
-      then come down the event stream, resulting in a duplicate message. Clients 
-      should clobber based on the global message ID, or event ID.
-
-
-Get current state for all rooms (aka IM Initial Sync API)
--------------------------------
-  REST Path: /im/sync
-  Valid methods: GET
-  This API can use pagination query parameters. Pagination is applied on a per
-  *room* basis. E.g. limit=1 means "get 1 message for each room" and not "get 1
-  room's messages". If there is no limit, all messages for all rooms will be
-  returned.
-  If you want 1 room's messages, see "Receiving messages (bulk/pagination)".
-  Additional query parameters: 
-    feedback: [true] - Bundles feedback with messages.
-  Returns:
-    An array of RoomStateInfo.
-
-RoomStateInfo: A snapshot of information about a single room.
-  {
-    "room_id" : "string",
-    "membership" : "string [join|invite]",
-    "messages" : {
-      "start": "string",
-      "end": "string",
-      "chunk":
-      m.room.message pagination stream events (with feedback if specified),
-      this is the same as "Receiving messages (bulk/pagination)".
-    }
-  }
-The "membership" key is the calling user's membership state in the given 
-"room_id". The "messages" key may be omitted if the "membership" value is 
-"invite". Additional keys may be added to the top-level object, such as:
-  "topic" : "string" - The topic for the room in question.
-  "room_image_url" : "string" - The URL of the room image if specified.
-  "num_members" : integer - The number of members in the room.
-
-
-Profiles
-========
-
-Getting/Setting your own displayname
-------------------------------------
-  REST Path: /profile/$user_id/displayname
-  Valid methods: GET/PUT
-  Required keys:
-    displayname : The displayname text
-
-Getting/Setting your own avatar image URL
------------------------------------------
-The homeserver does not currently store the avatar image itself, but offers
-storage for the user to specify a web URL that points at the required image,
-leaving it up to clients to fetch it themselves.
-  REST Path: /profile/$user_id/avatar_url
-  Valid methods: GET/PUT
-  Required keys:
-    avatar_url : The URL path to the required image
-
-Getting other user's profile information
-----------------------------------------
-Either of the above REST methods may be used to fetch other user's profile
-information by the client, either on other local users on the same homeserver or
-for users from other servers entirely.
-
-
-Presence
-========
-
-In the following messages, the presence state is a presence string as described in
-the main specification document.
-
-Getting/Setting your own presence state
----------------------------------------
-  REST Path: /presence/$user_id/status
-  Valid methods: GET/PUT
-  Required keys:
-    presence :  - The user's new presence state
-  Optional keys:
-    status_msg : text string provided by the user to explain their status
-
-Fetching your presence list
----------------------------
-  REST Path: /presence_list/$user_id
-  Valid methods: GET/(post)
-  Returns:
-    An array of presence list entries. Each entry is an object with the
-    following keys:
-      {
-        "user_id" : string giving the observed user's ID
-        "presence" : int giving their status
-        "status_msg" : optional text string
-        "displayname" : optional text string from the user's profile
-        "avatar_url" : optional text string from the user's profile
-      }
-
-Maintaining your presence list
-------------------------------
-  REST Path: /presence_list/$user_id
-  Valid methods: POST/(get)
-  With: A JSON object optionally containing either of the following keys:
-    "invite" : a list of strings giving user IDs to invite for presence
-      subscription
-    "drop" : a list of strings giving user IDs to remove from your presence
-      list
-
-Receiving presence update events
---------------------------------
-  Event Type: m.presence
-  Keys of the event's content are the same as those returned by the presence
-    list.
-
-Examples
-========
-
-The following example is the story of "bob", who signs up at "sy.org" and joins 
-the public room "room_beta@sy.org". They get the 2 most recent
-messages (with feedback) in that room and then send a message in that room. 
-
-For context, here is the complete chat log for room_beta@sy.org:
-
-Room: "Hello world" (room_beta@sy.org)
-Members: (2) alice@randomhost.org, friend_of_alice@randomhost.org
-Messages:
-  alice@randomhost.org : hi friend!                     
-  [friend_of_alice@randomhost.org DELIVERED]
-  alice@randomhost.org : you're my only friend          
-  [friend_of_alice@randomhost.org DELIVERED]
-  alice@randomhost.org : afk                            
-  [friend_of_alice@randomhost.org DELIVERED]
-  [ bob@sy.org joins ]
-  bob@sy.org : Hi everyone
-  [ alice@randomhost.org changes the topic to "FRIENDS ONLY" ]
-  alice@randomhost.org : Hello!!!!
-  alice@randomhost.org : Let's go to another room
-  alice@randomhost.org : You're not my friend
-  [ alice@randomhost.org invites bob@sy.org to the room 
-  commoners@randomhost.org]
-
-
-REGISTER FOR AN ACCOUNT
-POST: /register
-Content: {}
-Returns: { "user_id" : "bob@sy.org" , "access_token" : "abcdef0123456789" }
-
-GET PUBLIC ROOM LIST
-GET: /rooms/list?access_token=abcdef0123456789
-Returns: 
-{ 
-  "total":3,
-  "chunk":
-  [
-    { "room_id":"room_alpha@sy.org", "topic":"I am a fish" },
-    { "room_id":"room_beta@sy.org", "topic":"Hello world" },
-    { "room_id":"room_xyz@sy.org", "topic":"Goodbye cruel world" }
-  ]
-}
-
-JOIN ROOM room_beta@sy.org
-PUT 
-/rooms/room_beta%40sy.org/members/bob%40sy.org/state?
-                                    access_token=abcdef0123456789
-Content: { "membership" : "join" }
-Returns: 200 OK
-
-GET LATEST 2 MESSAGES WITH FEEDBACK
-GET 
-/rooms/room_beta%40sy.org/messages/list?from=END&to=START&limit=2&
-                                    feedback=true&access_token=abcdef0123456789
-Returns:
-{
-  "chunk":
-    [
-      { 
-        "event_id":"01948374", 
-        "type":"m.room.message",
-        "room_id":"room_beta@sy.org",
-        "msg_id":"avefifu",
-        "from":"alice@randomhost.org",
-        "hs_ts":139985736,
-        "content":{
-          "msgtype":"m.text",
-          "body":"afk"
-        }
-        "feedback": [
-          {
-            "from":"friend_of_alice@randomhost.org",
-            "feedback":"d",
-            "hs_ts":139985850,
-            "content":{
-              "sender_ts":139985843
-            }
-          }
-        ]
-      },
-      { 
-        "event_id":"028dfe8373", 
-        "type":"m.room.message",
-        "room_id":"room_beta@sy.org",
-        "msg_id":"afhgfff",
-        "from":"alice@randomhost.org",
-        "hs_ts":139970006,
-        "content":{
-          "msgtype":"m.text",
-          "body":"you're my only friend"
-        }
-        "feedback": [
-          {
-            "from":"friend_of_alice@randomhost.org",
-            "feedback":"d",
-            "hs_ts":139970144,
-            "content":{
-              "sender_ts":139970122
-            }
-          }
-        ]
-      },
-    ],
-  "start": "stok_04823947",
-  "end": "etok_1426425"
-}
-
-SEND MESSAGE IN ROOM
-PUT 
-/rooms/room_beta%40sy.org/messages/bob%40sy.org/m0001?
-                            access_token=abcdef0123456789
-Content: { "msgtype" : "text" , "body" : "Hi everyone" }
-Returns: 200 OK
-
-
-Checking the event stream for this user:
-GET: /events?from=START&access_token=abcdef0123456789
-Returns:
-{
-  "chunk": 
-    [
-      { 
-        "event_id":"e10f3d2b", 
-        "type":"m.room.member",
-        "room_id":"room_beta@sy.org",
-        "user_id":"bob@sy.org",
-        "content":{
-          "membership":"join"
-        }
-      },
-      { 
-        "event_id":"1b352d32", 
-        "type":"m.room.message",
-        "room_id":"room_beta@sy.org",
-        "msg_id":"m0001",
-        "from":"bob@sy.org",
-        "hs_ts":140193857,
-        "content":{
-          "msgtype":"m.text",
-          "body":"Hi everyone"
-        }
-      }
-    ],
-  "start": "stok_9348635",
-  "end": "etok_1984723"
-}
-
-Client disconnects for a while and the topic is updated in this room, 3 new 
-messages arrive whilst offline, and bob is invited to another room.
-
-GET /events?from=etok_1984723&access_token=abcdef0123456789
-Returns:
-{
-  "chunk": 
-    [
-      { 
-        "event_id":"feee0294", 
-        "type":"m.room.topic",
-        "room_id":"room_beta@sy.org",
-        "from":"alice@randomhost.org",
-        "content":{
-          "topic":"FRIENDS ONLY",
-        }
-      },
-      { 
-        "event_id":"a028bd9e", 
-        "type":"m.room.message",
-        "room_id":"room_beta@sy.org",
-        "msg_id":"z839409",
-        "from":"alice@randomhost.org",
-        "hs_ts":140195000,
-        "content":{
-          "msgtype":"m.text",
-          "body":"Hello!!!"
-        }
-      },
-      { 
-        "event_id":"49372d9e", 
-        "type":"m.room.message",
-        "room_id":"room_beta@sy.org",
-        "msg_id":"z839410",
-        "from":"alice@randomhost.org",
-        "hs_ts":140196000,
-        "content":{
-          "msgtype":"m.text",
-          "body":"Let's go to another room"
-        }
-      },
-      { 
-        "event_id":"10abdd01", 
-        "type":"m.room.message",
-        "room_id":"room_beta@sy.org",
-        "msg_id":"z839411",
-        "from":"alice@randomhost.org",
-        "hs_ts":140197000,
-        "content":{
-          "msgtype":"m.text",
-          "body":"You're not my friend"
-        }
-      },
-      { 
-        "event_id":"0018453d", 
-        "type":"m.room.member",
-        "room_id":"commoners@randomhost.org",
-        "from":"alice@randomhost.org",
-        "user_id":"bob@sy.org",
-        "content":{
-          "membership":"invite"
-        }
-      },
-    ],
-  "start": "stok_0184288",
-  "end": "etok_1348723"
-}

From ef1eb4c888378d2aecd66db57081bf90dd06b7fd Mon Sep 17 00:00:00 2001
From: Matthew Hodgson 
Date: Wed, 12 Nov 2014 01:14:06 +0200
Subject: [PATCH 221/265] this got merged into
 matrix-doc/specification/00_basis.rst by someone

---
 docs/client-server/model/terminology.rst | 86 ------------------------
 1 file changed, 86 deletions(-)
 delete mode 100644 docs/client-server/model/terminology.rst

diff --git a/docs/client-server/model/terminology.rst b/docs/client-server/model/terminology.rst
deleted file mode 100644
index cc6e6760a..000000000
--- a/docs/client-server/model/terminology.rst
+++ /dev/null
@@ -1,86 +0,0 @@
-===========
-Terminology
-===========
-
-A list of definitions of specific terminology used among these documents.
-These terms were originally taken from the server-server documentation, and may
-not currently match the exact meanings used in other places; though as a
-medium-term goal we should encourage the unification of this terminology.
-
-
-Terms
-=====
-
-Backfilling:
-  The process of synchronising historic state from one home server to another,
-  to backfill the event storage so that scrollback can be presented to the
-  client(s).  (Formerly, and confusingly, called 'pagination')
-
-Context:
-  A single human-level entity of interest (currently, a chat room)
-
-EDU (Ephemeral Data Unit):
-  A message that relates directly to a given pair of home servers that are
-  exchanging it. EDUs are short-lived messages that related only to one single
-  pair of servers; they are not persisted for a long time and are not forwarded
-  on to other servers. Because of this, they have no internal ID nor previous
-  EDUs reference chain.
-
-Event:
-  A record of activity that records a single thing that happened on to a context
-  (currently, a chat room). These are the "chat messages" that Synapse makes
-  available.
-  [[NOTE(paul): The current server-server implementation calls these simply
-  "messages" but the term is too ambiguous here; I've called them Events]]
-
-PDU (Persistent Data Unit):
-  A message that relates to a single context, irrespective of the server that
-  is communicating it. PDUs either encode a single Event, or a single State
-  change. A PDU is referred to by its PDU ID; the pair of its origin server
-  and local reference from that server.
-
-PDU ID:
-  The pair of PDU Origin and PDU Reference, that together globally uniquely
-  refers to a specific PDU.
-
-PDU Origin:
-  The name of the origin server that generated a given PDU. This may not be the
-  server from which it has been received, due to the way they are copied around
-  from server to server. The origin always records the original server that
-  created it.
-
-PDU Reference:
-  A local ID used to refer to a specific PDU from a given origin server. These
-  references are opaque at the protocol level, but may optionally have some
-  structured meaning within a given origin server or implementation.
-
-Presence:
-  The concept of whether a user is currently online, how available they declare
-  they are, and so on. See also: doc/model/presence
-
-Profile:
-  A set of metadata about a user, such as a display name, provided for the
-  benefit of other users. See also: doc/model/profiles
-
-Room ID:
-  An opaque string (of as-yet undecided format) that identifies a particular
-  room and used in PDUs referring to it.
-
-Room Alias:
-  A human-readable string of the form #name:some.domain that users can use as a
-  pointer to identify a room; a Directory Server will map this to its Room ID
-
-State:
-  A set of metadata maintained about a Context, which is replicated among the
-  servers in addition to the history of Events.
-
-User ID:
-  A string of the form @localpart:domain.name that identifies a user for
-  wire-protocol purposes. The localpart is meaningless outside of a particular
-  home server. This takes a human-readable form that end-users can use directly
-  if they so wish, avoiding the 3PIDs.
-
-Transaction:
-  A message which relates to the communication between a given pair of servers.
-  A transaction contains possibly-empty lists of PDUs and EDUs.
-

From 33e9e0fb2dafeb47ad60169c2cf484bc32d585d0 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson 
Date: Wed, 12 Nov 2014 01:16:38 +0200
Subject: [PATCH 222/265] move model/ into matrix-doc/drafts

---
 docs/client-server/model/presence.rst         | 149 ----------
 docs/client-server/model/profiles.rst         | 232 ---------------
 .../client-server/model/protocol_examples.rst |  64 ----
 .../model/room-join-workflow.rst              | 113 --------
 docs/client-server/model/rooms.rst            | 274 ------------------
 docs/client-server/model/third-party-id.rst   | 108 -------
 6 files changed, 940 deletions(-)
 delete mode 100644 docs/client-server/model/presence.rst
 delete mode 100644 docs/client-server/model/profiles.rst
 delete mode 100644 docs/client-server/model/protocol_examples.rst
 delete mode 100644 docs/client-server/model/room-join-workflow.rst
 delete mode 100644 docs/client-server/model/rooms.rst
 delete mode 100644 docs/client-server/model/third-party-id.rst

diff --git a/docs/client-server/model/presence.rst b/docs/client-server/model/presence.rst
deleted file mode 100644
index 811bac3fa..000000000
--- a/docs/client-server/model/presence.rst
+++ /dev/null
@@ -1,149 +0,0 @@
-API Efficiency
-==============
-
-A simple implementation of presence messaging has the ability to cause a large
-amount of Internet traffic relating to presence updates. In order to minimise
-the impact of such a feature, the following observations can be made:
-
- * There is no point in a Home Server polling status for peers in a user's
-   presence list if the user has no clients connected that care about it.
-
- * It is highly likely that most presence subscriptions will be symmetric - a
-   given user watching another is likely to in turn be watched by that user.
-
- * It is likely that most subscription pairings will be between users who share
-   at least one Room in common, and so their Home Servers are actively
-   exchanging message PDUs or transactions relating to that Room.
-
- * Presence update messages do not need realtime guarantees. It is acceptable to
-   delay delivery of updates for some small amount of time (10 seconds to a
-   minute).
-
-The general model of presence information is that of a HS registering its
-interest in receiving presence status updates from other HSes, which then
-promise to send them when required. Rather than actively polling for the
-currentt state all the time, HSes can rely on their relative stability to only
-push updates when required.
-
-A Home Server should not rely on the longterm validity of this presence
-information, however, as this would not cover such cases as a user's server
-crashing and thus failing to inform their peers that users it used to host are
-no longer available online. Therefore, each promise of future updates should
-carry with a timeout value (whether explicit in the message, or implicit as some
-defined default in the protocol), after which the receiving HS should consider
-the information potentially stale and request it again.
-
-However, because of the likelyhood that two home servers are exchanging messages
-relating to chat traffic in a room common to both of them, the ongoing receipt
-of these messages can be taken by each server as an implicit notification that
-the sending server is still up and running, and therefore that no status changes
-have happened; because if they had the server would have sent them. A second,
-larger timeout should be applied to this implicit inference however, to protect
-against implementation bugs or other reasons that the presence state cache may
-become invalid; eventually the HS should re-enquire the current state of users
-and update them with its own.
-
-The following workflows can therefore be used to handle presence updates:
-
- 1 When a user first appears online their HS sends a message to each other HS
-   containing at least one user to be watched; each message carrying both a
-   notification of the sender's new online status, and a request to obtain and
-   watch the target users' presence information. This message implicitly
-   promises the sending HS will now push updates to the target HSes.
-
- 2 The target HSes then respond a single message each, containing the current
-   status of the requested user(s). These messages too implicitly promise the
-   target HSes will themselves push updates to the sending HS.
-
-   As these messages arrive at the sending user's HS they can be pushed to the
-   user's client(s), possibly batched again to ensure not too many small
-   messages which add extra protocol overheads.
-
-At this point, all the user's clients now have the current presence status
-information for this moment in time, and have promised to send each other
-updates in future.
-
- 3 The HS maintains two watchdog timers per peer HS it is exchanging presence
-   information with. The first timer should have a relatively small expiry
-   (perhaps 1 minute), and the second timer should have a much longer time
-   (perhaps 1 hour).
-
- 4 Any time any kind of message is received from a peer HS, the short-term
-   presence timer associated with it is reset.
-
- 5 Whenever either of these timers expires, an HS should push a status reminder
-   to the target HS whose timer has now expired, and request again from that
-   server the status of the subscribed users.
-
- 6 On receipt of one of these presence status reminders, an HS can reset both
-   of its presence watchdog timers.
-
-To avoid bursts of traffic, implementations should attempt to stagger the expiry
-of the longer-term watchdog timers for different peer HSes.
-
-When individual users actively change their status (either by explicit requests
-from clients, or inferred changes due to idle timers or client timeouts), the HS
-should batch up any status changes for some reasonable amount of time (10
-seconds to a minute). This allows for reduced protocol overheads in the case of
-multiple messages needing to be sent to the same peer HS; as is the likely
-scenario in many cases, such as a given human user having multiple user
-accounts.
-
-
-API Requirements
-================
-
-The data model presented here puts the following requirements on the APIs:
-
-Client-Server
--------------
-
-Requests that a client can make to its Home Server
-
- * get/set current presence state
-   Basic enumeration + ability to set a custom piece of text
-
- * report per-device idle time
-   After some (configurable?) idle time the device should send a single message
-   to set the idle duration. The HS can then infer a "start of idle" instant and
-   use that to keep the device idleness up to date. At some later point the
-   device can cancel this idleness.
-
- * report per-device type
-   Inform the server that this device is a "mobile" device, or perhaps some
-   other to-be-defined category of reduced capability that could be presented to
-   other users.
-
- * start/stop presence polling for my presence list
-   It is likely that these messages could be implicitly inferred by other
-   messages, though having explicit control is always useful.
-
- * get my presence list
-   [implicit poll start?]
-   It is possible that the HS doesn't yet have current presence information when
-   the client requests this. There should be a "don't know" type too.
-
- * add/remove a user to my presence list
-
-Server-Server
--------------
-
-Requests that Home Servers make to others
-
- * request permission to add a user to presence list
-
- * allow/deny a request to add to a presence list
-
- * perform a combined presence state push and subscription request
-   For each sending user ID, the message contains their new status.
-   For each receiving user ID, the message should contain an indication on
-   whether the sending server is also interested in receiving status from that
-   user; either as an immediate update response now, or as a promise to send
-   future updates.
-
-Server to Client
-----------------
-
-[[TODO(paul): There also needs to be some way for a user's HS to push status
-updates of the presence list to clients, but the general server-client event
-model currently lacks a space to do that.]]
diff --git a/docs/client-server/model/profiles.rst b/docs/client-server/model/profiles.rst
deleted file mode 100644
index f7d6bd567..000000000
--- a/docs/client-server/model/profiles.rst
+++ /dev/null
@@ -1,232 +0,0 @@
-========
-Profiles
-========
-
-A description of Synapse user profile metadata support.
-
-
-Overview
-========
-
-Internally within Synapse users are referred to by an opaque ID, which consists
-of some opaque localpart combined with the domain name of their home server.
-Obviously this does not yield a very nice user experience; users would like to
-see readable names for other users that are in some way meaningful to them.
-Additionally, users like to be able to publish "profile" details to inform other
-users of other information about them.
-
-It is also conceivable that since we are attempting to provide a
-worldwide-applicable messaging system, that users may wish to present different
-subsets of information in their profile to different other people, from a
-privacy and permissions perspective.
-
-A Profile consists of a display name, an (optional?) avatar picture, and a set
-of other metadata fields that the user may wish to publish (email address, phone
-numbers, website URLs, etc...). We put no requirements on the display name other
-than it being a valid Unicode string. Since it is likely that users will end up
-having multiple accounts (perhaps by necessity of being hosted in multiple
-places, perhaps by choice of wanting multiple distinct identifies), it would be
-useful that a metadata field type exists that can refer to another Synapse User
-ID, so that clients and HSes can make use of this information.
-
-Metadata Fields
----------------
-
-[[TODO(paul): Likely this list is incomplete; more fields can be defined as we
-think of them. At the very least, any sort of supported ID for the 3rd Party ID
-servers should be accounted for here.]]
-
- * Synapse Directory Server username(s)
-
- * Email address
-
- * Phone number - classify "home"/"work"/"mobile"/custom?
- 
- * Twitter/Facebook/Google+/... social networks
-
- * Location - keep this deliberately vague to allow people to choose how
-     granular it is
- 
- * "Bio" information - date of birth, etc...
-
- * Synapse User ID of another account
-
- * Web URL
-
- * Freeform description text
-
-
-Visibility Permissions
-======================
-
-A home server implementation could offer the ability to set permissions on
-limited visibility of those fields. When another user requests access to the
-target user's profile, their own identity should form part of that request. The
-HS implementation can then decide which fields to make available to the
-requestor.
-
-A particular detail of implementation could allow the user to create one or more
-ACLs; where each list is granted permission to see a given set of non-public
-fields (compare to Google+ Circles) and contains a set of other people allowed
-to use it. By giving these ACLs strong identities within the HS, they can be
-referenced in communications with it, granting other users who encounter these
-the "ACL Token" to use the details in that ACL.
-
-If we further allow an ACL Token to be present on Room join requests or stored
-by 3PID servers, then users of these ACLs gain the extra convenience of not
-having to manually curate people in the access list; anyone in the room or with
-knowledge of the 3rd Party ID is automatically granted access. Every HS and
-client implementation would have to be aware of the existence of these ACL
-Token, and include them in requests if present, but not every HS implementation
-needs to actually provide the full permissions model. This can be used as a
-distinguishing feature among competing implementations. However, servers MUST
-NOT serve profile information from a cache if there is a chance that its limited
-understanding could lead to information leakage.
-
-
-Client Concerns of Multiple Accounts
-====================================
-
-Because a given person may want to have multiple Synapse User accounts, client
-implementations should allow the use of multiple accounts simultaneously
-(especially in the field of mobile phone clients, which generally don't support
-running distinct instances of the same application). Where features like address
-books, presence lists or rooms are presented, the client UI should remember to
-make distinct with user account is in use for each.
-
-
-Directory Servers
-=================
-
-Directory Servers can provide a forward mapping from human-readable names to
-User IDs. These can provide a service similar to giving domain-namespaced names
-for Rooms; in this case they can provide a way for a user to reference their
-User ID in some external form (e.g. that can be printed on a business card).
-
-The format for Synapse user name will consist of a localpart specific to the
-directory server, and the domain name of that directory server:
-
-  @localname:some.domain.name
-
-The localname is separated from the domain name using a colon, so as to ensure
-the localname can still contain periods, as users may want this for similarity
-to email addresses or the like, which typically can contain them. The format is
-also visually quite distinct from email addresses, phone numbers, etc... so
-hopefully reasonably "self-describing" when written on e.g. a business card
-without surrounding context.
-
-[[TODO(paul): we might have to think about this one - too close to email?
-  Twitter? Also it suggests a format scheme for room names of
-  #localname:domain.name, which I quite like]]
-
-Directory server administrators should be able to make some kind of policy
-decision on how these are allocated. Servers within some "closed" domain (such
-as company-specific ones) may wish to verify the validity of a mapping using
-their own internal mechanisms; "public" naming servers can operate on a FCFS
-basis. There are overlapping concerns here with the idea of the 3rd party
-identity servers as well, though in this specific case we are creating a new
-namespace to allocate names into.
-
-It would also be nice from a user experience perspective if the profile that a
-given name links to can also declare that name as part of its metadata.
-Furthermore as a security and consistency perspective it would be nice if each
-end (the directory server and the user's home server) check the validity of the
-mapping in some way. This needs investigation from a security perspective to
-ensure against spoofing.
-
-One such model may be that the user starts by declaring their intent to use a
-given user name link to their home server, which then contacts the directory
-service. At some point later (maybe immediately for "public open FCFS servers",
-maybe after some kind of human intervention for verification) the DS decides to
-honour this link, and includes it in its served output. It should also tell the
-HS of this fact, so that the HS can present this as fact when requested for the
-profile information. For efficiency, it may further wish to provide the HS with
-a cryptographically-signed certificate as proof, so the HS serving the profile
-can provide that too when asked, avoiding requesting HSes from constantly having
-to contact the DS to verify this mapping. (Note: This is similar to the security
-model often applied in DNS to verify PTR <-> A bidirectional mappings).
-
-
-Identity Servers
-================
-
-The identity servers should support the concept of pointing a 3PID being able to
-store an ACL Token as well as the main User ID. It is however, beyond scope to
-do any kind of verification that any third-party IDs that the profile is
-claiming match up to the 3PID mappings.
-
-
-User Interface and Expectations Concerns
-========================================
-
-Given the weak "security" of some parts of this model as compared to what users
-might expect, some care should be taken on how it is presented to users,
-specifically in the naming or other wording of user interface components.
-
-Most notably mere knowledge of an ACL Pointer is enough to read the information
-stored in it. It is possible that Home or Identity Servers could leak this
-information, allowing others to see it. This is a security-vs-convenience
-balancing choice on behalf of the user who would choose, or not, to make use of
-such a feature to publish their information.
-
-Additionally, unless some form of strong end-to-end user-based encryption is
-used, a user of ACLs for information privacy has to trust other home servers not
-to lie about the identify of the user requesting access to the Profile.
-
-
-API Requirements
-================
-
-The data model presented here puts the following requirements on the APIs:
-
-Client-Server
--------------
-
-Requests that a client can make to its Home Server
-
- * get/set my Display Name
-   This should return/take a simple "text/plain" field
-
- * get/set my Avatar URL
-   The avatar image data itself is not stored by this API; we'll just store a
-   URL to let the clients fetch it. Optionally HSes could integrate this with
-   their generic content attacmhent storage service, allowing a user to set
-   upload their profile Avatar and update the URL to point to it.
-
- * get/add/remove my metadata fields
-   Also we need to actually define types of metadata
-
- * get another user's Display Name / Avatar / metadata fields
-
-[[TODO(paul): At some later stage we should consider the API for:
-
- * get/set ACL permissions on my metadata fields
-
- * manage my ACL tokens
-]]
-
-Server-Server
--------------
-
-Requests that Home Servers make to others
-
- * get a user's Display Name / Avatar
-
- * get a user's full profile - name/avatar + MD fields
-   This request must allow for specifying the User ID of the requesting user,
-   for permissions purposes. It also needs to take into account any ACL Tokens
-   the requestor has.
-
- * push a change of Display Name to observers (overlaps with the presence API)
-
-Room Event PDU Types
---------------------
-
-Events that are pushed from Home Servers to other Home Servers or clients.
-
- * user Display Name change
- 
- * user Avatar change
-   [[TODO(paul): should the avatar image itself be stored in all the room
-   histories? maybe this event should just be a hint to clients that they should
-   re-fetch the avatar image]]
diff --git a/docs/client-server/model/protocol_examples.rst b/docs/client-server/model/protocol_examples.rst
deleted file mode 100644
index 61a599b43..000000000
--- a/docs/client-server/model/protocol_examples.rst
+++ /dev/null
@@ -1,64 +0,0 @@
-PUT /send/abc/ HTTP/1.1
-Host: ...
-Content-Length: ...
-Content-Type: application/json
-
-{
-    "origin": "localhost:5000",
-    "pdus": [
-        {
-            "content": {},
-            "context": "tng",
-            "depth": 12,
-            "is_state": false,
-            "origin": "localhost:5000",
-            "pdu_id": 1404381396854,
-            "pdu_type": "feedback",
-            "prev_pdus": [
-                [
-                    "1404381395883",
-                    "localhost:6000"
-                ]
-            ],
-            "ts": 1404381427581
-        }
-    ],
-    "prev_ids": [
-        "1404381396852"
-    ],
-    "ts": 1404381427823
-}
-
-HTTP/1.1 200 OK
-...
-
-======================================
-
-GET /pull/-1/ HTTP/1.1
-Host: ...
-Content-Length: 0
-
-HTTP/1.1 200 OK
-Content-Length: ...
-Content-Type: application/json
-
-{
-    origin: ...,
-    prev_ids: ...,
-    data: [
-        {
-            data_id: ...,
-            prev_pdus: [...],
-            depth: ...,
-            ts: ...,
-            context: ...,
-            origin: ...,
-            content: {
-                ...
-            }
-        },
-        ...,
-    ]
-}
-
-
diff --git a/docs/client-server/model/room-join-workflow.rst b/docs/client-server/model/room-join-workflow.rst
deleted file mode 100644
index c321a64fa..000000000
--- a/docs/client-server/model/room-join-workflow.rst
+++ /dev/null
@@ -1,113 +0,0 @@
-==================
-Room Join Workflow
-==================
-
-An outline of the workflows required when a user joins a room.
-
-Discovery
-=========
-
-To join a room, a user has to discover the room by some mechanism in order to
-obtain the (opaque) Room ID and a candidate list of likely home servers that
-contain it.
-
-Sending an Invitation
----------------------
-
-The most direct way a user discovers the existence of a room is from a
-invitation from some other user who is a member of that room.
-
-The inviter's HS sets the membership status of the invitee to "invited" in the
-"m.members" state key by sending a state update PDU. The HS then broadcasts this
-PDU among the existing members in the usual way. An invitation message is also
-sent to the invited user, containing the Room ID and the PDU ID of this
-invitation state change and potentially a list of some other home servers to use
-to accept the invite. The user's client can then choose to display it in some
-way to alert the user.
-
-[[TODO(paul): At present, no API has been designed or described to actually send
-that invite to the invited user. Likely it will be some facet of the larger
-user-user API required for presence, profile management, etc...]]
-
-Directory Service
------------------
-
-Alternatively, the user may discover the channel via a directory service; either
-by performing a name lookup, or some kind of browse or search acitivty. However
-this is performed, the end result is that the user's home server requests the
-Room ID and candidate list from the directory service.
-
-[[TODO(paul): At present, no API has been designed or described for this
-directory service]]
-
-
-Joining
-=======
-
-Once the ID and home servers are obtained, the user can then actually join the
-room.
-
-Accepting an Invite
--------------------
-
-If a user has received and accepted an invitation to join a room, the invitee's
-home server can now send an invite acceptance message to a chosen candidate
-server from the list given in the invitation, citing also the PDU ID of the
-invitation as "proof" of their invite. (This is required as due to late message
-propagation it could be the case that the acceptance is received before the
-invite by some servers). If this message is allowed by the candidate server, it
-generates a new PDU that updates the invitee's membership status to "joined",
-referring back to the acceptance PDU, and broadcasts that as a state change in
-the usual way. The newly-invited user is now a full member of the room, and
-state propagation proceeds as usual.
-
-Joining a Public Room
----------------------
-
-If a user has discovered the existence of a room they wish to join but does not
-have an active invitation, they can request to join it directly by sending a
-join message to a candidate server on the list provided by the directory
-service. As this list may be out of date, the HS should be prepared to retry
-other candidates if the chosen one is no longer aware of the room, because it
-has no users as members in it.
-
-Once a candidate server that is aware of the room has been found, it can
-broadcast an update PDU to add the member into the "m.members" key setting their
-state directly to "joined" (i.e. bypassing the two-phase invite semantics),
-remembering to include the new user's HS in that list.
-
-Knocking on a Semi-Public Room
-------------------------------
-
-If a user requests to join a room but the join mode of the room is "knock", the
-join is not immediately allowed. Instead, if the user wishes to proceed, they
-can instead post a "knock" message, which informs other members of the room that
-the would-be joiner wishes to become a member and sets their membership value to
-"knocked". If any of them wish to accept this, they can then send an invitation
-in the usual way described above. Knowing that the user has already knocked and
-expressed an interest in joining, the invited user's home server should
-immediately accept that invitation on the user's behalf, and go on to join the
-room in the usual way.
-
-[[NOTE(Erik): Though this may confuse users who expect 'X has joined' to
-actually be a user initiated action, i.e. they may expect that 'X' is actually
-looking at synapse right now?]]
-
-[[NOTE(paul): Yes, a fair point maybe we should suggest HSes don't do that, and
-just offer an invite to the user as normal]]
-
-Private and Non-Existent Rooms
-------------------------------
-
-If a user requests to join a room but the room is either unknown by the home
-server receiving the request, or is known by the join mode is "invite" and the
-user has not been invited, the server must respond that the room does not exist.
-This is to prevent leaking information about the existence and identity of
-private rooms.
-
-
-Outstanding Questions
-=====================
-
- * Do invitations or knocks time out and expire at some point? If so when? Time
-   is hard in distributed systems.
diff --git a/docs/client-server/model/rooms.rst b/docs/client-server/model/rooms.rst
deleted file mode 100644
index 0007e48e3..000000000
--- a/docs/client-server/model/rooms.rst
+++ /dev/null
@@ -1,274 +0,0 @@
-===========
-Rooms Model
-===========
-
-A description of the general data model used to implement Rooms, and the
-user-level visible effects and implications.
-
-
-Overview
-========
-
-"Rooms" in Synapse are shared messaging channels over which all the participant
-users can exchange messages. Rooms have an opaque persistent identify, a
-globally-replicated set of state (consisting principly of a membership set of
-users, and other management and miscellaneous metadata), and a message history.
-
-
-Room Identity and Naming
-========================
-
-Rooms can be arbitrarily created by any user on any home server; at which point
-the home server will sign the message that creates the channel, and the
-fingerprint of this signature becomes the strong persistent identify of the
-room. This now identifies the room to any home server in the network regardless
-of its original origin. This allows the identify of the room to outlive any
-particular server. Subject to appropriate permissions [to be discussed later],
-any current member of a room can invite others to join it, can post messages
-that become part of its history, and can change the persistent state of the room
-(including its current set of permissions).
-
-Home servers can provide a directory service, allowing a lookup from a
-convenient human-readable form of room label to a room ID. This mapping is
-scoped to the particular home server domain and so simply represents that server
-administrator's opinion of what room should take that label; it does not have to
-be globally replicated and does not form part of the stored state of that room.
-
-This room name takes the form
-
-  #localname:some.domain.name
-
-for similarity and consistency with user names on directories.
-
-To join a room (and therefore to be allowed to inspect past history, post new
-messages to it, and read its state), a user must become aware of the room's
-fingerprint ID. There are two mechanisms to allow this:
-
- * An invite message from someone else in the room
-
- * A referral from a room directory service
-
-As room IDs are opaque and ephemeral, they can serve as a mechanism to create
-"ad-hoc" rooms deliberately unnamed, for small group-chats or even private
-one-to-one message exchange.
-
-
-Stored State and Permissions
-============================
-
-Every room has a globally-replicated set of stored state. This state is a set of
-key/value or key/subkey/value pairs. The value of every (sub)key is a
-JSON-representable object. The main key of a piece of stored state establishes
-its meaning; some keys store sub-keys to allow a sub-structure within them [more
-detail below]. Some keys have special meaning to Synapse, as they relate to
-management details of the room itself, storing such details as user membership,
-and permissions of users to alter the state of the room itself. Other keys may
-store information to present to users, which the system does not directly rely
-on. The key space itself is namespaced, allowing 3rd party extensions, subject
-to suitable permission.
-
-Permission management is based on the concept of "power-levels". Every user
-within a room has an integer assigned, being their "power-level" within that
-room. Along with its actual data value, each key (or subkey) also stores the
-minimum power-level a user must have in order to write to that key, the
-power-level of the last user who actually did write to it, and the PDU ID of
-that state change.
-
-To be accepted as valid, a change must NOT:
-
- * Be made by a user having a power-level lower than required to write to the
-   state key
-
- * Alter the required power-level for that state key to a value higher than the
-   user has
-
- * Increase that user's own power-level
-
- * Grant any other user a power-level higher than the level of the user making
-   the change
-
-[[TODO(paul): consider if relaxations should be allowed; e.g. is the current
-outright-winner allowed to raise their own level, to allow for "inflation"?]]
-
-
-Room State Keys
-===============
-
-[[TODO(paul): if this list gets too big it might become necessary to move it
-into its own doc]]
-
-The following keys have special semantics or meaning to Synapse itself:
-
-m.member (has subkeys)
-  Stores a sub-key for every Synapse User ID which is currently a member of
-  this room. Its value gives the membership type ("knocked", "invited",
-  "joined").
-
-m.power_levels
-  Stores a mapping from Synapse User IDs to their power-level in the room. If
-  they are not present in this mapping, the default applies.
-
-  The reason to store this as a single value rather than a value with subkeys
-  is that updates to it are atomic; allowing a number of colliding-edit
-  problems to be avoided.
-
-m.default_level
-  Gives the default power-level for members of the room that do not have one
-  specified in their membership key.
-
-m.invite_level
-  If set, gives the minimum power-level required for members to invite others
-  to join, or to accept knock requests from non-members requesting access. If
-  absent, then invites are not allowed. An invitation involves setting their
-  membership type to "invited", in addition to sending the invite message.
-
-m.join_rules
-  Encodes the rules on how non-members can join the room. Has the following
-  possibilities:
-    "public" - a non-member can join the room directly
-    "knock" - a non-member cannot join the room, but can post a single "knock"
-        message requesting access, which existing members may approve or deny
-    "invite" - non-members cannot join the room without an invite from an
-        existing member
-    "private" - nobody who is not in the 'may_join' list or already a member
-        may join by any mechanism
-
-  In any of the first three modes, existing members with sufficient permission
-  can send invites to non-members if allowed by the "m.invite_level" key. A
-  "private" room is not allowed to have the "m.invite_level" set.
-
-  A client may use the value of this key to hint at the user interface
-  expectations to provide; in particular, a private chat with one other use
-  might warrant specific handling in the client.
-
-m.may_join
-  A list of User IDs that are always allowed to join the room, regardless of any
-  of the prevailing join rules and invite levels. These apply even to private
-  rooms. These are stored in a single list with normal update-powerlevel
-  permissions applied; users cannot arbitrarily remove themselves from the list.
-
-m.add_state_level
-  The power-level required for a user to be able to add new state keys.
-
-m.public_history
-  If set and true, anyone can request the history of the room, without needing
-  to be a member of the room.
-
-m.archive_servers
-  For "public" rooms with public history, gives a list of home servers that
-  should be included in message distribution to the room, even if no users on
-  that server are present. These ensure that a public room can still persist
-  even if no users are currently members of it. This list should be consulted by
-  the dirctory servers as the candidate list they respond with.
-
-The following keys are provided by Synapse for user benefit, but their value is
-not otherwise used by Synapse.
-
-m.name
-  Stores a short human-readable name for the room, such that clients can display
-  to a user to assist in identifying which room is which.
-  
-  This name specifically is not the strong ID used by the message transport
-  system to refer to the room, because it may be changed from time to time.
-
-m.topic
-  Stores the current human-readable topic
-
-
-Room Creation Templates
-=======================
-
-A client (or maybe home server?) could offer a few templates for the creation of
-new rooms. For example, for a simple private one-to-one chat the channel could
-assign the creator a power-level of 1, requiring a level of 1 to invite, and
-needing an invite before members can join. An invite is then sent to the other
-party, and if accepted and the other user joins, the creator's power-level can
-now be reduced to 0. This now leaves a room with two participants in it being
-unable to add more.
-
-
-Rooms that Continue History
-===========================
-
-An option that could be considered for room creation, is that when a new room is
-created the creator could specify a PDU ID into an existing room, as the history
-continuation point. This would be stored as an extra piece of meta-data on the
-initial PDU of the room's creation. (It does not appear in the normal previous
-PDU linkage).
-
-This would allow users in rooms to "fork" a room, if it is considered that the
-conversations in the room no longer fit its original purpose, and wish to
-diverge. Existing permissions on the original room would continue to apply of
-course, for viewing that history. If both rooms are considered "public" we might
-also want to define a message to post into the original room to represent this
-fork point, and give a reference to the new room.
-
-
-User Direct Message Rooms
-=========================
-
-There is no need to build a mechanism for directly sending messages between
-users, because a room can handle this ability. To allow direct user-to-user chat
-messaging we simply need to be able to create rooms with specific set of
-permissions to allow this direct messaging.
-
-Between any given pair of user IDs that wish to exchange private messages, there
-will exist a single shared Room, created lazily by either side. These rooms will
-need a certain amount of special handling in both home servers and display on
-clients, but as much as possible should be treated by the lower layers of code
-the same as other rooms.
-
-Specially, a client would likely offer a special menu choice associated with
-another user (in room member lists, presence list, etc..) as "direct chat". That
-would perform all the necessary steps to create the private chat room. Receiving
-clients should display these in a special way too as the room name is not
-important; instead it should distinguish them on the Display Name of the other
-party.
-
-Home Servers will need a client-API option to request setting up a new user-user
-chat room, which will then need special handling within the server. It will
-create a new room with the following 
-
-  m.member: the proposing user
-  m.join_rules: "private"
-  m.may_join: both users
-  m.power_levels: empty
-  m.default_level: 0
-  m.add_state_level: 0
-  m.public_history: False
-
-Having created the room, it can send an invite message to the other user in the
-normal way - the room permissions state that no users can be set to the invited
-state, but because they're in the may_join list then they'd be allowed to join
-anyway.
-
-In this arrangement there is now a room with both users may join but neither has
-the power to invite any others. Both users now have the confidence that (at
-least within the messaging system itself) their messages remain private and
-cannot later be provably leaked to a third party. They can freely set the topic
-or name if they choose and add or edit any other state of the room. The update
-powerlevel of each of these fixed properties should be 1, to lock out the users
-from being able to alter them.
-
-
-Anti-Glare
-==========
-
-There exists the possibility of a race condition if two users who have no chat
-history with each other simultaneously create a room and invite the other to it.
-This is called a "glare" situation. There are two possible ideas for how to
-resolve this:
-
- * Each Home Server should persist the mapping of (user ID pair) to room ID, so
-   that duplicate requests can be suppressed. On receipt of a room creation
-   request that the HS thinks there already exists a room for, the invitation to
-   join can be rejected if:
-      a) the HS believes the sending user is already a member of the room (and
-         maybe their HS has forgotten this fact), or
-      b) the proposed room has a lexicographically-higher ID than the existing
-         room (to resolve true race condition conflicts)
-      
- * The room ID for a private 1:1 chat has a special form, determined by
-   concatenting the User IDs of both members in a deterministic order, such that
-   it doesn't matter which side creates it first; the HSes can just ignore
-   (or merge?) received PDUs that create the room twice.
diff --git a/docs/client-server/model/third-party-id.rst b/docs/client-server/model/third-party-id.rst
deleted file mode 100644
index 1f8138ddf..000000000
--- a/docs/client-server/model/third-party-id.rst
+++ /dev/null
@@ -1,108 +0,0 @@
-======================
-Third Party Identities
-======================
-
-A description of how email addresses, mobile phone numbers and other third
-party identifiers can be used to authenticate and discover users in Matrix.
-
-
-Overview
-========
-
-New users need to authenticate their account. An email or SMS text message can 
-be a convenient form of authentication. Users already have email addresses 
-and phone numbers for contacts in their address book. They want to communicate
-with those contacts in Matrix without manually exchanging a Matrix User ID with 
-them.
-
-Third Party IDs
----------------
-
-[[TODO(markjh): Describe the format of a 3PID]]
-
-
-Third Party ID Associations
----------------------------
-
-An Associaton is a binding between a Matrix User ID and a Third Party ID (3PID).
-Each 3PID can be associated with one Matrix User ID at a time.
-
-[[TODO(markjh): JSON format of the association.]]
-
-Verification 
-------------
-
-An Assocation must be verified by a trusted Verification Server. Email 
-addresses and phone numbers can be verified by sending a token to the address 
-which a client can supply to the verifier to confirm ownership.
-
-An email Verification Server may be capable of verifying all email 3PIDs or may
-be restricted to verifying addresses for a particular domain. A phone number
-Verification Server may be capable of verifying all phone numbers or may be
-restricted to verifying numbers for a given country or phone prefix.
-
-Verification Servers fulfil a similar role to Certificate Authorities in PKI so
-a similar level of vetting should be required before clients trust their
-signatures.
-
-A Verification Server may wish to check for existing Associations for a 3PID 
-before creating a new Association.
-
-Discovery
----------
-
-Users can discover Associations using a trusted Identity Server. Each 
-Association will be signed by the Identity Server. An Identity Server may store
-the entire space of Associations or may delegate to other Identity Servers when
-looking up Associations.
-
-Each Association returned from an Identity Server must be signed by a 
-Verification Server. Clients should check these signatures.
-
-Identity Servers fulfil a similar role to DNS servers.
-
-Privacy
--------
-
-A User may publish the association between their phone number and Matrix User ID
-on the Identity Server without publishing the number in their Profile hosted on
-their Home Server.
-
-Identity Servers should refrain from publishing reverse mappings and should 
-take steps, such as rate limiting, to prevent attackers enumerating the space of
-mappings.
-
-Federation
-==========
-
-Delegation
-----------
-
-Verification Servers could delegate signing to another server by issuing 
-certificate to that server allowing it to verify and sign a subset of 3PID on 
-its behalf. It would be necessary to provide a language for describing which
-subset of 3PIDs that server had authority to validate. Alternatively it could 
-delegate the verification step to another server but sign the resulting
-association itself.
-
-The 3PID space will have a heirachical structure like DNS so Identity Servers
-can delegate lookups to other servers. An Identity Server should be prepared 
-to host or delegate any valid association within the subset of the 3PIDs it is 
-resonsible for.
-
-Multiple Root Verification Servers
-----------------------------------
-
-There can be multiple root Verification Servers and an Association could be
-signed by multiple servers if different clients trust different subsets of
-the verification servers.
-
-Multiple Root Identity Servers
-------------------------------
-
-There can be be multiple root Identity Servers. Clients will add each
-Association to all root Identity Servers.
-
-[[TODO(markjh): Describe how clients find the list of root Identity Servers]]
-
-

From 9d0efedaee1e522e0fbc1300aee0c27e3ebf3eae Mon Sep 17 00:00:00 2001
From: Kegan Dougal 
Date: Wed, 12 Nov 2014 11:14:19 +0000
Subject: [PATCH 223/265] Move room alias/id mapping logic from matrixService
 to modelService.

---
 .../matrix/event-handler-service.js           |  2 +-
 .../components/matrix/matrix-filter.js        |  2 +-
 .../components/matrix/matrix-service.js       | 60 -------------------
 .../components/matrix/model-service.js        | 60 ++++++++++++++++++-
 syweb/webclient/home/home-controller.js       |  8 +--
 syweb/webclient/room/room-controller.js       |  4 +-
 syweb/webclient/test/unit/filters.spec.js     | 10 ++--
 7 files changed, 72 insertions(+), 74 deletions(-)

diff --git a/syweb/webclient/components/matrix/event-handler-service.js b/syweb/webclient/components/matrix/event-handler-service.js
index f51031f4c..7b2a75507 100644
--- a/syweb/webclient/components/matrix/event-handler-service.js
+++ b/syweb/webclient/components/matrix/event-handler-service.js
@@ -92,7 +92,7 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati
     };
 
     var handleRoomAliases = function(event, isLiveEvent) {
-        matrixService.createRoomIdToAliasMapping(event.room_id, event.content.aliases[0]);
+        modelService.createRoomIdToAliasMapping(event.room_id, event.content.aliases[0]);
     };
     
     var displayNotification = function(event) {
diff --git a/syweb/webclient/components/matrix/matrix-filter.js b/syweb/webclient/components/matrix/matrix-filter.js
index 69de97b05..20a29c4ab 100644
--- a/syweb/webclient/components/matrix/matrix-filter.js
+++ b/syweb/webclient/components/matrix/matrix-filter.js
@@ -28,7 +28,7 @@ function($rootScope, matrixService, eventHandlerService, modelService) {
 
         // If there is an alias, use it
         // TODO: only one alias is managed for now
-        var alias = matrixService.getRoomIdToAliasMapping(room_id);
+        var alias = modelService.getRoomIdToAliasMapping(room_id);
         var room = modelService.getRoom(room_id).current_room_state;
         
         var room_name_event = room.state("m.room.name");
diff --git a/syweb/webclient/components/matrix/matrix-service.js b/syweb/webclient/components/matrix/matrix-service.js
index 63051c4f4..70747155f 100644
--- a/syweb/webclient/components/matrix/matrix-service.js
+++ b/syweb/webclient/components/matrix/matrix-service.js
@@ -36,9 +36,6 @@ angular.module('matrixService', [])
     */    
     var config;
     
-    var roomIdToAlias = {};
-    var aliasToRoomId = {};
-    
     // Current version of permanent storage
     var configVersion = 0;
     var prefixPath = "/_matrix/client/api/v1";
@@ -671,63 +668,6 @@ angular.module('matrixService', [])
             config.version = configVersion;
             localStorage.setItem("config", JSON.stringify(config));
         },
-
-
-        /****** Room aliases management ******/
-
-        /**
-         * Get the room_alias & room_display_name which are computed from data 
-         * already retrieved from the server.
-         * @param {Room object} room one element of the array returned by the response
-         *  of rooms() and publicRooms()
-         * @returns {Object} {room_alias: "...", room_display_name: "..."}
-         */
-        getRoomAliasAndDisplayName: function(room) {
-            var result = {
-                room_alias: undefined,
-                room_display_name: undefined
-            };
-            var alias = this.getRoomIdToAliasMapping(room.room_id);
-            if (alias) {
-                // use the existing alias from storage
-                result.room_alias = alias;
-                result.room_display_name = alias;
-            }
-            // XXX: this only lets us learn aliases from our local HS - we should
-            // make the client stop returning this if we can trust m.room.aliases state events
-            else if (room.aliases && room.aliases[0]) {
-                // save the mapping
-                // TODO: select the smarter alias from the array
-                this.createRoomIdToAliasMapping(room.room_id, room.aliases[0]);
-                result.room_display_name = room.aliases[0];
-                result.room_alias = room.aliases[0];
-            }
-            else if (room.membership === "invite" && "inviter" in room) {
-                result.room_display_name = room.inviter + "'s room";
-            }
-            else {
-                // last resort use the room id
-                result.room_display_name = room.room_id;
-            }
-            return result;
-        },
-        
-        createRoomIdToAliasMapping: function(roomId, alias) {
-            roomIdToAlias[roomId] = alias;
-            aliasToRoomId[alias] = roomId;
-        },
-        
-        getRoomIdToAliasMapping: function(roomId) {
-            var alias = roomIdToAlias[roomId];
-            //console.log("looking for alias for " + roomId + "; found: " + alias);
-            return alias;
-        },
-
-        getAliasToRoomIdMapping: function(alias) {
-            var roomId = aliasToRoomId[alias];
-            //console.log("looking for roomId for " + alias + "; found: " + roomId);
-            return roomId;
-        },
             
         /**
          * Change or reset the power level of a user
diff --git a/syweb/webclient/components/matrix/model-service.js b/syweb/webclient/components/matrix/model-service.js
index 8e0ce8d1a..c6f0defb6 100644
--- a/syweb/webclient/components/matrix/model-service.js
+++ b/syweb/webclient/components/matrix/model-service.js
@@ -27,6 +27,10 @@ dependency.
 //     $rootScope.
 angular.module('modelService', [])
 .factory('modelService', ['matrixService', function(matrixService) {
+
+    // alias / id lookups
+    var roomIdToAlias = {};
+    var aliasToRoomId = {};
     
     /***** Room Object *****/
     var Room = function Room(room_id) {
@@ -166,7 +170,61 @@ angular.module('modelService', [])
         getMember: function(room_id, user_id) {
             var room = this.getRoom(room_id);
             return room.current_room_state.members[user_id];
-        }
+        },
+        
+        /**
+         * Get the room_alias & room_display_name which are computed from data 
+         * already retrieved from the server.
+         * @param {Room object} room one element of the array returned by the response
+         *  of rooms() and publicRooms()
+         * @returns {Object} {room_alias: "...", room_display_name: "..."}
+         */
+        getRoomAliasAndDisplayName: function(room) {
+            var result = {
+                room_alias: undefined,
+                room_display_name: undefined
+            };
+            var alias = this.getRoomIdToAliasMapping(room.room_id);
+            if (alias) {
+                // use the existing alias from storage
+                result.room_alias = alias;
+                result.room_display_name = alias;
+            }
+            // XXX: this only lets us learn aliases from our local HS - we should
+            // make the client stop returning this if we can trust m.room.aliases state events
+            else if (room.aliases && room.aliases[0]) {
+                // save the mapping
+                // TODO: select the smarter alias from the array
+                this.createRoomIdToAliasMapping(room.room_id, room.aliases[0]);
+                result.room_display_name = room.aliases[0];
+                result.room_alias = room.aliases[0];
+            }
+            else if (room.membership === "invite" && "inviter" in room) {
+                result.room_display_name = room.inviter + "'s room";
+            }
+            else {
+                // last resort use the room id
+                result.room_display_name = room.room_id;
+            }
+            return result;
+        },
+        
+        createRoomIdToAliasMapping: function(roomId, alias) {
+            roomIdToAlias[roomId] = alias;
+            aliasToRoomId[alias] = roomId;
+        },
+        
+        getRoomIdToAliasMapping: function(roomId) {
+            var alias = roomIdToAlias[roomId];
+            //console.log("looking for alias for " + roomId + "; found: " + alias);
+            return alias;
+        },
+
+        getAliasToRoomIdMapping: function(alias) {
+            var roomId = aliasToRoomId[alias];
+            //console.log("looking for roomId for " + alias + "; found: " + roomId);
+            return roomId;
+        },
     
     };
 }]);
diff --git a/syweb/webclient/home/home-controller.js b/syweb/webclient/home/home-controller.js
index 3a48e64ab..467db09b3 100644
--- a/syweb/webclient/home/home-controller.js
+++ b/syweb/webclient/home/home-controller.js
@@ -17,8 +17,8 @@ limitations under the License.
 'use strict';
 
 angular.module('HomeController', ['matrixService', 'eventHandlerService', 'RecentsController'])
-.controller('HomeController', ['$scope', '$location', 'matrixService', 'eventHandlerService', 
-                               function($scope, $location, matrixService, eventHandlerService) {
+.controller('HomeController', ['$scope', '$location', 'matrixService', 'eventHandlerService', 'modelService',
+                               function($scope, $location, matrixService, eventHandlerService, modelService) {
 
     $scope.config = matrixService.config();
     $scope.public_rooms = [];
@@ -56,7 +56,7 @@ angular.module('HomeController', ['matrixService', 'eventHandlerService', 'Recen
                     var room = $scope.public_rooms[i];
 
                     // Add room_alias & room_display_name members
-                    angular.extend(room, matrixService.getRoomAliasAndDisplayName(room));
+                    angular.extend(room, modelService.getRoomAliasAndDisplayName(room));
                     
                 }
             }
@@ -75,7 +75,7 @@ angular.module('HomeController', ['matrixService', 'eventHandlerService', 'Recen
                 // This room has been created. Refresh the rooms list
                 console.log("Created room " + response.data.room_alias + " with id: "+
                 response.data.room_id);
-                matrixService.createRoomIdToAliasMapping(
+                modelService.createRoomIdToAliasMapping(
                     response.data.room_id, response.data.room_alias);
             },
             function(error) {
diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js
index be433d6e8..2a79fbfbb 100644
--- a/syweb/webclient/room/room-controller.js
+++ b/syweb/webclient/room/room-controller.js
@@ -490,7 +490,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a
                                 // with or without port as is appropriate and append it at this point
                             }
                             
-                            var room_id = matrixService.getAliasToRoomIdMapping(room_alias);
+                            var room_id = modelService.getAliasToRoomIdMapping(room_alias);
                             console.log("joining " + room_alias + " id=" + room_id);
                             if ($scope.room) { // TODO actually check that you = join
                                 // don't send a join event for a room you're already in.
@@ -677,7 +677,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a
         if (room_id_or_alias && '!' === room_id_or_alias[0]) {
             // Yes. We can go on right now
             $scope.room_id = room_id_or_alias;
-            $scope.room_alias = matrixService.getRoomIdToAliasMapping($scope.room_id);
+            $scope.room_alias = modelService.getRoomIdToAliasMapping($scope.room_id);
             onInit2();
         }
         else {
diff --git a/syweb/webclient/test/unit/filters.spec.js b/syweb/webclient/test/unit/filters.spec.js
index 7324a8e02..ef3381293 100644
--- a/syweb/webclient/test/unit/filters.spec.js
+++ b/syweb/webclient/test/unit/filters.spec.js
@@ -8,10 +8,6 @@ describe('mRoomName filter', function() {
     
     // mocked services which return the test values above.
     var matrixService = {
-        getRoomIdToAliasMapping: function(room_id) {
-            return testAlias;
-        },
-        
         config: function() {
             return {
                 user_id: testUserId
@@ -33,7 +29,11 @@ describe('mRoomName filter', function() {
             return {
                 current_room_state: testRoomState
             };
-        }
+        },
+        
+        getRoomIdToAliasMapping: function(room_id) {
+            return testAlias;
+        },
     };
     
     beforeEach(function() {

From 2c400363e8f4c7d726aee06119fb4c4f18edfddd Mon Sep 17 00:00:00 2001
From: Kegan Dougal 
Date: Wed, 12 Nov 2014 11:24:05 +0000
Subject: [PATCH 224/265] SYWEB-146: Fix room ID leaking on recents page when
 the name of the room is just an alias.

---
 syweb/webclient/components/matrix/matrix-service.js |  1 -
 syweb/webclient/components/matrix/model-service.js  | 10 ++++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/syweb/webclient/components/matrix/matrix-service.js b/syweb/webclient/components/matrix/matrix-service.js
index 70747155f..c1264887c 100644
--- a/syweb/webclient/components/matrix/matrix-service.js
+++ b/syweb/webclient/components/matrix/matrix-service.js
@@ -39,7 +39,6 @@ angular.module('matrixService', [])
     // Current version of permanent storage
     var configVersion = 0;
     var prefixPath = "/_matrix/client/api/v1";
-    var MAPPING_PREFIX = "alias_for_";
 
     var doRequest = function(method, path, params, data, $httpParams) {
         if (!config) {
diff --git a/syweb/webclient/components/matrix/model-service.js b/syweb/webclient/components/matrix/model-service.js
index c6f0defb6..578804c99 100644
--- a/syweb/webclient/components/matrix/model-service.js
+++ b/syweb/webclient/components/matrix/model-service.js
@@ -31,6 +31,10 @@ angular.module('modelService', [])
     // alias / id lookups
     var roomIdToAlias = {};
     var aliasToRoomId = {};
+    var setRoomIdToAliasMapping = function(roomId, alias) {
+        roomIdToAlias[roomId] = alias;
+        aliasToRoomId[alias] = roomId;
+    };
     
     /***** Room Object *****/
     var Room = function Room(room_id) {
@@ -114,6 +118,9 @@ angular.module('modelService', [])
                 rm.event = event;
                 this.members[event.state_key] = rm;
             }
+            else if (event.type === "m.room.aliases") {
+                setRoomIdToAliasMapping(event.room_id, event.content.aliases[0]);
+            }
         },
         
         storeStateEvents: function storeState(events) {
@@ -210,8 +217,7 @@ angular.module('modelService', [])
         },
         
         createRoomIdToAliasMapping: function(roomId, alias) {
-            roomIdToAlias[roomId] = alias;
-            aliasToRoomId[alias] = roomId;
+            setRoomIdToAliasMapping(roomId, alias);
         },
         
         getRoomIdToAliasMapping: function(roomId) {

From 6fea478d2e7737c2462b074b935d4427ced5f3d4 Mon Sep 17 00:00:00 2001
From: Erik Johnston 
Date: Wed, 12 Nov 2014 11:22:51 +0000
Subject: [PATCH 225/265] Fix bugs with invites/joins across federatiom.

Both in terms of auth and not trying to fetch missing PDUs for invites,
joins etc.
---
 synapse/api/auth.py               | 19 ++++++++++++++++---
 synapse/federation/replication.py |  7 +------
 synapse/handlers/federation.py    | 12 +++---------
 synapse/handlers/room.py          | 10 ++++++++--
 synapse/storage/__init__.py       | 14 +++++++++++---
 synapse/storage/state.py          |  9 ++++++---
 tests/handlers/test_room.py       | 22 +++++++++-------------
 7 files changed, 54 insertions(+), 39 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 6c2d3db26..87f19a96d 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -36,6 +36,7 @@ class Auth(object):
     def __init__(self, hs):
         self.hs = hs
         self.store = hs.get_datastore()
+        self.state = hs.get_state_handler()
 
     def check(self, event, raises=False):
         """ Checks if this event is correctly authed.
@@ -90,7 +91,7 @@ class Auth(object):
             )
             logger.info("Denying! %s", event)
             if raises:
-                raise e
+                raise
 
         return False
 
@@ -109,9 +110,21 @@ class Auth(object):
 
     @defer.inlineCallbacks
     def check_host_in_room(self, room_id, host):
-        joined_hosts = yield self.store.get_joined_hosts_for_room(room_id)
+        curr_state = yield self.state.get_current_state(room_id)
 
-        defer.returnValue(host in joined_hosts)
+        for event in curr_state:
+            if event.type == RoomMemberEvent.TYPE:
+                try:
+                    if self.hs.parse_userid(event.state_key).domain != host:
+                        continue
+                except:
+                    logger.warn("state_key not user_id: %s", event.state_key)
+                    continue
+
+                if event.content["membership"] == Membership.JOIN:
+                    defer.returnValue(True)
+
+        defer.returnValue(False)
 
     def check_event_sender_in_room(self, event):
         key = (RoomMemberEvent.TYPE, event.user_id, )
diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py
index 5c625ddab..beec17e38 100644
--- a/synapse/federation/replication.py
+++ b/synapse/federation/replication.py
@@ -267,8 +267,6 @@ class ReplicationLayer(object):
         transaction = Transaction(**transaction_data)
 
         pdus = [Pdu(outlier=True, **p) for p in transaction.pdus]
-        for pdu in pdus:
-            yield self._handle_new_pdu(destination, pdu)
 
         defer.returnValue(pdus)
 
@@ -452,15 +450,12 @@ class ReplicationLayer(object):
         )
 
         logger.debug("Got content: %s", content)
+
         state = [Pdu(outlier=True, **p) for p in content.get("state", [])]
-        for pdu in state:
-            yield self._handle_new_pdu(destination, pdu)
 
         auth_chain = [
             Pdu(outlier=True, **p) for p in content.get("auth_chain", [])
         ]
-        for pdu in auth_chain:
-            yield self._handle_new_pdu(destination, pdu)
 
         defer.returnValue(state)
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index d8d5730b6..99655c8bb 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -229,12 +229,6 @@ class FederationHandler(BaseHandler):
     @log_function
     @defer.inlineCallbacks
     def do_invite_join(self, target_host, room_id, joinee, content, snapshot):
-        hosts = yield self.store.get_joined_hosts_for_room(room_id)
-        if self.hs.hostname in hosts:
-            # We are already in the room.
-            logger.debug("We're already in the room apparently")
-            defer.returnValue(False)
-
         pdu = yield self.replication_layer.make_join(
             target_host,
             room_id,
@@ -268,7 +262,7 @@ class FederationHandler(BaseHandler):
 
             logger.debug("do_invite_join state: %s", state)
 
-            is_new_state = yield self.state_handler.annotate_event_with_state(
+            yield self.state_handler.annotate_event_with_state(
                 event,
                 old_state=state
             )
@@ -296,13 +290,13 @@ class FederationHandler(BaseHandler):
                 yield self.store.persist_event(
                     e,
                     backfilled=False,
-                    is_new_state=False
+                    is_new_state=True
                 )
 
             yield self.store.persist_event(
                 event,
                 backfilled=False,
-                is_new_state=is_new_state
+                is_new_state=True
             )
         finally:
             room_queue = self.room_queues[room_id]
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 3642fcfc6..825957f72 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -24,6 +24,7 @@ from synapse.api.events.room import (
     RoomTopicEvent, RoomNameEvent, RoomJoinRulesEvent,
 )
 from synapse.util import stringutils
+from synapse.util.async import run_on_reactor
 from ._base import BaseHandler
 
 import logging
@@ -432,9 +433,12 @@ class RoomMemberHandler(BaseHandler):
         # that we are allowed to join when we decide whether or not we
         # need to do the invite/join dance.
 
-        hosts = yield self.store.get_joined_hosts_for_room(room_id)
+        is_host_in_room = yield self.auth.check_host_in_room(
+            event.room_id,
+            self.hs.hostname
+        )
 
-        if self.hs.hostname in hosts:
+        if is_host_in_room:
             should_do_dance = False
         elif room_host:
             should_do_dance = True
@@ -517,6 +521,8 @@ class RoomMemberHandler(BaseHandler):
     @defer.inlineCallbacks
     def _do_local_membership_update(self, event, membership, snapshot,
                                     do_auth):
+        yield run_on_reactor()
+
         # If we're inviting someone, then we should also send it to that
         # HS.
         target_user_id = event.state_key
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 4034437f6..72290eb5a 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -186,6 +186,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 "events",
                 vals,
                 or_replace=(not outlier),
+                or_ignore=bool(outlier),
             )
         except:
             logger.warn(
@@ -217,7 +218,12 @@ class DataStore(RoomMemberStore, RoomStore,
             if hasattr(event, "replaces_state"):
                 vals["prev_state"] = event.replaces_state
 
-            self._simple_insert_txn(txn, "state_events", vals)
+            self._simple_insert_txn(
+                txn,
+                "state_events",
+                vals,
+                or_replace=True,
+            )
 
             self._simple_insert_txn(
                 txn,
@@ -227,7 +233,8 @@ class DataStore(RoomMemberStore, RoomStore,
                     "room_id": event.room_id,
                     "type": event.type,
                     "state_key": event.state_key,
-                }
+                },
+                or_replace=True,
             )
 
             for e_id, h in event.prev_state:
@@ -252,7 +259,8 @@ class DataStore(RoomMemberStore, RoomStore,
                         "room_id": event.room_id,
                         "type": event.type,
                         "state_key": event.state_key,
-                    }
+                    },
+                    or_replace=True,
                 )
 
                 for prev_state_id, _ in event.prev_state:
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 68975969f..2f3a70b4e 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -70,7 +70,8 @@ class StateStore(SQLBaseStore):
                 values={
                     "room_id": event.room_id,
                     "event_id": event.event_id,
-                }
+                },
+                or_ignore=True,
             )
 
             for state in event.state_events.values():
@@ -83,7 +84,8 @@ class StateStore(SQLBaseStore):
                         "type": state.type,
                         "state_key": state.state_key,
                         "event_id": state.event_id,
-                    }
+                    },
+                    or_ignore=True,
                 )
 
         self._simple_insert_txn(
@@ -92,5 +94,6 @@ class StateStore(SQLBaseStore):
             values={
                 "state_group": state_group,
                 "event_id": event.event_id,
-            }
+            },
+            or_replace=True,
         )
diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py
index ee264e5ee..cbe591ab9 100644
--- a/tests/handlers/test_room.py
+++ b/tests/handlers/test_room.py
@@ -44,7 +44,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
             ]),
             datastore=NonCallableMock(spec_set=[
                 "persist_event",
-                "get_joined_hosts_for_room",
                 "get_room_member",
                 "get_room",
                 "store_room",
@@ -58,9 +57,14 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
                 "profile_handler",
                 "federation_handler",
             ]),
-            auth=NonCallableMock(spec_set=["check", "add_auth_events"]),
+            auth=NonCallableMock(spec_set=[
+                "check",
+                "add_auth_events",
+                "check_host_in_room",
+            ]),
             state_handler=NonCallableMock(spec_set=[
                 "annotate_event_with_state",
+                "get_current_state",
             ]),
             config=self.mock_config,
         )
@@ -76,6 +80,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
         self.notifier = hs.get_notifier()
         self.state_handler = hs.get_state_handler()
         self.distributor = hs.get_distributor()
+        self.auth = hs.get_auth()
         self.hs = hs
 
         self.handlers.federation_handler = self.federation
@@ -108,11 +113,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
             content=content,
         )
 
-        joined = ["red", "green"]
-
-        self.datastore.get_joined_hosts_for_room.return_value = (
-            defer.succeed(joined)
-        )
+        self.auth.check_host_in_room.return_value = defer.succeed(True)
 
         store_id = "store_id_fooo"
         self.datastore.persist_event.return_value = defer.succeed(store_id)
@@ -164,12 +165,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
             room_id=room_id,
         )
 
-        joined = ["red", "green"]
-
-        def get_joined(*args):
-            return defer.succeed(joined)
-
-        self.datastore.get_joined_hosts_for_room.side_effect = get_joined
+        self.auth.check_host_in_room.return_value = defer.succeed(True)
 
         store_id = "store_id_fooo"
         self.datastore.persist_event.return_value = defer.succeed(store_id)

From 3db0efa69f5a12c40ad58c3a6bb45424a81a0954 Mon Sep 17 00:00:00 2001
From: Erik Johnston 
Date: Wed, 12 Nov 2014 11:27:02 +0000
Subject: [PATCH 226/265] Fix pyflake warnings and add a FIXME comment to deal
 with auth_chains received when joining

---
 synapse/federation/replication.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py
index beec17e38..a07e30784 100644
--- a/synapse/federation/replication.py
+++ b/synapse/federation/replication.py
@@ -453,9 +453,12 @@ class ReplicationLayer(object):
 
         state = [Pdu(outlier=True, **p) for p in content.get("state", [])]
 
-        auth_chain = [
-            Pdu(outlier=True, **p) for p in content.get("auth_chain", [])
-        ]
+        # FIXME: We probably want to do something with the auth_chain given
+        # to us
+
+        # auth_chain = [
+        #    Pdu(outlier=True, **p) for p in content.get("auth_chain", [])
+        # ]
 
         defer.returnValue(state)
 

From e7ccd26c707a9786a349a2d5a6ad1b87206c9a82 Mon Sep 17 00:00:00 2001
From: Kegan Dougal 
Date: Wed, 12 Nov 2014 11:40:19 +0000
Subject: [PATCH 227/265] SYWEB-140: Redact button layout.

---
 syweb/webclient/app.css                 | 4 ++++
 syweb/webclient/room/room-controller.js | 1 +
 syweb/webclient/room/room.html          | 8 ++++++--
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css
index 23ec42f12..403d615bf 100755
--- a/syweb/webclient/app.css
+++ b/syweb/webclient/app.css
@@ -458,6 +458,10 @@ textarea, input {
     font-family: monospace;
 }
 
+.redact-button {
+    float: left
+}
+
 .room-info-textarea-content {
     height: auto;
     width: 100%;
diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js
index 2a79fbfbb..6928754c5 100644
--- a/syweb/webclient/room/room-controller.js
+++ b/syweb/webclient/room/room-controller.js
@@ -1087,6 +1087,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a
         console.log("Redact event >> " + JSON.stringify($scope.event_selected));
         $modalInstance.close("redact");
     };
+    $scope.dismiss = $modalInstance.dismiss;
 })
 .controller('RoomInfoController', function($scope, $modalInstance, $filter, matrixService) {
     console.log("Displaying room info.");
diff --git a/syweb/webclient/room/room.html b/syweb/webclient/room/room.html
index 430a37afd..1f1cd9bae 100644
--- a/syweb/webclient/room/room.html
+++ b/syweb/webclient/room/room.html
@@ -5,11 +5,15 @@
             
 {{event_selected | json}} 

         
         

-                    
+            
+
+            
         

     
 

From 78ff63a9c7eb6cc9adb485c956cd31de69480733 Mon Sep 17 00:00:00 2001
From: Kegan Dougal 
Date: Wed, 12 Nov 2014 11:49:27 +0000
Subject: [PATCH 228/265] Remove getRoomAliasAndDisplayName: room name logic is
 in mRoomName filter, and this method was only used for /publicRooms requests.

---
 .../components/matrix/model-service.js        | 37 -------------------
 syweb/webclient/home/home-controller.js       | 13 +++++--
 2 files changed, 10 insertions(+), 40 deletions(-)

diff --git a/syweb/webclient/components/matrix/model-service.js b/syweb/webclient/components/matrix/model-service.js
index 578804c99..6e6f74159 100644
--- a/syweb/webclient/components/matrix/model-service.js
+++ b/syweb/webclient/components/matrix/model-service.js
@@ -179,43 +179,6 @@ angular.module('modelService', [])
             return room.current_room_state.members[user_id];
         },
         
-        /**
-         * Get the room_alias & room_display_name which are computed from data 
-         * already retrieved from the server.
-         * @param {Room object} room one element of the array returned by the response
-         *  of rooms() and publicRooms()
-         * @returns {Object} {room_alias: "...", room_display_name: "..."}
-         */
-        getRoomAliasAndDisplayName: function(room) {
-            var result = {
-                room_alias: undefined,
-                room_display_name: undefined
-            };
-            var alias = this.getRoomIdToAliasMapping(room.room_id);
-            if (alias) {
-                // use the existing alias from storage
-                result.room_alias = alias;
-                result.room_display_name = alias;
-            }
-            // XXX: this only lets us learn aliases from our local HS - we should
-            // make the client stop returning this if we can trust m.room.aliases state events
-            else if (room.aliases && room.aliases[0]) {
-                // save the mapping
-                // TODO: select the smarter alias from the array
-                this.createRoomIdToAliasMapping(room.room_id, room.aliases[0]);
-                result.room_display_name = room.aliases[0];
-                result.room_alias = room.aliases[0];
-            }
-            else if (room.membership === "invite" && "inviter" in room) {
-                result.room_display_name = room.inviter + "'s room";
-            }
-            else {
-                // last resort use the room id
-                result.room_display_name = room.room_id;
-            }
-            return result;
-        },
-        
         createRoomIdToAliasMapping: function(roomId, alias) {
             setRoomIdToAliasMapping(roomId, alias);
         },
diff --git a/syweb/webclient/home/home-controller.js b/syweb/webclient/home/home-controller.js
index 467db09b3..6a3c07929 100644
--- a/syweb/webclient/home/home-controller.js
+++ b/syweb/webclient/home/home-controller.js
@@ -54,10 +54,17 @@ angular.module('HomeController', ['matrixService', 'eventHandlerService', 'Recen
                 $scope.public_rooms = response.data.chunk;
                 for (var i = 0; i < $scope.public_rooms.length; i++) {
                     var room = $scope.public_rooms[i];
-
-                    // Add room_alias & room_display_name members
-                    angular.extend(room, modelService.getRoomAliasAndDisplayName(room));
                     
+                    if (room.aliases && room.aliases.length > 0) {
+                        room.room_display_name = room.aliases[0];
+                        room.room_alias = room.aliases[0];
+                    }
+                    else if (room.name) {
+                        room.room_display_name = room.name;
+                    }
+                    else {
+                        room.room_display_name = room.room_id;
+                    }
                 }
             }
         );

From e632fcd933b7259853ba0522e052e73fd4f4e71a Mon Sep 17 00:00:00 2001
From: Kegan Dougal 
Date: Wed, 12 Nov 2014 14:31:30 +0000
Subject: [PATCH 229/265] SYWEB-57: Highlight rooms where the history has
 changed.

This highlights rooms when something has happened and you haven't viewed
it yet. It highlights entries in a slightly red background colour.
---
 syweb/webclient/app.css                       |  4 +++
 syweb/webclient/recents/recents-controller.js | 28 +++++++++++++++++--
 syweb/webclient/recents/recents.html          |  6 ++--
 syweb/webclient/room/room-controller.js       |  2 +-
 4 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css
index 403d615bf..7a4ff0467 100755
--- a/syweb/webclient/app.css
+++ b/syweb/webclient/app.css
@@ -812,6 +812,10 @@ textarea, input {
     background-color: #eee;
 }
 
+.recentsRoomUnread {
+    background-color: #fee;
+}
+
 .recentsRoomName {
     font-size: 16px;
     padding-top: 7px;
diff --git a/syweb/webclient/recents/recents-controller.js b/syweb/webclient/recents/recents-controller.js
index 6f0be18f1..062d10601 100644
--- a/syweb/webclient/recents/recents-controller.js
+++ b/syweb/webclient/recents/recents-controller.js
@@ -25,10 +25,32 @@ angular.module('RecentsController', ['matrixService', 'matrixFilter'])
     
     // retrieve all rooms and expose them
     $scope.rooms = modelService.getRooms();
+    
+    if (!$rootScope.unreadMessages) {
+        $rootScope.unreadMessages = {
+            // room_id: 
+        };
+    }
 
-    // $rootScope of the parent where the recents component is included can override this value
-    // in order to highlight a specific room in the list
-    $rootScope.recentsSelectedRoomID;
+    // $rootScope.recentsSelectedRoomID is used in the html, and is set by room-controller.
+    
+    
+    $scope.selectRoom = function(room) {
+        if ($rootScope.unreadMessages[room.room_id]) {
+            $rootScope.unreadMessages[room.room_id] = 0;
+        }
+        $rootScope.goToPage('room/' + (room.room_alias ? room.room_alias : room.room_id) );
+    };
+    
+    $scope.$on(eventHandlerService.MSG_EVENT, function(ngEvent, event, isLive) {
+        if (isLive && event.room_id !== $rootScope.recentsSelectedRoomID) {
+            if (!$rootScope.unreadMessages[event.room_id]) {
+                $rootScope.unreadMessages[event.room_id] = 0;
+            }
+            $rootScope.unreadMessages[event.room_id] += 1;
+            console.log("sel="+$rootScope.recentsSelectedRoomID+" unread:"+JSON.stringify($rootScope.unreadMessages, undefined, 2));
+        }
+    });
 
 }]);
 
diff --git a/syweb/webclient/recents/recents.html b/syweb/webclient/recents/recents.html
index 7297e2370..386edb141 100644
--- a/syweb/webclient/recents/recents.html
+++ b/syweb/webclient/recents/recents.html
@@ -1,9 +1,9 @@
- {{ state.user_id }} - - \n "; - return buffer; - } - -function program7(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n
\n
\n "; - return buffer; - } - -function program9(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.isFile, {hash:{},inverse:self.program(10, program10, data),fn:self.program(2, program2, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program10(depth0,data) { - - var buffer = "", stack1; - buffer += "\n "; - stack1 = helpers['if'].call(depth0, depth0.defaultValue, {hash:{},inverse:self.program(13, program13, data),fn:self.program(11, program11, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n "; - return buffer; - } -function program11(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - -function program13(depth0,data) { - - var buffer = "", stack1; - buffer += "\n \n "; - return buffer; - } - - buffer += "
"; - if (stack1 = helpers.name) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.name; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + "\n\n "; - stack1 = helpers['if'].call(depth0, depth0.isBody, {hash:{},inverse:self.program(9, program9, data),fn:self.program(1, program1, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n\n"; - if (stack1 = helpers.description) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.description; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += ""; - if (stack1 = helpers.paramType) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.paramType; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n \n"; - if (stack1 = helpers.name) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.name; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + "\n \n"; - if (stack2 = helpers.description) { stack2 = stack2.call(depth0, {hash:{},data:data}); } - else { stack2 = depth0.description; stack2 = typeof stack2 === functionType ? stack2.apply(depth0) : stack2; } - if(stack2 || stack2 === 0) { buffer += stack2; } - buffer += ""; - if (stack2 = helpers.paramType) { stack2 = stack2.call(depth0, {hash:{},data:data}); } - else { stack2 = depth0.paramType; stack2 = typeof stack2 === functionType ? stack2.apply(depth0) : stack2; } - if(stack2 || stack2 === 0) { buffer += stack2; } - buffer += ""; - if (stack1 = helpers.name) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.name; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + "\n "; - stack1 = helpers['if'].call(depth0, depth0.isBody, {hash:{},inverse:self.program(3, program3, data),fn:self.program(1, program1, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n"; - if (stack1 = helpers.description) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.description; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += ""; - if (stack1 = helpers.paramType) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.paramType; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += ""; - if (stack1 = helpers.name) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.name; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + "\n "; - stack1 = helpers['if'].call(depth0, depth0.isBody, {hash:{},inverse:self.program(3, program3, data),fn:self.program(1, program1, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n"; - if (stack1 = helpers.description) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.description; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += ""; - if (stack1 = helpers.paramType) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.paramType; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += ""; - if (stack1 = helpers.name) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.name; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + "\n "; - stack1 = helpers['if'].call(depth0, depth0.isBody, {hash:{},inverse:self.program(9, program9, data),fn:self.program(1, program1, data),data:data}); - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n\n "; - if (stack1 = helpers.description) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.description; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n"; - if (stack1 = helpers.paramType) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.paramType; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += ""; - if (stack1 = helpers.code) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.code; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - buffer += escapeExpression(stack1) - + ""; - if (stack1 = helpers.message) { stack1 = stack1.call(depth0, {hash:{},data:data}); } - else { stack1 = depth0.message; stack1 = typeof stack1 === functionType ? stack1.apply(depth0) : stack1; } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "
+ ng-click="selectRoom(room)" + class="recentsRoom" + ng-class="{'recentsRoomSelected': (room.room_id === recentsSelectedRoomID), 'recentsRoomUnread': (unreadMessages[room.room_id])}"> + ng-class="{'recentsRoomSelected': (room.room_id === recentsSelectedRoomID), 'recentsRoomBing': (unreadBings[room.room_id]), 'recentsRoomUnread': (unreadMessages[room.room_id])}">
{{ room.room_id | mRoomName }} diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index 6928754c5..cf6ec0893 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -804,7 +804,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a console.log("onInit3"); // Make recents highlight the current room - $scope.recentsSelectedRoomID = $scope.room_id; + $rootScope.recentsSelectedRoomID = $scope.room_id; // Init the history for this room history.init(); From 58c0ef90c9e723697076fe419fa1e2cfa1000fd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 12 Nov 2014 14:30:53 +0000 Subject: [PATCH 230/265] Add indices to state group tables --- synapse/storage/schema/state.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql index b44c56b51..44f7aafb2 100644 --- a/synapse/storage/schema/state.sql +++ b/synapse/storage/schema/state.sql @@ -30,4 +30,17 @@ CREATE TABLE IF NOT EXISTS state_groups_state( CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id TEXT NOT NULL, state_group INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); + +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state( + state_group +); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state( + room_id, type, state_key +); + +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups( + event_id ); \ No newline at end of file From e24d5cb97d076ec56aa7f52cc002efa8ac6dfc71 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 12 Nov 2014 14:33:34 +0000 Subject: [PATCH 231/265] Document StateStore and use transactions --- synapse/storage/state.py | 82 +++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 2f3a70b4e..55ea56779 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -14,43 +14,71 @@ # limitations under the License. from ._base import SQLBaseStore -from twisted.internet import defer class StateStore(SQLBaseStore): + """ Keeps track of the state at a given event. + + This is done by the concept of `state groups`. Every event is a assigned + a state group (identified by an arbitrary string), which references a + collection of state events. The current state of an event is then the + collection of state events referenced by the event's state group. + + Hence, every change in the current state causes a new state group to be + generated. However, if no change happens (e.g., if we get a message event + with only one parent it inherits the state group from its parent.) + + There are three tables: + * `state_groups`: Stores group name, first event with in the group and + room id. + * `event_to_state_groups`: Maps events to state groups. + * `state_groups_state`: Maps state group to state events. + """ - @defer.inlineCallbacks def get_state_groups(self, event_ids): - groups = set() - for event_id in event_ids: - group = yield self._simple_select_one_onecol( - table="event_to_state_groups", - keyvalues={"event_id": event_id}, - retcol="state_group", - allow_none=True, - ) - if group: - groups.add(group) + """ Get the state groups for the given list of event_ids - res = {} - for group in groups: - state_ids = yield self._simple_select_onecol( - table="state_groups_state", - keyvalues={"state_group": group}, - retcol="event_id", - ) - state = [] - for state_id in state_ids: - s = yield self.get_event( - state_id, + The return value is a dict mapping group names to lists of events. + """ + + def f(txn): + groups = set() + for event_id in event_ids: + group = self._simple_select_one_onecol_txn( + txn, + table="event_to_state_groups", + keyvalues={"event_id": event_id}, + retcol="state_group", allow_none=True, ) - if s: - state.append(s) + if group: + groups.add(group) - res[group] = state + res = {} + for group in groups: + state_ids = self._simple_select_onecol_txn( + txn, + table="state_groups_state", + keyvalues={"state_group": group}, + retcol="event_id", + ) + state = [] + for state_id in state_ids: + s = self._get_events_txn( + txn, + [state_id], + ) + if s: + state.extend(s) - defer.returnValue(res) + res[group] = state + + return res + + return self.runInteraction( + "get_state_groups", + f, + ) def store_state_groups(self, event): return self.runInteraction( From 96cd467cfa129acaf8814e574c7cc2a9a94459c5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 12 Nov 2014 14:55:57 +0000 Subject: [PATCH 232/265] Add recents-service to store shared state between recents-controllers. Remove the selectedRoomId from rootScope and instead store it in recents-service. Add a broadcast to notify listeners (recents-controller) to updates of this. --- syweb/webclient/app.js | 1 + .../components/matrix/recents-service.js | 51 +++++++++++++++++++ syweb/webclient/index.html | 1 + syweb/webclient/recents/recents-controller.js | 15 +++--- syweb/webclient/room/room-controller.js | 6 +-- 5 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 syweb/webclient/components/matrix/recents-service.js diff --git a/syweb/webclient/app.js b/syweb/webclient/app.js index 17b2bb6e8..35190a71f 100644 --- a/syweb/webclient/app.js +++ b/syweb/webclient/app.js @@ -31,6 +31,7 @@ var matrixWebClient = angular.module('matrixWebClient', [ 'eventStreamService', 'eventHandlerService', 'notificationService', + 'recentsService', 'modelService', 'infinite-scroll', 'ui.bootstrap', diff --git a/syweb/webclient/components/matrix/recents-service.js b/syweb/webclient/components/matrix/recents-service.js new file mode 100644 index 000000000..64d1ab93b --- /dev/null +++ b/syweb/webclient/components/matrix/recents-service.js @@ -0,0 +1,51 @@ +/* +Copyright 2014 OpenMarket Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +'use strict'; + +/* +This service manages shared state between *instances* of recent lists. The +recents controller will hook into this central service to get things like: +- which rooms should be highlighted +- which rooms have been binged +- which room is currently selected +- etc. +This is preferable to polluting the $rootScope with recents specific info, and +makes the dependency on this shared state *explicit*. +*/ +angular.module('recentsService', []) +.factory('recentsService', ['$rootScope', function($rootScope) { + // notify listeners when variables in the service are updated. We need to do + // this since we do not tie them to any scope. + var BROADCAST_SELECTED_ROOM_ID = "recentsService:BROADCAST_SELECTED_ROOM_ID"; + var selectedRoomId = undefined; + + + return { + BROADCAST_SELECTED_ROOM_ID: BROADCAST_SELECTED_ROOM_ID, + + getSelectedRoomId: function() { + return selectedRoomId; + }, + + setSelectedRoomId: function(room_id) { + selectedRoomId = room_id; + $rootScope.$broadcast(BROADCAST_SELECTED_ROOM_ID, room_id); + } + + }; + +}]); diff --git a/syweb/webclient/index.html b/syweb/webclient/index.html index f6487f381..4bca320e7 100644 --- a/syweb/webclient/index.html +++ b/syweb/webclient/index.html @@ -44,6 +44,7 @@ + diff --git a/syweb/webclient/recents/recents-controller.js b/syweb/webclient/recents/recents-controller.js index 062d10601..78aeca128 100644 --- a/syweb/webclient/recents/recents-controller.js +++ b/syweb/webclient/recents/recents-controller.js @@ -17,8 +17,8 @@ 'use strict'; angular.module('RecentsController', ['matrixService', 'matrixFilter']) -.controller('RecentsController', ['$rootScope', '$scope', 'eventHandlerService', 'modelService', - function($rootScope, $scope, eventHandlerService, modelService) { +.controller('RecentsController', ['$rootScope', '$scope', 'eventHandlerService', 'modelService', 'recentsService', + function($rootScope, $scope, eventHandlerService, modelService, recentsService) { // Expose the service to the view $scope.eventHandlerService = eventHandlerService; @@ -31,8 +31,11 @@ angular.module('RecentsController', ['matrixService', 'matrixFilter']) // room_id: }; } - - // $rootScope.recentsSelectedRoomID is used in the html, and is set by room-controller. + + $scope.recentsSelectedRoomID = recentsService.getSelectedRoomId(); + $scope.$on(recentsService.BROADCAST_SELECTED_ROOM_ID, function(ngEvent, room_id) { + $scope.recentsSelectedRoomID = room_id; + }); $scope.selectRoom = function(room) { @@ -43,12 +46,12 @@ angular.module('RecentsController', ['matrixService', 'matrixFilter']) }; $scope.$on(eventHandlerService.MSG_EVENT, function(ngEvent, event, isLive) { - if (isLive && event.room_id !== $rootScope.recentsSelectedRoomID) { + if (isLive && event.room_id !== $scope.recentsSelectedRoomID) { if (!$rootScope.unreadMessages[event.room_id]) { $rootScope.unreadMessages[event.room_id] = 0; } $rootScope.unreadMessages[event.room_id] += 1; - console.log("sel="+$rootScope.recentsSelectedRoomID+" unread:"+JSON.stringify($rootScope.unreadMessages, undefined, 2)); + console.log("sel="+$scope.recentsSelectedRoomID+" unread:"+JSON.stringify($rootScope.unreadMessages, undefined, 2)); } }); diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index cf6ec0893..ab1772e9e 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -15,8 +15,8 @@ limitations under the License. */ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'angular-peity']) -.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'notificationService', 'modelService', - function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, notificationService, modelService) { +.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'notificationService', 'modelService', 'recentsService', + function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, notificationService, modelService, recentsService) { 'use strict'; var MESSAGES_PER_PAGINATION = 30; var THUMBNAIL_SIZE = 320; @@ -804,7 +804,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a console.log("onInit3"); // Make recents highlight the current room - $rootScope.recentsSelectedRoomID = $scope.room_id; + recentsService.setSelectedRoomId($scope.room_id); // Init the history for this room history.init(); From 99c445a6d69f0560c08c0264d9b844a4b03599b6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 12 Nov 2014 15:11:34 +0000 Subject: [PATCH 233/265] Migrate unread messages logic to recentsService. --- .../components/matrix/recents-service.js | 31 +++++++++++++++++-- syweb/webclient/recents/recents-controller.js | 26 +++++----------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/syweb/webclient/components/matrix/recents-service.js b/syweb/webclient/components/matrix/recents-service.js index 64d1ab93b..237f1e391 100644 --- a/syweb/webclient/components/matrix/recents-service.js +++ b/syweb/webclient/components/matrix/recents-service.js @@ -27,15 +27,31 @@ This is preferable to polluting the $rootScope with recents specific info, and makes the dependency on this shared state *explicit*. */ angular.module('recentsService', []) -.factory('recentsService', ['$rootScope', function($rootScope) { +.factory('recentsService', ['$rootScope', 'eventHandlerService', function($rootScope, eventHandlerService) { // notify listeners when variables in the service are updated. We need to do // this since we do not tie them to any scope. - var BROADCAST_SELECTED_ROOM_ID = "recentsService:BROADCAST_SELECTED_ROOM_ID"; + var BROADCAST_SELECTED_ROOM_ID = "recentsService:BROADCAST_SELECTED_ROOM_ID(room_id)"; var selectedRoomId = undefined; + var BROADCAST_UNREAD_MESSAGES = "recentsService:BROADCAST_UNREAD_MESSAGES(room_id, unreadCount)"; + var unreadMessages = { + // room_id: + }; + + // listen for new unread messages + $rootScope.$on(eventHandlerService.MSG_EVENT, function(ngEvent, event, isLive) { + if (isLive && event.room_id !== selectedRoomId) { + if (!unreadMessages[event.room_id]) { + unreadMessages[event.room_id] = 0; + } + unreadMessages[event.room_id] += 1; + $rootScope.$broadcast(BROADCAST_UNREAD_MESSAGES, event.room_id, unreadMessages[event.room_id]); + } + }); return { BROADCAST_SELECTED_ROOM_ID: BROADCAST_SELECTED_ROOM_ID, + BROADCAST_UNREAD_MESSAGES: BROADCAST_UNREAD_MESSAGES, getSelectedRoomId: function() { return selectedRoomId; @@ -44,6 +60,17 @@ angular.module('recentsService', []) setSelectedRoomId: function(room_id) { selectedRoomId = room_id; $rootScope.$broadcast(BROADCAST_SELECTED_ROOM_ID, room_id); + }, + + getUnreadMessages: function() { + return unreadMessages; + }, + + markAsRead: function(room_id) { + if (unreadMessages[room_id]) { + unreadMessages[room_id] = 0; + } + $rootScope.$broadcast(BROADCAST_UNREAD_MESSAGES, room_id, 0); } }; diff --git a/syweb/webclient/recents/recents-controller.js b/syweb/webclient/recents/recents-controller.js index 78aeca128..5c2cf270c 100644 --- a/syweb/webclient/recents/recents-controller.js +++ b/syweb/webclient/recents/recents-controller.js @@ -26,34 +26,22 @@ angular.module('RecentsController', ['matrixService', 'matrixFilter']) // retrieve all rooms and expose them $scope.rooms = modelService.getRooms(); - if (!$rootScope.unreadMessages) { - $rootScope.unreadMessages = { - // room_id: - }; - } - + // track the selected room ID: the html will use this $scope.recentsSelectedRoomID = recentsService.getSelectedRoomId(); $scope.$on(recentsService.BROADCAST_SELECTED_ROOM_ID, function(ngEvent, room_id) { $scope.recentsSelectedRoomID = room_id; }); + // track the list of unread messages: the html will use this + $scope.unreadMessages = recentsService.getUnreadMessages(); + $scope.$on(recentsService.BROADCAST_UNREAD_MESSAGES, function(ngEvent, room_id, unreadCount) { + $scope.unreadMessages = recentsService.getUnreadMessages(); + }); $scope.selectRoom = function(room) { - if ($rootScope.unreadMessages[room.room_id]) { - $rootScope.unreadMessages[room.room_id] = 0; - } + recentsService.markAsRead(room.room_id); $rootScope.goToPage('room/' + (room.room_alias ? room.room_alias : room.room_id) ); }; - - $scope.$on(eventHandlerService.MSG_EVENT, function(ngEvent, event, isLive) { - if (isLive && event.room_id !== $scope.recentsSelectedRoomID) { - if (!$rootScope.unreadMessages[event.room_id]) { - $rootScope.unreadMessages[event.room_id] = 0; - } - $rootScope.unreadMessages[event.room_id] += 1; - console.log("sel="+$scope.recentsSelectedRoomID+" unread:"+JSON.stringify($rootScope.unreadMessages, undefined, 2)); - } - }); }]); From ca386a4b258a086e9c95922c90e5cf750c8b77c6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 12 Nov 2014 17:26:50 +0200 Subject: [PATCH 234/265] various fixes based on truphone feedback --- README.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index ddff42983..1ca91bb94 100644 --- a/README.rst +++ b/README.rst @@ -131,17 +131,20 @@ header files for python C extensions. Installing prerequisites on Ubuntu:: - $ sudo apt-get install build-essential python2.7-dev libffi-dev + $ sudo apt-get install build-essential python2.7-dev libffi-dev \ + python-pip python-setuptools Installing prerequisites on Mac OS X:: $ xcode-select --install -Synapse uses NaCl (http://nacl.cr.yp.to/) for encryption and digital -signatures. Unfortunately PyNACL currently has a few issues +Synapse uses NaCl (http://nacl.cr.yp.to/) for encryption and digital signatures. +Unfortunately PyNACL currently has a few issues (https://github.com/pyca/pynacl/issues/53) and (https://github.com/pyca/pynacl/issues/79) that mean it may not install -correctly. To fix try re-installing from PyPI or directly from (https://github.com/pyca/pynacl):: +correctly, causing all tests to fail with errors about missing "sodium.h". To +fix try re-installing from PyPI or directly from +(https://github.com/pyca/pynacl):: $ # Install from PyPI $ pip install --user --upgrade --force pynacl @@ -158,9 +161,19 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into ``$HOME/.local/lib/``. +To actually run your new homeserver, , ``synctl start`` and follow the instructions. + Homeserver Development ====================== +To check out a homeserver for development, clone the git repo into a working +directory of your choice: + + $ git clone https://github.com/matrix-org/synapse.git + $ cd synapse + The homeserver has a number of external dependencies, that are easiest to install by making setup.py do so, in --user mode:: From 960b28c90a6ca8b9c61a3b376b4f9223203f4f06 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 12 Nov 2014 15:31:06 +0000 Subject: [PATCH 235/265] SYWEB-57: Highlight rooms which have had their bingers go off in blue. Priority is the same as xchat so selected > blue > red. --- syweb/webclient/app.css | 4 ++++ .../matrix/event-handler-service.js | 24 ++++++++++++++----- .../components/matrix/recents-service.js | 21 ++++++++++++++++ syweb/webclient/recents/recents-controller.js | 6 +++++ syweb/webclient/recents/recents.html | 2 +- 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css index 7a4ff0467..648388cdb 100755 --- a/syweb/webclient/app.css +++ b/syweb/webclient/app.css @@ -816,6 +816,10 @@ textarea, input { background-color: #fee; } +.recentsRoomBing { + background-color: #eef; +} + .recentsRoomName { font-size: 16px; padding-top: 7px; diff --git a/syweb/webclient/components/matrix/event-handler-service.js b/syweb/webclient/components/matrix/event-handler-service.js index 7b2a75507..6645d2037 100644 --- a/syweb/webclient/components/matrix/event-handler-service.js +++ b/syweb/webclient/components/matrix/event-handler-service.js @@ -95,14 +95,22 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati modelService.createRoomIdToAliasMapping(event.room_id, event.content.aliases[0]); }; + var containsBingWord = function(event) { + if (!event.content || !event.content.body) { + return false; + } + + return notificationService.containsBingWord( + matrixService.config().user_id, + matrixService.config().display_name, + matrixService.config().bingWords, + event.content.body + ); + }; + var displayNotification = function(event) { if (window.Notification && event.user_id != matrixService.config().user_id) { - var shouldBing = notificationService.containsBingWord( - matrixService.config().user_id, - matrixService.config().display_name, - matrixService.config().bingWords, - event.content.body - ); + var shouldBing = containsBingWord(event); // Ideally we would notify only when the window is hidden (i.e. document.hidden = true). // @@ -529,6 +537,10 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati resetRoomMessages(room_id); }, + eventContainsBingWord: function(event) { + return containsBingWord(event); + }, + /** * Return the last message event of a room * @param {String} room_id the room id diff --git a/syweb/webclient/components/matrix/recents-service.js b/syweb/webclient/components/matrix/recents-service.js index 237f1e391..3d82b8218 100644 --- a/syweb/webclient/components/matrix/recents-service.js +++ b/syweb/webclient/components/matrix/recents-service.js @@ -38,9 +38,22 @@ angular.module('recentsService', []) // room_id: }; + var BROADCAST_UNREAD_BING_MESSAGES = "recentsService:BROADCAST_UNREAD_BING_MESSAGES(room_id, event)"; + var unreadBingMessages = { + // room_id: bingEvent + }; + // listen for new unread messages $rootScope.$on(eventHandlerService.MSG_EVENT, function(ngEvent, event, isLive) { if (isLive && event.room_id !== selectedRoomId) { + if (eventHandlerService.eventContainsBingWord(event)) { + if (!unreadBingMessages[event.room_id]) { + unreadBingMessages[event.room_id] = {}; + } + unreadBingMessages[event.room_id] = event; + $rootScope.$broadcast(BROADCAST_UNREAD_BING_MESSAGES, event.room_id, event); + } + if (!unreadMessages[event.room_id]) { unreadMessages[event.room_id] = 0; } @@ -66,11 +79,19 @@ angular.module('recentsService', []) return unreadMessages; }, + getUnreadBingMessages: function() { + return unreadBingMessages; + }, + markAsRead: function(room_id) { if (unreadMessages[room_id]) { unreadMessages[room_id] = 0; } + if (unreadBingMessages[room_id]) { + unreadBingMessages[room_id] = undefined; + } $rootScope.$broadcast(BROADCAST_UNREAD_MESSAGES, room_id, 0); + $rootScope.$broadcast(BROADCAST_UNREAD_BING_MESSAGES, room_id, undefined); } }; diff --git a/syweb/webclient/recents/recents-controller.js b/syweb/webclient/recents/recents-controller.js index 5c2cf270c..41720d4cb 100644 --- a/syweb/webclient/recents/recents-controller.js +++ b/syweb/webclient/recents/recents-controller.js @@ -38,6 +38,12 @@ angular.module('RecentsController', ['matrixService', 'matrixFilter']) $scope.unreadMessages = recentsService.getUnreadMessages(); }); + // track the list of unread BING messages: the html will use this + $scope.unreadBings = recentsService.getUnreadBingMessages(); + $scope.$on(recentsService.BROADCAST_UNREAD_BING_MESSAGES, function(ngEvent, room_id, event) { + $scope.unreadBings = recentsService.getUnreadBingMessages(); + }); + $scope.selectRoom = function(room) { recentsService.markAsRead(room.room_id); $rootScope.goToPage('room/' + (room.room_alias ? room.room_alias : room.room_id) ); diff --git a/syweb/webclient/recents/recents.html b/syweb/webclient/recents/recents.html index 386edb141..0b3a77ca1 100644 --- a/syweb/webclient/recents/recents.html +++ b/syweb/webclient/recents/recents.html @@ -3,7 +3,7 @@
{{ room.room_id | mRoomName }} From 9412110c825915f5323d7d266c488c006d26afe3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 12 Nov 2014 15:35:46 +0000 Subject: [PATCH 236/265] comment typo --- syweb/webclient/components/matrix/matrix-call.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syweb/webclient/components/matrix/matrix-call.js b/syweb/webclient/components/matrix/matrix-call.js index b560cf7da..7687cb697 100644 --- a/syweb/webclient/components/matrix/matrix-call.js +++ b/syweb/webclient/components/matrix/matrix-call.js @@ -82,7 +82,7 @@ angular.module('MatrixCall', []) }); } - // FIXME: we should prevent any class from being placed or accepted before this has finished + // FIXME: we should prevent any calls from being placed or accepted before this has finished MatrixCall.getTurnServer(); MatrixCall.CALL_TIMEOUT = 60000; From 92ea45070ce79122d2d3ddc80c868e5ad20f6656 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 12 Nov 2014 15:58:22 +0000 Subject: [PATCH 237/265] Add recentsService unit tests. --- .../test/unit/recents-service.spec.js | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 syweb/webclient/test/unit/recents-service.spec.js diff --git a/syweb/webclient/test/unit/recents-service.spec.js b/syweb/webclient/test/unit/recents-service.spec.js new file mode 100644 index 000000000..6aba140ef --- /dev/null +++ b/syweb/webclient/test/unit/recents-service.spec.js @@ -0,0 +1,114 @@ +describe('RecentsService', function() { + var scope; + var MSG_EVENT = "__test__"; + + var testEventContainsBingWord, testIsLive, testEvent; + + var eventHandlerService = { + MSG_EVENT: MSG_EVENT, + eventContainsBingWord: function(event) { + return testEventContainsBingWord; + } + }; + + // setup the service and mocked dependencies + beforeEach(function() { + + // set default mock values + testEventContainsBingWord = false; + testIsLive = true; + testEvent = { + content: { + body: "Hello world", + msgtype: "m.text" + }, + user_id: "@alfred:localhost", + room_id: "!fl1bb13:localhost", + event_id: "fwuegfw@localhost" + } + + // mocked dependencies + module(function ($provide) { + $provide.value('eventHandlerService', eventHandlerService); + }); + + // tested service + module('recentsService'); + }); + + beforeEach(inject(function($rootScope) { + scope = $rootScope; + })); + + it('should start with no unread messages.', inject( + function(recentsService) { + expect(recentsService.getUnreadMessages()).toEqual({}); + expect(recentsService.getUnreadBingMessages()).toEqual({}); + })); + + it('should NOT add an unread message to the room currently selected.', inject( + function(recentsService) { + recentsService.setSelectedRoomId(testEvent.room_id); + scope.$broadcast(MSG_EVENT, testEvent, testIsLive); + expect(recentsService.getUnreadMessages()).toEqual({}); + expect(recentsService.getUnreadBingMessages()).toEqual({}); + })); + + it('should add an unread message to the room NOT currently selected.', inject( + function(recentsService) { + recentsService.setSelectedRoomId("!someotherroomid:localhost"); + scope.$broadcast(MSG_EVENT, testEvent, testIsLive); + + var unread = {}; + unread[testEvent.room_id] = 1; + expect(recentsService.getUnreadMessages()).toEqual(unread); + })); + + it('should add an unread message and an unread bing message if a message contains a bing word.', inject( + function(recentsService) { + recentsService.setSelectedRoomId("!someotherroomid:localhost"); + testEventContainsBingWord = true; + scope.$broadcast(MSG_EVENT, testEvent, testIsLive); + + var unread = {}; + unread[testEvent.room_id] = 1; + expect(recentsService.getUnreadMessages()).toEqual(unread); + + var bing = {}; + bing[testEvent.room_id] = testEvent; + expect(recentsService.getUnreadBingMessages()).toEqual(bing); + })); + + it('should clear both unread and unread bing messages when markAsRead is called.', inject( + function(recentsService) { + recentsService.setSelectedRoomId("!someotherroomid:localhost"); + testEventContainsBingWord = true; + scope.$broadcast(MSG_EVENT, testEvent, testIsLive); + + var unread = {}; + unread[testEvent.room_id] = 1; + expect(recentsService.getUnreadMessages()).toEqual(unread); + + var bing = {}; + bing[testEvent.room_id] = testEvent; + expect(recentsService.getUnreadBingMessages()).toEqual(bing); + + recentsService.markAsRead(testEvent.room_id); + + unread[testEvent.room_id] = 0; + expect(recentsService.getUnreadMessages()).toEqual(unread); + expect(recentsService.getUnreadBingMessages()).toEqual({}); + })); + + it('should not add messages as unread if they are not live.', inject( + function(recentsService) { + testIsLive = false; + + recentsService.setSelectedRoomId("!someotherroomid:localhost"); + testEventContainsBingWord = true; + scope.$broadcast(MSG_EVENT, testEvent, testIsLive); + + expect(recentsService.getUnreadMessages()).toEqual({}); + expect(recentsService.getUnreadBingMessages()).toEqual({}); + })); +}); From 813125e122d395fb199fc6fd52aca5b252b960c0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 12 Nov 2014 16:01:01 +0000 Subject: [PATCH 238/265] Make earlier versions of jasmine happy by doing explicit object comparisons --- syweb/webclient/test/unit/recents-service.spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/syweb/webclient/test/unit/recents-service.spec.js b/syweb/webclient/test/unit/recents-service.spec.js index 6aba140ef..93d351d45 100644 --- a/syweb/webclient/test/unit/recents-service.spec.js +++ b/syweb/webclient/test/unit/recents-service.spec.js @@ -96,8 +96,9 @@ describe('RecentsService', function() { recentsService.markAsRead(testEvent.room_id); unread[testEvent.room_id] = 0; + bing[testEvent.room_id] = undefined; expect(recentsService.getUnreadMessages()).toEqual(unread); - expect(recentsService.getUnreadBingMessages()).toEqual({}); + expect(recentsService.getUnreadBingMessages()).toEqual(bing); })); it('should not add messages as unread if they are not live.', inject( From e715741abc42ed39ecaf38048f7b41e1076bfb70 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 12 Nov 2014 15:02:31 +0000 Subject: [PATCH 239/265] Update some of the docs in event_federation --- synapse/storage/event_federation.py | 31 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index a027db386..6c559f8f6 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -23,6 +23,14 @@ logger = logging.getLogger(__name__) class EventFederationStore(SQLBaseStore): + """ Responsible for storing and serving up the various graphs associated + with an event. Including the main event graph and the auth chains for an + event. + + Also has methods for getting the front (latest) and back (oldest) edges + of the event graphs. These are used to generate the parents for new events + and backfilling from another server respectively. + """ def get_auth_chain(self, event_id): return self.runInteraction( @@ -205,6 +213,8 @@ class EventFederationStore(SQLBaseStore): return results def get_min_depth(self, room_id): + """ For hte given room, get the minimum depth we have seen for it. + """ return self.runInteraction( "get_min_depth", self._get_min_depth_interaction, @@ -240,6 +250,10 @@ class EventFederationStore(SQLBaseStore): def _handle_prev_events(self, txn, outlier, event_id, prev_events, room_id): + """ + For the given event, update the event edges table and forward and + backward extremities tables. + """ for e_id, _ in prev_events: # TODO (erikj): This could be done as a bulk insert self._simple_insert_txn( @@ -267,8 +281,8 @@ class EventFederationStore(SQLBaseStore): } ) - # We only insert as a forward extremity the new pdu if there are - # no other pdus that reference it as a prev pdu + # We only insert as a forward extremity the new event if there are + # no other events that reference it as a prev event query = ( "INSERT OR IGNORE INTO %(table)s (event_id, room_id) " "SELECT ?, ? WHERE NOT EXISTS (" @@ -284,7 +298,7 @@ class EventFederationStore(SQLBaseStore): txn.execute(query, (event_id, room_id, event_id)) - # Insert all the prev_pdus as a backwards thing, they'll get + # Insert all the prev_events as a backwards thing, they'll get # deleted in a second if they're incorrect anyway. for e_id, _ in prev_events: # TODO (erikj): This could be done as a bulk insert @@ -299,7 +313,7 @@ class EventFederationStore(SQLBaseStore): ) # Also delete from the backwards extremities table all ones that - # reference pdus that we have already seen + # reference events that we have already seen query = ( "DELETE FROM event_backward_extremities WHERE EXISTS (" "SELECT 1 FROM events " @@ -311,17 +325,14 @@ class EventFederationStore(SQLBaseStore): txn.execute(query) def get_backfill_events(self, room_id, event_list, limit): - """Get a list of Events for a given topic that occured before (and - including) the pdus in pdu_list. Return a list of max size `limit`. + """Get a list of Events for a given topic that occurred before (and + including) the events in event_list. Return a list of max size `limit` Args: txn room_id (str) event_list (list) limit (int) - - Return: - list: A list of PduTuples """ return self.runInteraction( "get_backfill_events", @@ -334,7 +345,6 @@ class EventFederationStore(SQLBaseStore): room_id, repr(event_list), limit ) - # We seed the pdu_results with the things from the pdu_list. event_results = event_list front = event_list @@ -373,5 +383,4 @@ class EventFederationStore(SQLBaseStore): front = new_front event_results += new_front - # We also want to update the `prev_pdus` attributes before returning. return self._get_events_txn(txn, event_results) From b2596c660b13f52c6d9b7114e5458cc1ba055a48 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 12 Nov 2014 16:20:21 +0000 Subject: [PATCH 240/265] Add a few more comments to the federation handler --- synapse/handlers/federation.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 99655c8bb..5e096f465 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -40,6 +40,8 @@ class FederationHandler(BaseHandler): of the home server (including auth and state conflict resoultion) b) converting events that were produced by local clients that may need to be sent to remote home servers. + c) doing the necessary dances to invite remote users and join remote + rooms. """ def __init__(self, hs): @@ -102,6 +104,8 @@ class FederationHandler(BaseHandler): logger.debug("Got event: %s", event.event_id) + # If we are currently in the process of joining this room, then we + # queue up events for later processing. if event.room_id in self.room_queues: self.room_queues[event.room_id].append(pdu) return @@ -187,6 +191,8 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks def backfill(self, dest, room_id, limit): + """ Trigger a backfill request to `dest` for the given `room_id` + """ extremities = yield self.store.get_oldest_events_in_room(room_id) pdus = yield self.replication_layer.backfill( @@ -212,6 +218,10 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def send_invite(self, target_host, event): + """ Sends the invite to the remote server for signing. + + Invites must be signed by the invitee's server before distribution. + """ pdu = yield self.replication_layer.send_invite( destination=target_host, context=event.room_id, @@ -229,6 +239,17 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks def do_invite_join(self, target_host, room_id, joinee, content, snapshot): + """ Attempts to join the `joinee` to the room `room_id` via the + server `target_host`. + + This first triggers a /make_join/ request that returns a partial + event that we can fill out and sign. This is then sent to the + remote server via /send_join/ which responds with the state at that + event and the auth_chains. + + We suspend processing of any received events from this room until we + have finished processing the join. + """ pdu = yield self.replication_layer.make_join( target_host, room_id, @@ -313,6 +334,10 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function def on_make_join_request(self, context, user_id): + """ We've received a /make_join/ request, so we create a partial + join event for the room and return that. We don *not* persist or + process it until the other server has signed it and sent it back. + """ event = self.event_factory.create_event( etype=RoomMemberEvent.TYPE, content={"membership": Membership.JOIN}, @@ -335,6 +360,9 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function def on_send_join_request(self, origin, pdu): + """ We have received a join event for a room. Fully process it and + respond with the current state and auth chains. + """ event = self.pdu_codec.event_from_pdu(pdu) event.outlier = False @@ -403,6 +431,10 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def on_invite_request(self, origin, pdu): + """ We've got an invite event. Process and persist it. Sign it. + + Respond with the now signed event. + """ event = self.pdu_codec.event_from_pdu(pdu) event.outlier = True From 3d3f692fd8a3fe07daf2f5b8811ebc00846b60c1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 12 Nov 2014 16:22:14 +0000 Subject: [PATCH 241/265] Add test coverage to the webclient. Update .gitignore --- .gitignore | 3 ++- .../components/matrix/matrix-service.js | 2 +- syweb/webclient/test/karma.conf.js | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3d14ac8c0..339a99e0d 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ graph/*.png graph/*.dot **/webclient/config.js -webclient/test/environment-protractor.js +**/webclient/test/coverage/ +**/webclient/test/environment-protractor.js uploads diff --git a/syweb/webclient/components/matrix/matrix-service.js b/syweb/webclient/components/matrix/matrix-service.js index c1264887c..cfe8691f8 100644 --- a/syweb/webclient/components/matrix/matrix-service.js +++ b/syweb/webclient/components/matrix/matrix-service.js @@ -23,7 +23,7 @@ This serves to isolate the caller from changes to the underlying url paths, as well as attach common params (e.g. access_token) to requests. */ angular.module('matrixService', []) -.factory('matrixService', ['$http', '$q', '$rootScope', function($http, $q, $rootScope) { +.factory('matrixService', ['$http', '$q', function($http, $q) { /* * Permanent storage of user information diff --git a/syweb/webclient/test/karma.conf.js b/syweb/webclient/test/karma.conf.js index 5f0642ca3..37a9eaf1c 100644 --- a/syweb/webclient/test/karma.conf.js +++ b/syweb/webclient/test/karma.conf.js @@ -52,18 +52,32 @@ module.exports = function(config) { // preprocess matching files before serving them to the browser // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor preprocessors: { + '../login/**/*.js': 'coverage', + '../room/**/*.js': 'coverage', + '../components/**/*.js': 'coverage', + '../user/**/*.js': 'coverage', + '../home/**/*.js': 'coverage', + '../recents/**/*.js': 'coverage', + '../settings/**/*.js': 'coverage', + '../app.js': 'coverage' }, // test results reporter to use // possible values: 'dots', 'progress' // available reporters: https://npmjs.org/browse/keyword/karma-reporter - reporters: ['progress', 'junit'], + reporters: ['progress', 'junit', 'coverage'], junitReporter: { outputFile: 'test-results.xml', suite: '' }, + coverageReporter: { + type: 'cobertura', + dir: 'coverage/', + file: 'coverage.xml' + }, + // web server port port: 9876, From 59cf6f5ec90ab0beac74739934b93c2c3163147e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 12 Nov 2014 16:32:17 +0000 Subject: [PATCH 242/265] Add more recents service unit tests. --- .../test/unit/recents-service.spec.js | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/syweb/webclient/test/unit/recents-service.spec.js b/syweb/webclient/test/unit/recents-service.spec.js index 93d351d45..a2f9ecbaf 100644 --- a/syweb/webclient/test/unit/recents-service.spec.js +++ b/syweb/webclient/test/unit/recents-service.spec.js @@ -112,4 +112,42 @@ describe('RecentsService', function() { expect(recentsService.getUnreadMessages()).toEqual({}); expect(recentsService.getUnreadBingMessages()).toEqual({}); })); + + it('should increment the unread message count.', inject( + function(recentsService) { + recentsService.setSelectedRoomId("!someotherroomid:localhost"); + scope.$broadcast(MSG_EVENT, testEvent, testIsLive); + + var unread = {}; + unread[testEvent.room_id] = 1; + expect(recentsService.getUnreadMessages()).toEqual(unread); + + scope.$broadcast(MSG_EVENT, testEvent, testIsLive); + + unread[testEvent.room_id] = 2; + expect(recentsService.getUnreadMessages()).toEqual(unread); + })); + + it('should set the bing event to the latest message to contain a bing word.', inject( + function(recentsService) { + recentsService.setSelectedRoomId("!someotherroomid:localhost"); + testEventContainsBingWord = true; + scope.$broadcast(MSG_EVENT, testEvent, testIsLive); + + var nextEvent = angular.copy(testEvent); + nextEvent.content.body = "Goodbye cruel world."; + nextEvent.event_id = "erfuerhfeaaaa@localhost"; + scope.$broadcast(MSG_EVENT, nextEvent, testIsLive); + + var bing = {}; + bing[testEvent.room_id] = nextEvent; + expect(recentsService.getUnreadBingMessages()).toEqual(bing); + })); + + it('should do nothing when marking an unknown room ID as read.', inject( + function(recentsService) { + recentsService.markAsRead("!someotherroomid:localhost"); + expect(recentsService.getUnreadMessages()).toEqual({}); + expect(recentsService.getUnreadBingMessages()).toEqual({}); + })); }); From f04b3d5042b85fa81efff9b561ca7af8d9709756 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 12 Nov 2014 17:02:18 +0000 Subject: [PATCH 243/265] Store all signatures on events rather than just dropping them --- synapse/storage/__init__.py | 15 +++++++------ synapse/storage/_base.py | 7 ++++-- synapse/storage/schema/event_signatures.sql | 6 +++--- synapse/storage/signatures.py | 24 +++++++++++++-------- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 72290eb5a..d8f351a67 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -279,13 +279,14 @@ class DataStore(RoomMemberStore, RoomStore, ) if hasattr(event, "signatures"): - signatures = event.signatures.get(event.origin, {}) - - for key_id, signature_base64 in signatures.items(): - signature_bytes = decode_base64(signature_base64) - self._store_event_origin_signature_txn( - txn, event.event_id, event.origin, key_id, signature_bytes, - ) + logger.debug("sigs: %s", event.signatures) + for name, sigs in event.signatures.items(): + for key_id, signature_base64 in sigs.items(): + signature_bytes = decode_base64(signature_base64) + self._store_event_signature_txn( + txn, event.event_id, name, key_id, + signature_bytes, + ) for prev_event_id, prev_hashes in event.prev_events: for alg, hash_base64 in prev_hashes.items(): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a1ee0318f..670387b04 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -470,12 +470,15 @@ class SQLBaseStore(object): select_event_sql = "SELECT * FROM events WHERE event_id = ?" for i, ev in enumerate(events): - signatures = self._get_event_origin_signatures_txn( + signatures = self._get_event_signatures_txn( txn, ev.event_id, ) ev.signatures = { - k: encode_base64(v) for k, v in signatures.items() + n: { + k: encode_base64(v) for k, v in s.items() + } + for n, s in signatures.items() } prevs = self._get_prev_events_and_state(txn, ev.event_id) diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/event_signatures.sql index 5491c7ece..4efa8a3e6 100644 --- a/synapse/storage/schema/event_signatures.sql +++ b/synapse/storage/schema/event_signatures.sql @@ -37,15 +37,15 @@ CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( ); -CREATE TABLE IF NOT EXISTS event_origin_signatures ( +CREATE TABLE IF NOT EXISTS event_signatures ( event_id TEXT, - origin TEXT, + signature_name TEXT, key_id TEXT, signature BLOB, CONSTRAINT uniqueness UNIQUE (event_id, key_id) ); -CREATE INDEX IF NOT EXISTS event_origin_signatures_id ON event_origin_signatures ( +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( event_id ); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 84a49088a..d90e08fff 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -103,24 +103,30 @@ class SignatureStore(SQLBaseStore): or_ignore=True, ) - - def _get_event_origin_signatures_txn(self, txn, event_id): + def _get_event_signatures_txn(self, txn, event_id): """Get all the signatures for a given PDU. Args: txn (cursor): event_id (str): Id for the Event. Returns: - A dict of key_id -> signature_bytes. + A dict of sig name -> dict(key_id -> signature_bytes) """ query = ( - "SELECT key_id, signature" - " FROM event_origin_signatures" + "SELECT signature_name, key_id, signature" + " FROM event_signatures" " WHERE event_id = ? " ) txn.execute(query, (event_id, )) - return dict(txn.fetchall()) + rows = txn.fetchall() - def _store_event_origin_signature_txn(self, txn, event_id, origin, key_id, + res = {} + + for name, key, sig in rows: + res.setdefault(name, {})[key] = sig + + return res + + def _store_event_signature_txn(self, txn, event_id, signature_name, key_id, signature_bytes): """Store a signature from the origin server for a PDU. Args: @@ -132,10 +138,10 @@ class SignatureStore(SQLBaseStore): """ self._simple_insert_txn( txn, - "event_origin_signatures", + "event_signatures", { "event_id": event_id, - "origin": origin, + "signature_name": signature_name, "key_id": key_id, "signature": buffer(signature_bytes), }, From f4a3b194da7e599863534fc8444a4f6fe9b6ee90 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 12 Nov 2014 17:06:03 +0000 Subject: [PATCH 244/265] Fix ability to invite users. Remove unused variables. --- syweb/webclient/room/room-controller.js | 42 ++++++++++--------------- syweb/webclient/room/room.html | 2 +- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index ab1772e9e..fa542129b 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -22,14 +22,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a var THUMBNAIL_SIZE = 320; // .html needs this - $scope.containsBingWord = function(content) { - return notificationService.containsBingWord( - matrixService.config().user_id, - matrixService.config().display_name, - matrixService.config().bingWords, - content - ); - }; + $scope.containsBingWord = eventHandlerService.eventContainsBingWord; // Room ids. Computed and resolved in onInit $scope.room_id = undefined; @@ -46,12 +39,8 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a messages_visibility: "hidden", // In order to avoid flickering when scrolling down the message table at the page opening, delay the message table display }; $scope.members = {}; - $scope.autoCompleting = false; - $scope.autoCompleteIndex = 0; - $scope.autoCompleteOriginal = ""; $scope.imageURLToSend = ""; - $scope.userIDToInvite = ""; // vars and functions for updating the name @@ -162,7 +151,6 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a $scope.$on(eventHandlerService.MSG_EVENT, function(ngEvent, event, isLive) { if (isLive && event.room_id === $scope.room_id) { - scrollToBottom(); } }); @@ -841,19 +829,6 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a } ); }; - - $scope.inviteUser = function() { - - matrixService.invite($scope.room_id, $scope.userIDToInvite).then( - function() { - console.log("Invited."); - $scope.feedback = "Invite successfully sent to " + $scope.userIDToInvite; - $scope.userIDToInvite = ""; - }, - function(reason) { - $scope.feedback = "Failure: " + reason.data.error; - }); - }; $scope.leaveRoom = function() { @@ -1091,6 +1066,21 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a }) .controller('RoomInfoController', function($scope, $modalInstance, $filter, matrixService) { console.log("Displaying room info."); + + $scope.userIDToInvite = ""; + + $scope.inviteUser = function() { + + matrixService.invite($scope.room_id, $scope.userIDToInvite).then( + function() { + console.log("Invited."); + $scope.feedback = "Invite successfully sent to " + $scope.userIDToInvite; + $scope.userIDToInvite = ""; + }, + function(reason) { + $scope.feedback = "Failure: " + reason.data.error; + }); + }; $scope.submit = function(event) { if (event.content) { diff --git a/syweb/webclient/room/room.html b/syweb/webclient/room/room.html index 1f1cd9bae..b97b839b4 100644 --- a/syweb/webclient/room/room.html +++ b/syweb/webclient/room/room.html @@ -203,7 +203,7 @@ From 2b64c573c3ad9fcfe3d884047804c5b5f1e19967 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 12 Nov 2014 17:27:41 +0000 Subject: [PATCH 245/265] Oops, change videoElement / selector in audio call too. --- syweb/webclient/room/room-controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index fa542129b..667020170 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -898,7 +898,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a call.onError = $rootScope.onCallError; call.onHangup = $rootScope.onCallHangup; // remote video element is used for playing audio in voice calls - call.remoteVideoElement = angular.element('#remoteVideo')[0]; + call.remoteVideoSelector = angular.element('#remoteVideo')[0]; call.placeVoiceCall(); $rootScope.currentCall = call; }; From 9950ce23342a754e9598419fc1b71cb1dc5c4de6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 12 Nov 2014 17:34:00 +0000 Subject: [PATCH 246/265] Detect OpenWebRTC and add workarounds, but comment out the turn server removal for now so we have a live demo of it not working. --- .../webclient/components/matrix/matrix-call.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/syweb/webclient/components/matrix/matrix-call.js b/syweb/webclient/components/matrix/matrix-call.js index 7687cb697..a1c3aaa10 100644 --- a/syweb/webclient/components/matrix/matrix-call.js +++ b/syweb/webclient/components/matrix/matrix-call.js @@ -92,7 +92,8 @@ angular.module('MatrixCall', []) var pc; if (window.mozRTCPeerConnection) { var iceServers = []; - if (MatrixCall.turnServer) { + // https://github.com/EricssonResearch/openwebrtc/issues/85 + if (MatrixCall.turnServer /*&& !this.isOpenWebRTC()*/) { if (MatrixCall.turnServer.uris) { for (var i = 0; i < MatrixCall.turnServer.uris.length; i++) { iceServers.push({ @@ -110,7 +111,8 @@ angular.module('MatrixCall', []) pc = new window.mozRTCPeerConnection({"iceServers":iceServers}); } else { var iceServers = []; - if (MatrixCall.turnServer) { + // https://github.com/EricssonResearch/openwebrtc/issues/85 + if (MatrixCall.turnServer /*&& !this.isOpenWebRTC()*/) { if (MatrixCall.turnServer.uris) { iceServers.push({ 'urls': MatrixCall.turnServer.uris, @@ -492,6 +494,8 @@ angular.module('MatrixCall', []) $timeout(function() { var vel = self.getRemoteVideoElement(); if (vel.play) vel.play(); + // OpenWebRTC does not support oniceconnectionstatechange yet + if (self.isOpenWebRTC()) self.state = 'connected'; }); } }; @@ -641,5 +645,15 @@ angular.module('MatrixCall', []) return null; }; + MatrixCall.prototype.isOpenWebRTC = function() { + var scripts = angular.element('script'); + for (var i = 0; i < scripts.length; i++) { + if (scripts[i].src.indexOf("owr.js") > -1) { + return true; + } + } + return false; + }; + return MatrixCall; }]); From c2609b239fe24a65c0193c0bf3d261486bc0f176 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 13 Nov 2014 11:59:33 +0200 Subject: [PATCH 247/265] suggest ~/.synapse --- README.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 1ca91bb94..8c4ebd52d 100644 --- a/README.rst +++ b/README.rst @@ -161,9 +161,11 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into ``$HOME/.local/lib/``. -To actually run your new homeserver, , ``synctl start`` and follow the instructions. +To actually run your new homeserver, pick a working directory for Synapse to run (e.g. ``~/.synapse``), and:: + + $ mkdir ~/.synapse + $ cd ~/.synapse + $ synctl start Homeserver Development ====================== From 0046df4b51d7c72953a09efa23988cacd0e80b3f Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 13 Nov 2014 10:19:09 +0000 Subject: [PATCH 248/265] This gives just enough space for the vertical scrollbar to be shown without adding a horizontal scrollbar. --- syweb/webclient/app.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css index 648388cdb..070e41064 100755 --- a/syweb/webclient/app.css +++ b/syweb/webclient/app.css @@ -489,7 +489,7 @@ textarea, input { #usersTableWrapper { float: right; clear: right; - width: 100px; + width: 101px; height: 100%; overflow-y: auto; } From 8ce69e802d22322386e2e17ee27a26a41933d660 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 13 Nov 2014 11:55:02 +0000 Subject: [PATCH 249/265] SYWEB-152: Migrate IRC command logic to commands-service. --- syweb/webclient/app.js | 1 + .../components/matrix/commands-service.js | 164 ++++++++++++++++ syweb/webclient/index.html | 1 + syweb/webclient/room/room-controller.js | 182 ++---------------- 4 files changed, 180 insertions(+), 168 deletions(-) create mode 100644 syweb/webclient/components/matrix/commands-service.js diff --git a/syweb/webclient/app.js b/syweb/webclient/app.js index 35190a71f..f9d5c18ec 100644 --- a/syweb/webclient/app.js +++ b/syweb/webclient/app.js @@ -33,6 +33,7 @@ var matrixWebClient = angular.module('matrixWebClient', [ 'notificationService', 'recentsService', 'modelService', + 'commandsService', 'infinite-scroll', 'ui.bootstrap', 'monospaced.elastic' diff --git a/syweb/webclient/components/matrix/commands-service.js b/syweb/webclient/components/matrix/commands-service.js new file mode 100644 index 000000000..3c516ad1e --- /dev/null +++ b/syweb/webclient/components/matrix/commands-service.js @@ -0,0 +1,164 @@ +/* +Copyright 2014 OpenMarket Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +'use strict'; + +/* +This service contains logic for parsing and performing IRC style commands. +*/ +angular.module('commandsService', []) +.factory('commandsService', ['$q', '$location', 'matrixService', 'modelService', function($q, $location, matrixService, modelService) { + + // create a rejected promise with the given message + var reject = function(msg) { + var deferred = $q.defer(); + deferred.reject({ + data: { + error: msg + } + }); + return deferred.promise; + }; + + // Change your nickname + var doNick = function(room_id, args) { + if (args) { + return matrixService.setDisplayName(args); + } + return reject("Usage: /nick "); + }; + + // Join a room + var doJoin = function(room_id, args) { + if (args) { + var matches = args.match(/^(\S+)$/); + if (matches) { + var room_alias = matches[1]; + $location.url("room/" + room_alias); + // NB: We don't need to actually do the join, since that happens + // automatically if we are not joined onto a room already when + // the page loads. + return reject("Joining "+room_alias); + } + } + return reject("Usage: /join "); + }; + + // Kick a user from the room with an optional reason + var doKick = function(room_id, args) { + if (args) { + var matches = args.match(/^(\S+?)( +(.*))?$/); + if (matches) { + return matrixService.kick(room_id, matches[1], matches[3]); + } + } + return reject("Usage: /kick []"); + }; + + // Ban a user from the room with an optional reason + var doBan = function(room_id, args) { + if (args) { + var matches = args.match(/^(\S+?)( +(.*))?$/); + if (matches) { + return matrixService.ban(room_id, matches[1], matches[3]); + } + } + return reject("Usage: /ban []"); + }; + + // Unban a user from the room + var doUnban = function(room_id, args) { + if (args) { + var matches = args.match(/^(\S+)$/); + if (matches) { + // Reset the user membership to "leave" to unban him + return matrixService.unban(room_id, matches[1]); + } + } + return reject("Usage: /unban "); + }; + + // Define the power level of a user + var doOp = function(room_id, args) { + if (args) { + var matches = args.match(/^(\S+?)( +(\d+))?$/); + var powerLevel = 50; // default power level for op + if (matches) { + var user_id = matches[1]; + if (matches.length === 4 && undefined !== matches[3]) { + powerLevel = parseInt(matches[3]); + } + if (powerLevel !== NaN) { + var powerLevelEvent = modelService.getRoom(room_id).current_room_state.state("m.room.power_levels"); + return matrixService.setUserPowerLevel(room_id, user_id, powerLevel, powerLevelEvent); + } + } + } + return reject("Usage: /op []"); + }; + + // Reset the power level of a user + var doDeop = function(room_id, args) { + if (args) { + var matches = args.match(/^(\S+)$/); + if (matches) { + var powerLevelEvent = modelService.getRoom(room_id).current_room_state.state("m.room.power_levels"); + return matrixService.setUserPowerLevel(room_id, args, undefined, powerLevelEvent); + } + } + return reject("Usage: /deop "); + }; + + + var commands = { + "nick": doNick, + "join": doJoin, + "kick": doKick, + "ban": doBan, + "unban": doUnban, + "op": doOp, + "deop": doDeop + }; + + return { + + /** + * Process the given text for commands and perform them. + * @param {String} roomId The room in which the input was performed. + * @param {String} input The raw text input by the user. + * @return {Promise} A promise of the pending command, or null if the + * input is not a command. + */ + processInput: function(roomId, input) { + // trim any trailing whitespace, as it can confuse the parser for + // IRC-style commands + input = input.replace(/\s+$/, ""); + if (input[0] === "/" && input[1] !== "/") { + var bits = input.match(/^(\S+?)( +(.*))?$/); + var cmd = bits[1].substring(1); + var args = bits[3]; + if (commands[cmd]) { + return commands[cmd](roomId, args); + } + return reject("Unrecognised IRC-style command: " + cmd); + } + return null; // not a command + } + + }; + +}]); + diff --git a/syweb/webclient/index.html b/syweb/webclient/index.html index 4bca320e7..fb3c3f528 100644 --- a/syweb/webclient/index.html +++ b/syweb/webclient/index.html @@ -45,6 +45,7 @@ + diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index 667020170..bea0db575 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -15,8 +15,8 @@ limitations under the License. */ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'angular-peity']) -.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'notificationService', 'modelService', 'recentsService', - function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, notificationService, modelService, recentsService) { +.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'notificationService', 'modelService', 'recentsService', 'commandsService', + function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, notificationService, modelService, recentsService, commandsService) { 'use strict'; var MESSAGES_PER_PAGINATION = 30; var THUMBNAIL_SIZE = 320; @@ -435,172 +435,18 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a // Store the command in the history history.push(input); - var promise; - var cmd; - var args; + var promise = commandsService.processInput($scope.room_id, input); var echo = false; + var isEmote = input.indexOf("/me ") === 0; - // Check for IRC style commands first - // trim any trailing whitespace, as it can confuse the parser for IRC-style commands - input = input.replace(/\s+$/, ""); - - if (input[0] === "/" && input[1] !== "/") { - var bits = input.match(/^(\S+?)( +(.*))?$/); - cmd = bits[1]; - args = bits[3]; - - console.log("cmd: " + cmd + ", args: " + args); - - switch (cmd) { - case "/me": - promise = matrixService.sendEmoteMessage($scope.room_id, args); - echo = true; - break; - - case "/nick": - // Change user display name - if (args) { - promise = matrixService.setDisplayName(args); - } - else { - $scope.feedback = "Usage: /nick "; - } - break; - - case "/join": - // Join a room - if (args) { - var matches = args.match(/^(\S+)$/); - if (matches) { - var room_alias = matches[1]; - if (room_alias.indexOf(':') == -1) { - // FIXME: actually track the :domain style name of our homeserver - // with or without port as is appropriate and append it at this point - } - - var room_id = modelService.getAliasToRoomIdMapping(room_alias); - console.log("joining " + room_alias + " id=" + room_id); - if ($scope.room) { // TODO actually check that you = join - // don't send a join event for a room you're already in. - $location.url("room/" + room_alias); - } - else { - promise = matrixService.joinAlias(room_alias).then( - function(response) { - // TODO: factor out the common housekeeping whenever we try to join a room or alias - matrixService.roomState(response.room_id).then( - function(response) { - eventHandlerService.handleEvents(response.data, false, true); - }, - function(error) { - $scope.feedback = "Failed to get room state for: " + response.room_id; - } - ); - $location.url("room/" + room_alias); - }, - function(error) { - $scope.feedback = "Can't join room: " + JSON.stringify(error.data); - } - ); - } - } - } - else { - $scope.feedback = "Usage: /join "; - } - break; - - case "/kick": - // Kick a user from the room with an optional reason - if (args) { - var matches = args.match(/^(\S+?)( +(.*))?$/); - if (matches) { - promise = matrixService.kick($scope.room_id, matches[1], matches[3]); - } - } - - if (!promise) { - $scope.feedback = "Usage: /kick []"; - } - break; - - case "/ban": - // Ban a user from the room with an optional reason - if (args) { - var matches = args.match(/^(\S+?)( +(.*))?$/); - if (matches) { - promise = matrixService.ban($scope.room_id, matches[1], matches[3]); - } - } - - if (!promise) { - $scope.feedback = "Usage: /ban []"; - } - break; - - case "/unban": - // Unban a user from the room - if (args) { - var matches = args.match(/^(\S+)$/); - if (matches) { - // Reset the user membership to "leave" to unban him - promise = matrixService.unban($scope.room_id, matches[1]); - } - } - - if (!promise) { - $scope.feedback = "Usage: /unban "; - } - break; - - case "/op": - // Define the power level of a user - if (args) { - var matches = args.match(/^(\S+?)( +(\d+))?$/); - var powerLevel = 50; // default power level for op - if (matches) { - var user_id = matches[1]; - if (matches.length === 4 && undefined !== matches[3]) { - powerLevel = parseInt(matches[3]); - } - if (powerLevel !== NaN) { - var powerLevelEvent = $scope.room.current_room_state.state("m.room.power_levels"); - promise = matrixService.setUserPowerLevel($scope.room_id, user_id, powerLevel, powerLevelEvent); - } - } - } - - if (!promise) { - $scope.feedback = "Usage: /op []"; - } - break; - - case "/deop": - // Reset the power level of a user - if (args) { - var matches = args.match(/^(\S+)$/); - if (matches) { - var powerLevelEvent = $scope.room.current_room_state.state("m.room.power_levels"); - promise = matrixService.setUserPowerLevel($scope.room_id, args, undefined, powerLevelEvent); - } - } - - if (!promise) { - $scope.feedback = "Usage: /deop "; - } - break; - - default: - $scope.feedback = ("Unrecognised IRC-style command: " + cmd); - break; - } - } - - // By default send this as a message unless it's an IRC-style command - if (!promise && !cmd) { - // Make the request - promise = matrixService.sendTextMessage($scope.room_id, input); + if (!promise) { // not a non-echoable command echo = true; + if (isEmote) { + promise = matrixService.sendEmoteMessage($scope.room_id, input.substring(4)); + } + else { + promise = matrixService.sendTextMessage($scope.room_id, input); + } } if (echo) { @@ -608,8 +454,8 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a // To do so, create a minimalist fake text message event and add it to the in-memory list of room messages var echoMessage = { content: { - body: (cmd === "/me" ? args : input), - msgtype: (cmd === "/me" ? "m.emote" : "m.text"), + body: (isEmote ? input.substring(4) : input), + msgtype: (isEmote ? "m.emote" : "m.text"), }, origin_server_ts: new Date().getTime(), // fake a timestamp room_id: $scope.room_id, @@ -642,7 +488,7 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a } }, function(error) { - $scope.feedback = "Request failed: " + error.data.error; + $scope.feedback = error.data.error; if (echoMessage) { // Mark the message as unsent for the rest of the page life From 51802854562351a04f8cfd248a9aefb25a5508e7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 13 Nov 2014 11:58:28 +0000 Subject: [PATCH 250/265] SYWEB-152: Unbreak /me --- syweb/webclient/room/room-controller.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index bea0db575..7094a703f 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -435,9 +435,13 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a // Store the command in the history history.push(input); - var promise = commandsService.processInput($scope.room_id, input); - var echo = false; var isEmote = input.indexOf("/me ") === 0; + var promise; + if (!isEmote) { + promise = commandsService.processInput($scope.room_id, input); + } + var echo = false; + if (!promise) { // not a non-echoable command echo = true; From 0a699df5e8fcb18098a1e53c60798de4bf51eef7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 13 Nov 2014 12:33:43 +0000 Subject: [PATCH 251/265] Wipe the selected room ID on the home screen. --- syweb/webclient/home/home-controller.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/syweb/webclient/home/home-controller.js b/syweb/webclient/home/home-controller.js index 6a3c07929..a9538a030 100644 --- a/syweb/webclient/home/home-controller.js +++ b/syweb/webclient/home/home-controller.js @@ -17,8 +17,8 @@ limitations under the License. 'use strict'; angular.module('HomeController', ['matrixService', 'eventHandlerService', 'RecentsController']) -.controller('HomeController', ['$scope', '$location', 'matrixService', 'eventHandlerService', 'modelService', - function($scope, $location, matrixService, eventHandlerService, modelService) { +.controller('HomeController', ['$scope', '$location', 'matrixService', 'eventHandlerService', 'modelService', 'recentsService', + function($scope, $location, matrixService, eventHandlerService, modelService, recentsService) { $scope.config = matrixService.config(); $scope.public_rooms = []; @@ -46,6 +46,8 @@ angular.module('HomeController', ['matrixService', 'eventHandlerService', 'Recen $scope.newChat = { user: "" }; + + recentsService.setSelectedRoomId(undefined); var refresh = function() { From f842bca471ec19fb0fcf8e71536b113983896140 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 13 Nov 2014 14:34:03 +0000 Subject: [PATCH 252/265] Kill ng-animate with fire because it's terrible (was causing the page to be very sluggish). Do the call icons in pure CSS3 and use one less image to boot (in some browsers the phone icon will be the wrong browser but they can deal). --- syweb/webclient/app-controller.js | 8 ++++---- syweb/webclient/app.css | 9 +++++++++ syweb/webclient/app.js | 1 - syweb/webclient/components/matrix/matrix-call.js | 2 +- syweb/webclient/img/red_phone.png | Bin 378 -> 0 bytes syweb/webclient/index.html | 4 +--- 6 files changed, 15 insertions(+), 9 deletions(-) delete mode 100644 syweb/webclient/img/red_phone.png diff --git a/syweb/webclient/app-controller.js b/syweb/webclient/app-controller.js index bbcf4ab5f..4d89897b1 100644 --- a/syweb/webclient/app-controller.js +++ b/syweb/webclient/app-controller.js @@ -21,8 +21,8 @@ limitations under the License. 'use strict'; angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'eventStreamService']) -.controller('MatrixWebClientController', ['$scope', '$location', '$rootScope', '$timeout', '$animate', 'matrixService', 'mPresence', 'eventStreamService', 'eventHandlerService', 'matrixPhoneService', 'modelService', - function($scope, $location, $rootScope, $timeout, $animate, matrixService, mPresence, eventStreamService, eventHandlerService, matrixPhoneService, modelService) { +.controller('MatrixWebClientController', ['$scope', '$location', '$rootScope', '$timeout', 'matrixService', 'mPresence', 'eventStreamService', 'eventHandlerService', 'matrixPhoneService', 'modelService', + function($scope, $location, $rootScope, $timeout, matrixService, mPresence, eventStreamService, eventHandlerService, matrixPhoneService, modelService) { // Check current URL to avoid to display the logout button on the login page $scope.location = $location.path(); @@ -30,8 +30,8 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even // disable nganimate for the local and remote video elements because ngAnimate appears // to be buggy and leaves animation classes on the video elements causing them to show // when they should not (their animations are pure CSS3) - $animate.enabled(false, angular.element('#localVideo')); - $animate.enabled(false, angular.element('#remoteVideo')); + //$animate.enabled(false, angular.element('#localVideo')); + //$animate.enabled(false, angular.element('#remoteVideo')); // Update the location state when the ng location changed $rootScope.$on('$routeChangeSuccess', function (event, current, previous) { diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css index 070e41064..76f3d13ac 100755 --- a/syweb/webclient/app.css +++ b/syweb/webclient/app.css @@ -66,6 +66,15 @@ textarea, input { margin-left: 4px; margin-right: 4px; margin-top: 8px; + transition: transform linear 0.5s; + transition: -webkit-transform linear 0.5s; +} + +.callIcon.ended { + transform: rotateZ(45deg); + -webkit-transform: rotateZ(45deg); + filter: hue-rotate(-90deg); + -webkit-filter: hue-rotate(-90deg); } #callEndedIcon { diff --git a/syweb/webclient/app.js b/syweb/webclient/app.js index 35190a71f..1a1be4bd0 100644 --- a/syweb/webclient/app.js +++ b/syweb/webclient/app.js @@ -16,7 +16,6 @@ limitations under the License. var matrixWebClient = angular.module('matrixWebClient', [ 'ngRoute', - 'ngAnimate', 'MatrixWebClientController', 'LoginController', 'RegisterController', diff --git a/syweb/webclient/components/matrix/matrix-call.js b/syweb/webclient/components/matrix/matrix-call.js index a1c3aaa10..56431817d 100644 --- a/syweb/webclient/components/matrix/matrix-call.js +++ b/syweb/webclient/components/matrix/matrix-call.js @@ -112,7 +112,7 @@ angular.module('MatrixCall', []) } else { var iceServers = []; // https://github.com/EricssonResearch/openwebrtc/issues/85 - if (MatrixCall.turnServer /*&& !this.isOpenWebRTC()*/) { + if (MatrixCall.turnServer && !this.isOpenWebRTC()) { if (MatrixCall.turnServer.uris) { iceServers.push({ 'urls': MatrixCall.turnServer.uris, diff --git a/syweb/webclient/img/red_phone.png b/syweb/webclient/img/red_phone.png deleted file mode 100644 index 11fc44940cb35654bc24874c4cd3c2377dbb353f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 378 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`EX7WqAsj$Z!;#Vf2?- z3=n2q{H)3b$dD{?jVKAuPb(=;EJ|f4FE7{2%*!rLPAo{(%P&fw{mw=TsOY<=i(`mI z@6<`Vy__9IT(7r895HGtOG-^mpZ&!1xo_U++9`m1-Tm}*Rr>4aN<~a!*R!X zmJ}_W>E9k`KHkl^X6iJ9lW+DfzI;ijsX)~BfAkIg^9OYzS0Bo~+TW~Z`GfJKc8`$R T(~5b(AYkxx^>bP0l+XkKHXN5X diff --git a/syweb/webclient/index.html b/syweb/webclient/index.html index 4bca320e7..22b6d50ca 100644 --- a/syweb/webclient/index.html +++ b/syweb/webclient/index.html @@ -16,7 +16,6 @@ - @@ -66,8 +65,7 @@
- - +
{{ currentCall.userProfile.displayname }}
From 11da8d0dffd330bc699b83972a78c9d20e7e5071 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 13 Nov 2014 16:34:51 +0200 Subject: [PATCH 253/265] remove nganimate dependency as it seems to feature disproportionately highly in the FF profiler, and removing it seems to have stopped my FF stalling for seconds on end --- syweb/webclient/app-controller.js | 8 +++++--- syweb/webclient/app.js | 1 - syweb/webclient/index.html | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/syweb/webclient/app-controller.js b/syweb/webclient/app-controller.js index bbcf4ab5f..0c419a546 100644 --- a/syweb/webclient/app-controller.js +++ b/syweb/webclient/app-controller.js @@ -21,18 +21,20 @@ limitations under the License. 'use strict'; angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'eventStreamService']) -.controller('MatrixWebClientController', ['$scope', '$location', '$rootScope', '$timeout', '$animate', 'matrixService', 'mPresence', 'eventStreamService', 'eventHandlerService', 'matrixPhoneService', 'modelService', - function($scope, $location, $rootScope, $timeout, $animate, matrixService, mPresence, eventStreamService, eventHandlerService, matrixPhoneService, modelService) { +.controller('MatrixWebClientController', ['$scope', '$location', '$rootScope', '$timeout', 'matrixService', 'mPresence', 'eventStreamService', 'eventHandlerService', 'matrixPhoneService', 'modelService', + function($scope, $location, $rootScope, $timeout, matrixService, mPresence, eventStreamService, eventHandlerService, matrixPhoneService, modelService) { // Check current URL to avoid to display the logout button on the login page $scope.location = $location.path(); +/* // disable nganimate for the local and remote video elements because ngAnimate appears // to be buggy and leaves animation classes on the video elements causing them to show // when they should not (their animations are pure CSS3) $animate.enabled(false, angular.element('#localVideo')); $animate.enabled(false, angular.element('#remoteVideo')); - +*/ + // Update the location state when the ng location changed $rootScope.$on('$routeChangeSuccess', function (event, current, previous) { $scope.location = $location.path(); diff --git a/syweb/webclient/app.js b/syweb/webclient/app.js index f9d5c18ec..9e5b85820 100644 --- a/syweb/webclient/app.js +++ b/syweb/webclient/app.js @@ -16,7 +16,6 @@ limitations under the License. var matrixWebClient = angular.module('matrixWebClient', [ 'ngRoute', - 'ngAnimate', 'MatrixWebClientController', 'LoginController', 'RegisterController', diff --git a/syweb/webclient/index.html b/syweb/webclient/index.html index fb3c3f528..6211ecd62 100644 --- a/syweb/webclient/index.html +++ b/syweb/webclient/index.html @@ -16,7 +16,9 @@ + From cadcc6cabed8e6c679fc5cf58bb3809f8f1465ba Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 13 Nov 2014 14:35:53 +0000 Subject: [PATCH 254/265] Add commands-service unit tests. --- .../test/unit/commands-service.spec.js | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 syweb/webclient/test/unit/commands-service.spec.js diff --git a/syweb/webclient/test/unit/commands-service.spec.js b/syweb/webclient/test/unit/commands-service.spec.js new file mode 100644 index 000000000..142044f15 --- /dev/null +++ b/syweb/webclient/test/unit/commands-service.spec.js @@ -0,0 +1,143 @@ +describe('CommandsService', function() { + var scope; + var roomId = "!dlwifhweu:localhost"; + + var testPowerLevelsEvent, testMatrixServicePromise; + + var matrixService = { // these will be spyed on by jasmine, hence stub methods + setDisplayName: function(args){}, + kick: function(args){}, + ban: function(args){}, + unban: function(args){}, + setUserPowerLevel: function(args){} + }; + + var modelService = { + getRoom: function(roomId) { + return { + room_id: roomId, + current_room_state: { + events: { + "m.room.power_levels": testPowerLevelsEvent + }, + state: function(type, key) { + return key ? this.events[type+key] : this.events[type]; + } + } + }; + } + }; + + + // helper function for asserting promise outcomes + NOTHING = "[Promise]"; + RESOLVED = "[Resolved promise]"; + REJECTED = "[Rejected promise]"; + var expectPromise = function(promise, expects) { + var value = NOTHING; + promise.then(function(result) { + value = RESOLVED; + }, function(fail) { + value = REJECTED; + }); + scope.$apply(); + expect(value).toEqual(expects); + }; + + // setup the service and mocked dependencies + beforeEach(function() { + + // set default mock values + testPowerLevelsEvent = { + content: { + default: 50 + }, + user_id: "@foo:bar", + room_id: roomId + } + + // mocked dependencies + module(function ($provide) { + $provide.value('matrixService', matrixService); + $provide.value('modelService', modelService); + }); + + // tested service + module('commandsService'); + }); + + beforeEach(inject(function($rootScope, $q) { + scope = $rootScope; + testMatrixServicePromise = $q.defer(); + })); + + it('should reject a no-arg "/nick".', inject( + function(commandsService) { + var promise = commandsService.processInput(roomId, "/nick"); + expectPromise(promise, REJECTED); + })); + + it('should be able to set a /nick with multiple words.', inject( + function(commandsService) { + spyOn(matrixService, 'setDisplayName').and.returnValue(testMatrixServicePromise); + var promise = commandsService.processInput(roomId, "/nick Bob Smith"); + expect(matrixService.setDisplayName).toHaveBeenCalledWith("Bob Smith"); + expect(promise).toBe(testMatrixServicePromise); + })); + + it('should be able to /kick a user without a reason.', inject( + function(commandsService) { + spyOn(matrixService, 'kick').and.returnValue(testMatrixServicePromise); + var promise = commandsService.processInput(roomId, "/kick @bob:matrix.org"); + expect(matrixService.kick).toHaveBeenCalledWith(roomId, "@bob:matrix.org", undefined); + expect(promise).toBe(testMatrixServicePromise); + })); + + it('should be able to /kick a user with a reason.', inject( + function(commandsService) { + spyOn(matrixService, 'kick').and.returnValue(testMatrixServicePromise); + var promise = commandsService.processInput(roomId, "/kick @bob:matrix.org he smells"); + expect(matrixService.kick).toHaveBeenCalledWith(roomId, "@bob:matrix.org", "he smells"); + expect(promise).toBe(testMatrixServicePromise); + })); + + it('should be able to /ban a user without a reason.', inject( + function(commandsService) { + spyOn(matrixService, 'ban').and.returnValue(testMatrixServicePromise); + var promise = commandsService.processInput(roomId, "/ban @bob:matrix.org"); + expect(matrixService.ban).toHaveBeenCalledWith(roomId, "@bob:matrix.org", undefined); + expect(promise).toBe(testMatrixServicePromise); + })); + + it('should be able to /ban a user with a reason.', inject( + function(commandsService) { + spyOn(matrixService, 'ban').and.returnValue(testMatrixServicePromise); + var promise = commandsService.processInput(roomId, "/ban @bob:matrix.org he smells"); + expect(matrixService.ban).toHaveBeenCalledWith(roomId, "@bob:matrix.org", "he smells"); + expect(promise).toBe(testMatrixServicePromise); + })); + + it('should be able to /unban a user.', inject( + function(commandsService) { + spyOn(matrixService, 'unban').and.returnValue(testMatrixServicePromise); + var promise = commandsService.processInput(roomId, "/unban @bob:matrix.org"); + expect(matrixService.unban).toHaveBeenCalledWith(roomId, "@bob:matrix.org"); + expect(promise).toBe(testMatrixServicePromise); + })); + + it('should be able to /op a user.', inject( + function(commandsService) { + spyOn(matrixService, 'setUserPowerLevel').and.returnValue(testMatrixServicePromise); + var promise = commandsService.processInput(roomId, "/op @bob:matrix.org 50"); + expect(matrixService.setUserPowerLevel).toHaveBeenCalledWith(roomId, "@bob:matrix.org", 50, testPowerLevelsEvent); + expect(promise).toBe(testMatrixServicePromise); + })); + + it('should be able to /deop a user.', inject( + function(commandsService) { + spyOn(matrixService, 'setUserPowerLevel').and.returnValue(testMatrixServicePromise); + var promise = commandsService.processInput(roomId, "/deop @bob:matrix.org"); + expect(matrixService.setUserPowerLevel).toHaveBeenCalledWith(roomId, "@bob:matrix.org", undefined, testPowerLevelsEvent); + expect(promise).toBe(testMatrixServicePromise); + })); +}); From e7c6d2c9d96124ce50940664bece4367dba0c972 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 13 Nov 2014 14:39:11 +0000 Subject: [PATCH 255/265] SYN-138: Rewrite synctl in python and include it in the python distribution --- setup.py | 1 + synapse/app/synctl.py | 66 +++++++++++++++++++++++++++++++++++++++++++ synctl | 35 ----------------------- 3 files changed, 67 insertions(+), 35 deletions(-) create mode 100755 synapse/app/synctl.py delete mode 100755 synctl diff --git a/setup.py b/setup.py index f5976cd76..7f46ce990 100755 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ setup( long_description=read("README.rst"), entry_points=""" [console_scripts] + synctl=synapse.app.synctl:main synapse-homeserver=synapse.app.homeserver:run """ ) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py new file mode 100755 index 000000000..e85073b06 --- /dev/null +++ b/synapse/app/synctl.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import subprocess +import signal + +SYNAPSE = ["python", "-m", "synapse.app.homeserver"] + +CONFIGFILE="homeserver.yaml" +PIDFILE="homeserver.pid" + +GREEN="\x1b[1;32m" +NORMAL="\x1b[m" + +def start(): + if not os.path.exists(CONFIGFILE): + sys.stderr.write( + "No config file found\n" + "To generate a config file, run '%s -c %s --generate-config" + " --server-name='\n" % ( + " ".join(SYNAPSE), CONFIGFILE + ) + ) + sys.exit(1) + print "Starting ...", + args = SYNAPSE + args.extend(["--daemonize", "-c", CONFIGFILE, "--pid-file", PIDFILE]) + subprocess.check_call(args) + print GREEN + "started" + NORMAL + +def stop(): + if os.path.exists(PIDFILE): + pid = int(open(PIDFILE).read()) + os.kill(pid, signal.SIGTERM) + print GREEN + "stopped" + NORMAL + +def main(): + action = sys.argv[1] if sys.argv[1:] else "usage" + if action == "start": + start() + elif action == "stop": + stop() + elif action == "restart": + start() + stop() + else: + sys.stderr.write("Usage: %s [start|stop|restart]\n" % (sys.argv[0],)) + sys.exit(1) + +if __name__=='__main__': + main() diff --git a/synctl b/synctl deleted file mode 100755 index c227a9e1e..000000000 --- a/synctl +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -SYNAPSE="python -m synapse.app.homeserver" - -CONFIGFILE="homeserver.yaml" -PIDFILE="homeserver.pid" - -GREEN=$'\e[1;32m' -NORMAL=$'\e[m' - -set -e - -case "$1" in - start) - if [ ! -f "$CONFIGFILE" ]; then - echo "No config file found" - echo "To generate a config file, run '$SYNAPSE -c $CONFIGFILE --generate-config --server-name='" - exit 1 - fi - - echo -n "Starting ..." - $SYNAPSE --daemonize -c "$CONFIGFILE" --pid-file "$PIDFILE" - echo "${GREEN}started${NORMAL}" - ;; - stop) - echo -n "Stopping ..." - test -f $PIDFILE && kill `cat $PIDFILE` && echo "${GREEN}stopped${NORMAL}" - ;; - restart) - $0 stop && $0 start - ;; - *) - echo "Usage: $0 [start|stop|restart]" >&2 - exit 1 -esac From bfe20c11c393665ce722942381b4168ece09adf1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 13 Nov 2014 14:42:08 +0000 Subject: [PATCH 256/265] remove now-unused styles --- syweb/webclient/app.css | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css index 76f3d13ac..a9e80aef9 100755 --- a/syweb/webclient/app.css +++ b/syweb/webclient/app.css @@ -77,18 +77,6 @@ textarea, input { -webkit-filter: hue-rotate(-90deg); } -#callEndedIcon { - transition:all linear 0.5s; -} - -#callEndedIcon { - transform: rotateZ(45deg); -} - -#callEndedIcon.ng-hide { - transform: rotateZ(0deg); -} - #callPeerImage { width: 32px; height: 32px; From 58ddff08817074b5ea5dee72f3b8237a5d645efb Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 13 Nov 2014 14:45:29 +0000 Subject: [PATCH 257/265] remove stray unmatched css comment --- syweb/webclient/app.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syweb/webclient/app.css b/syweb/webclient/app.css index a9e80aef9..25f7208a1 100755 --- a/syweb/webclient/app.css +++ b/syweb/webclient/app.css @@ -638,7 +638,7 @@ textarea, input { color: #ccc; height: 13px; margin-top: 4px; -*/ transition-property: opacity; + transition-property: opacity; transition-duration: 0.3s; } From d0858070703706d6938c5af9d273c90fcffd06ce Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 13 Nov 2014 15:21:42 +0000 Subject: [PATCH 258/265] Migrate random bits of desktop notification logic out of roomController and into eventHandlerService where everything else is. --- .../matrix/event-handler-service.js | 42 ++++++++++++++----- syweb/webclient/room/room-controller.js | 23 +--------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/syweb/webclient/components/matrix/event-handler-service.js b/syweb/webclient/components/matrix/event-handler-service.js index 6645d2037..e97350005 100644 --- a/syweb/webclient/components/matrix/event-handler-service.js +++ b/syweb/webclient/components/matrix/event-handler-service.js @@ -110,7 +110,33 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati var displayNotification = function(event) { if (window.Notification && event.user_id != matrixService.config().user_id) { - var shouldBing = containsBingWord(event); + var member = modelService.getMember(event.room_id, event.user_id); + var displayname = getUserDisplayName(event.room_id, event.user_id); + var message; + var shouldBing = false; + + if (event.type === "m.room.message") { + shouldBing = containsBingWord(event); + message = event.content.body; + if (event.content.msgtype === "m.emote") { + message = "* " + displayname + " " + message; + } + else if (event.content.msgtype === "m.image") { + message = displayname + " sent an image."; + } + } + else if (event.type == "m.room.member") { + // Notify when another user joins only + if (event.state_key !== matrixService.config().user_id && "join" === event.content.membership) { + member = modelService.getMember(event.room_id, event.state_key); + displayname = getUserDisplayName(event.room_id, event.state_key); + message = displayname + " joined"; + shouldBing = true; + } + else { + return; + } + } // Ideally we would notify only when the window is hidden (i.e. document.hidden = true). // @@ -133,17 +159,7 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati if (shouldBing && isIdle) { console.log("Displaying notification for "+JSON.stringify(event)); - var member = modelService.getMember(event.room_id, event.user_id); - var displayname = getUserDisplayName(event.room_id, event.user_id); - var message = event.content.body; - if (event.content.msgtype === "m.emote") { - message = "* " + displayname + " " + message; - } - else if (event.content.msgtype === "m.image") { - message = displayname + " sent an image."; - } - var roomTitle = $filter("mRoomName")(event.room_id); notificationService.showNotification( @@ -240,6 +256,10 @@ function(matrixService, $rootScope, $q, $timeout, $filter, mPresence, notificati // list. This has to be done after room state is updated. if (memberChanges) { room.addMessageEvent(event, !isLiveEvent); + + if (memberChanges === "membership" && isLiveEvent) { + displayNotification(event); + } } diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index 7094a703f..f68a2b504 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -15,8 +15,8 @@ limitations under the License. */ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'angular-peity']) -.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'notificationService', 'modelService', 'recentsService', 'commandsService', - function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, notificationService, modelService, recentsService, commandsService) { +.controller('RoomController', ['$modal', '$filter', '$scope', '$timeout', '$routeParams', '$location', '$rootScope', 'matrixService', 'mPresence', 'eventHandlerService', 'mFileUpload', 'matrixPhoneService', 'MatrixCall', 'modelService', 'recentsService', 'commandsService', + function($modal, $filter, $scope, $timeout, $routeParams, $location, $rootScope, matrixService, mPresence, eventHandlerService, mFileUpload, matrixPhoneService, MatrixCall, modelService, recentsService, commandsService) { 'use strict'; var MESSAGES_PER_PAGINATION = 30; var THUMBNAIL_SIZE = 320; @@ -185,25 +185,6 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a else { scrollToBottom(); updateMemberList(event); - - // Notify when a user joins - if ((document.hidden || matrixService.presence.unavailable === mPresence.getState()) - && event.state_key !== $scope.state.user_id && "join" === event.membership) { - var userName = event.content.displayname; - if (!userName) { - userName = event.state_key; - } - notificationService.showNotification( - userName + - " (" + $filter("mRoomName")(event.room_id) + ")", - userName + " joined", - event.content.avatar_url ? event.content.avatar_url : undefined, - function() { - console.log("notification.onclick() room=" + event.room_id); - $rootScope.goToPage('room/' + event.room_id); - } - ); - } } } }); From 8d8a133c89925086452bdec9739db589f0715363 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 13 Nov 2014 15:48:51 +0000 Subject: [PATCH 259/265] SYN-103: Remove "origin" and "destination" keys from edus --- synapse/federation/units.py | 9 ++++----- tests/federation/test_federation.py | 3 --- tests/handlers/test_presence.py | 3 --- tests/handlers/test_typing.py | 3 --- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index f4e7b62bd..70412439c 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -122,11 +122,10 @@ class Edu(JsonEncodedObject): "edu_type", ] -# TODO: SYN-103: Remove "origin" and "destination" keys. -# internal_keys = [ -# "origin", -# "destination", -# ] + internal_keys = [ + "origin", + "destination", + ] class Transaction(JsonEncodedObject): diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index eb329eec5..ad09fab39 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -218,9 +218,6 @@ class FederationTestCase(unittest.TestCase): "pdus": [], "edus": [ { - # TODO: SYN-103: Remove "origin" and "destination" - "origin": "test", - "destination": "remote", "edu_type": "m.test", "content": {"testing": "content here"}, } diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index a6af648de..cdaf93429 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -44,9 +44,6 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { - # TODO: SYN-103: Remove "origin" and "destination" keys. - "origin": origin, - "destination": destination, "edu_type": edu_type, "content": content, } diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 07acda5ee..adb514835 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -33,9 +33,6 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { - # TODO: SYN-103: Remove "origin" and "destination" keys. - "origin": origin, - "destination": destination, "edu_type": edu_type, "content": content, } From afd2e214bc6a4b9c99113f843dc79b18374aa39c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 13 Nov 2014 16:09:44 +0000 Subject: [PATCH 260/265] SYWEB-152: Move up/down history fully to a directive. Previously, there was some of it in a lovely generic directive, but the core of it was hard coded id attributes in RoomController. It's now all generic in a directive: the room history you get when you up/down arrow is determined by the value of the attribute e.g. command-history="!foo:bar" would present the history for !foo:bar. In practice, this is {{room_id}} in the html. --- syweb/webclient/room/room-controller.js | 84 +----------------- syweb/webclient/room/room-directive.js | 109 +++++++++++++++++++++--- syweb/webclient/room/room.html | 2 +- 3 files changed, 101 insertions(+), 94 deletions(-) diff --git a/syweb/webclient/room/room-controller.js b/syweb/webclient/room/room-controller.js index f68a2b504..9c1f67750 100644 --- a/syweb/webclient/room/room-controller.js +++ b/syweb/webclient/room/room-controller.js @@ -414,7 +414,8 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a scrollToBottom(true); // Store the command in the history - history.push(input); + $rootScope.$broadcast("commandHistory:BROADCAST_NEW_HISTORY_ITEM(item)", + input); var isEmote = input.indexOf("/me ") === 0; var promise; @@ -625,9 +626,6 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a // Make recents highlight the current room recentsService.setSelectedRoomId($scope.room_id); - // Init the history for this room - history.init(); - // Get the up-to-date the current member list matrixService.getMemberList($scope.room_id).then( function(response) { @@ -746,84 +744,6 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput', 'a $rootScope.currentCall = call; }; - // Manage history of typed messages - // History is saved in sessionStoratge so that it survives when the user - // navigates through the rooms and when it refreshes the page - var history = { - // The list of typed messages. Index 0 is the more recents - data: [], - - // The position in the history currently displayed - position: -1, - - // The message the user has started to type before going into the history - typingMessage: undefined, - - // Init/load data for the current room - init: function() { - var data = sessionStorage.getItem("history_" + $scope.room_id); - if (data) { - this.data = JSON.parse(data); - } - }, - - // Store a message in the history - push: function(message) { - this.data.unshift(message); - - // Update the session storage - sessionStorage.setItem("history_" + $scope.room_id, JSON.stringify(this.data)); - - // Reset history position - this.position = -1; - this.typingMessage = undefined; - }, - - // Move in the history - go: function(offset) { - - if (-1 === this.position) { - // User starts to go to into the history, save the current line - this.typingMessage = $('#mainInput').val(); - } - else { - // If the user modified this line in history, keep the change - this.data[this.position] = $('#mainInput').val(); - } - - // Bounds the new position to valid data - var newPosition = this.position + offset; - newPosition = Math.max(-1, newPosition); - newPosition = Math.min(newPosition, this.data.length - 1); - this.position = newPosition; - - if (-1 !== this.position) { - // Show the message from the history - $('#mainInput').val(this.data[this.position]); - } - else if (undefined !== this.typingMessage) { - // Go back to the message the user started to type - $('#mainInput').val(this.typingMessage); - } - } - }; - - // Make history singleton methods available from HTML - $scope.history = { - goUp: function($event) { - if ($scope.room_id) { - history.go(1); - } - $event.preventDefault(); - }, - goDown: function($event) { - if ($scope.room_id) { - history.go(-1); - } - $event.preventDefault(); - } - }; - $scope.openJson = function(content) { $scope.event_selected = angular.copy(content); diff --git a/syweb/webclient/room/room-directive.js b/syweb/webclient/room/room-directive.js index 05382cfcd..187032aa8 100644 --- a/syweb/webclient/room/room-directive.js +++ b/syweb/webclient/room/room-directive.js @@ -144,19 +144,106 @@ angular.module('RoomController') }); }; }]) +// A directive which stores text sent into it and restores it via up/down arrows .directive('commandHistory', [ function() { - return function (scope, element, attrs) { - element.bind("keydown", function (event) { - var keycodePressed = event.which; - var UP_ARROW = 38; - var DOWN_ARROW = 40; - if (keycodePressed === UP_ARROW) { - scope.history.goUp(event); + var BROADCAST_NEW_HISTORY_ITEM = "commandHistory:BROADCAST_NEW_HISTORY_ITEM(item)"; + + // Manage history of typed messages + // History is saved in sessionStorage so that it survives when the user + // navigates through the rooms and when it refreshes the page + var history = { + // The list of typed messages. Index 0 is the more recents + data: [], + + // The position in the history currently displayed + position: -1, + + element: undefined, + roomId: undefined, + + // The message the user has started to type before going into the history + typingMessage: undefined, + + // Init/load data for the current room + init: function(element, roomId) { + this.roomId = roomId; + this.element = element; + var data = sessionStorage.getItem("history_" + this.roomId); + if (data) { + this.data = JSON.parse(data); } - else if (keycodePressed === DOWN_ARROW) { - scope.history.goDown(event); - } - }); + }, + + // Store a message in the history + push: function(message) { + this.data.unshift(message); + + // Update the session storage + sessionStorage.setItem("history_" + this.roomId, JSON.stringify(this.data)); + + // Reset history position + this.position = -1; + this.typingMessage = undefined; + }, + + // Move in the history + go: function(offset) { + + if (-1 === this.position) { + // User starts to go to into the history, save the current line + this.typingMessage = this.element.val(); + } + else { + // If the user modified this line in history, keep the change + this.data[this.position] = this.element.val(); + } + + // Bounds the new position to valid data + var newPosition = this.position + offset; + newPosition = Math.max(-1, newPosition); + newPosition = Math.min(newPosition, this.data.length - 1); + this.position = newPosition; + + if (-1 !== this.position) { + // Show the message from the history + this.element.val(this.data[this.position]); + } + else if (undefined !== this.typingMessage) { + // Go back to the message the user started to type + this.element.val(this.typingMessage); + } + } + }; + + return { + restrict: "AE", + scope: { + roomId: "=commandHistory" + }, + link: function (scope, element, attrs) { + element.bind("keydown", function (event) { + var keycodePressed = event.which; + var UP_ARROW = 38; + var DOWN_ARROW = 40; + if (scope.roomId) { + if (keycodePressed === UP_ARROW) { + history.go(1); + event.preventDefault(); + } + else if (keycodePressed === DOWN_ARROW) { + history.go(-1); + event.preventDefault(); + } + } + }); + + scope.$on(BROADCAST_NEW_HISTORY_ITEM, function(ngEvent, item) { + history.push(item); + }); + + history.init(element, scope.roomId); + }, + } }]) diff --git a/syweb/webclient/room/room.html b/syweb/webclient/room/room.html index b97b839b4..17565f879 100644 --- a/syweb/webclient/room/room.html +++ b/syweb/webclient/room/room.html @@ -251,7 +251,7 @@