From 7fb93f2a47ab8abc44a9b6a94171950a40f6ba33 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 13 Aug 2014 17:43:10 +0100 Subject: [PATCH 01/17] Add a HomeServer.parse_roomalias() to avoid having to RoomAlias.from_sring(..., hs=hs) - similar to parse_userid() --- synapse/rest/directory.py | 13 +++++-------- synapse/rest/room.py | 6 +----- synapse/server.py | 7 ++++++- tests/test_types.py | 6 ++++++ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 31fd26e84..362f76c6c 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -16,7 +16,6 @@ from twisted.internet import defer -from synapse.types import RoomAlias, RoomID from base import RestServlet, client_path_pattern import json @@ -39,12 +38,11 @@ class ClientDirectoryServer(RestServlet): # TODO(erikj): Handle request local_only = "local_only" in request.args - room_alias = urllib.unquote(room_alias) - room_alias_obj = RoomAlias.from_string(room_alias, self.hs) + room_alias = self.hs.parse_roomalias(urllib.unquote(room_alias)) dir_handler = self.handlers.directory_handler res = yield dir_handler.get_association( - room_alias_obj, + room_alias, local_only=local_only ) @@ -57,10 +55,9 @@ class ClientDirectoryServer(RestServlet): logger.debug("Got content: %s", content) - room_alias = urllib.unquote(room_alias) - room_alias_obj = RoomAlias.from_string(room_alias, self.hs) + room_alias = self.hs.parse_roomalias(urllib.unquote(room_alias)) - logger.debug("Got room name: %s", room_alias_obj.to_string()) + logger.debug("Got room name: %s", room_alias.to_string()) room_id = content["room_id"] servers = content["servers"] @@ -75,7 +72,7 @@ class ClientDirectoryServer(RestServlet): try: yield dir_handler.create_association( - room_alias_obj, room_id, servers + room_alias, room_id, servers ) except: logger.exception("Failed to create association") diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 228bc9623..1fc0c996b 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -22,7 +22,6 @@ from synapse.api.events.room import (RoomTopicEvent, MessageEvent, RoomMemberEvent, FeedbackEvent) from synapse.api.constants import Feedback, Membership from synapse.api.streams import PaginationConfig -from synapse.types import RoomAlias import json import logging @@ -150,10 +149,7 @@ class JoinRoomAliasServlet(RestServlet): logger.debug("room_alias: %s", room_alias) - room_alias = RoomAlias.from_string( - urllib.unquote(room_alias), - self.hs - ) + room_alias = self.hs.parse_roomalias(urllib.unquote(room_alias)) handler = self.handlers.room_member_handler ret_dict = yield handler.join_room_alias(user, room_alias) diff --git a/synapse/server.py b/synapse/server.py index 0211972d0..96830a88b 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -28,7 +28,7 @@ from synapse.handlers import Handlers from synapse.rest import RestServletFactory from synapse.state import StateHandler from synapse.storage import DataStore -from synapse.types import UserID +from synapse.types import UserID, RoomAlias from synapse.util import Clock from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager @@ -120,6 +120,11 @@ class BaseHomeServer(object): object.""" return UserID.from_string(s, hs=self) + def parse_roomalias(self, s): + """Parse the string given by 's' as a Room Alias and return a RoomAlias + object.""" + return RoomAlias.from_string(s, hs=self) + # Build magic accessors for every dependency for depname in BaseHomeServer.DEPENDENCIES: BaseHomeServer._make_dependency_method(depname) diff --git a/tests/test_types.py b/tests/test_types.py index 522d52363..d2ccbcfa5 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -62,3 +62,9 @@ class RoomAliasTestCase(unittest.TestCase): room = RoomAlias("channel", "my.domain", True) self.assertEquals(room.to_string(), "#channel:my.domain") + + def test_via_homeserver(self): + room = mock_homeserver.parse_roomalias("#elsewhere:my.domain") + + self.assertEquals("elsewhere", room.localpart) + self.assertEquals("my.domain", room.domain) From 3a1cfe18cf07d463446f4fc9bed890a8a6100826 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 13 Aug 2014 18:03:37 +0100 Subject: [PATCH 02/17] Implement directory service federation by Federation Queries; avoid local_only hack; add unit tests --- synapse/handlers/directory.py | 48 +++++++------ synapse/rest/directory.py | 8 +-- tests/handlers/test_directory.py | 112 +++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 27 deletions(-) create mode 100644 tests/handlers/test_directory.py diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 3cc634890..df98e39f6 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -35,9 +35,11 @@ class DirectoryHandler(BaseHandler): def __init__(self, hs): super(DirectoryHandler, self).__init__(hs) - self.hs = hs - self.http_client = hs.get_http_client() - self.clock = hs.get_clock() + + self.federation = hs.get_replication_layer() + self.federation.register_query_handler( + "directory", self.on_directory_query + ) @defer.inlineCallbacks def create_association(self, room_alias, room_id, servers): @@ -58,9 +60,7 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def get_association(self, room_alias, local_only=False): - # TODO(erikj): Do auth - + def get_association(self, room_alias): room_id = None if room_alias.is_mine: result = yield self.store.get_association_from_room_alias( @@ -70,22 +70,13 @@ class DirectoryHandler(BaseHandler): if result: room_id = result.room_id servers = result.servers - elif not local_only: - path = "%s/ds/room/%s?local_only=1" % ( - PREFIX, - urllib.quote(room_alias.to_string()) + else: + result = yield self.federation.make_query( + destination=room_alias.domain, + query_type="directory", + args={"room_alias": room_alias.to_string()}, ) - result = None - try: - result = yield self.http_client.get_json( - destination=room_alias.domain, - path=path, - ) - except: - # TODO(erikj): Handle this better? - logger.exception("Failed to get remote room alias") - if result and "room_id" in result and "servers" in result: room_id = result["room_id"] servers = result["servers"] @@ -99,3 +90,20 @@ class DirectoryHandler(BaseHandler): "servers": servers, }) return + + @defer.inlineCallbacks + def on_directory_query(self, args): + room_alias = self.hs.parse_roomalias(args["room_alias"]) + if not room_alias.is_mine: + raise SynapseError( + 400, "Room Alias is not hosted on this Home Server" + ) + + result = yield self.store.get_association_from_room_alias( + room_alias + ) + + defer.returnValue({ + "room_id": result.room_id, + "servers": result.servers, + }) diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 362f76c6c..be9a3f5f9 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -35,16 +35,10 @@ class ClientDirectoryServer(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_alias): - # TODO(erikj): Handle request - local_only = "local_only" in request.args - room_alias = self.hs.parse_roomalias(urllib.unquote(room_alias)) dir_handler = self.handlers.directory_handler - res = yield dir_handler.get_association( - room_alias, - local_only=local_only - ) + res = yield dir_handler.get_association(room_alias) defer.returnValue((200, res)) diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py new file mode 100644 index 000000000..0ace2d0c9 --- /dev/null +++ b/tests/handlers/test_directory.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 matrix.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.trial import unittest +from twisted.internet import defer + +from mock import Mock +import logging + +from synapse.server import HomeServer +from synapse.handlers.directory import DirectoryHandler +from synapse.storage.directory import RoomAliasMapping + + +logging.getLogger().addHandler(logging.NullHandler()) + + +class DirectoryHandlers(object): + def __init__(self, hs): + self.directory_handler = DirectoryHandler(hs) + + +class DirectoryTestCase(unittest.TestCase): + """ Tests the directory service. """ + + def setUp(self): + self.mock_federation = Mock(spec=[ + "make_query", + ]) + + self.query_handlers = {} + def register_query_handler(query_type, handler): + self.query_handlers[query_type] = handler + self.mock_federation.register_query_handler = register_query_handler + + hs = HomeServer("test", + datastore=Mock(spec=[ + "get_association_from_room_alias", + ]), + http_client=None, + http_server=Mock(), + replication_layer=self.mock_federation, + ) + hs.handlers = DirectoryHandlers(hs) + + self.handler = hs.get_handlers().directory_handler + + self.datastore = hs.get_datastore() + + self.my_room = hs.parse_roomalias("#my-room:test") + self.remote_room = hs.parse_roomalias("#another:remote") + + @defer.inlineCallbacks + def test_get_local_association(self): + mocked_get = self.datastore.get_association_from_room_alias + mocked_get.return_value = defer.succeed( + RoomAliasMapping("!8765qwer:test", "#my-room:test", ["test"]) + ) + + result = yield self.handler.get_association(self.my_room) + + self.assertEquals({ + "room_id": "!8765qwer:test", + "servers": ["test"], + }, result) + + @defer.inlineCallbacks + def test_get_remote_association(self): + self.mock_federation.make_query.return_value = defer.succeed( + {"room_id": "!8765qwer:test", "servers": ["test", "remote"]} + ) + + result = yield self.handler.get_association(self.remote_room) + + self.assertEquals({ + "room_id": "!8765qwer:test", + "servers": ["test", "remote"], + }, result) + self.mock_federation.make_query.assert_called_with( + destination="remote", + query_type="directory", + args={"room_alias": "#another:remote"} + ) + + @defer.inlineCallbacks + def test_incoming_fed_query(self): + mocked_get = self.datastore.get_association_from_room_alias + mocked_get.return_value = defer.succeed( + RoomAliasMapping("!8765asdf:test", "#your-room:test", ["test"]) + ) + + response = yield self.query_handlers["directory"]( + {"room_alias": "#your-room:test"} + ) + + self.assertEquals({ + "room_id": "!8765asdf:test", + "servers": ["test"], + }, response) From b80b32d1c0373c0ab171c526bc3ea659a827bb57 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 13 Aug 2014 17:14:42 +0100 Subject: [PATCH 03/17] pagination was a terrible name --- docs/terminology.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/terminology.rst b/docs/terminology.rst index 575cc0c80..cc6e6760a 100644 --- a/docs/terminology.rst +++ b/docs/terminology.rst @@ -11,6 +11,11 @@ medium-term goal we should encourage the unification of this terminology. Terms ===== +Backfilling: + The process of synchronising historic state from one home server to another, + to backfill the event storage so that scrollback can be presented to the + client(s). (Formerly, and confusingly, called 'pagination') + Context: A single human-level entity of interest (currently, a chat room) @@ -28,11 +33,6 @@ Event: [[NOTE(paul): The current server-server implementation calls these simply "messages" but the term is too ambiguous here; I've called them Events]] -Pagination: - The process of synchronising historic state from one home server to another, - to backfill the event storage so that scrollback can be presented to the - client(s). - PDU (Persistent Data Unit): A message that relates to a single context, irrespective of the server that is communicating it. PDUs either encode a single Event, or a single State From f729f1373535015ad51e3e2870de13b7c1350708 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 13 Aug 2014 17:39:08 +0100 Subject: [PATCH 04/17] don't hammer after 403 --- webclient/room/room-controller.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 5d1c65641..a30f46baf 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -69,11 +69,14 @@ angular.module('RoomController', []) } }, function(response) { $scope.feedback = "Can't stream: " + JSON.stringify(response); + if (response.status == 403) { + $scope.stopPoll = true; + } if ($scope.stopPoll) { console.log("Stopping polling."); } else { - $timeout(shortPoll, 2000); + $timeout(shortPoll, 5000); } }); }; From 59dfbaba3b1ed236a832f8bfb2c6fc92d071f8b6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 13 Aug 2014 18:14:37 +0100 Subject: [PATCH 05/17] when we're talking about backfilling data in federation, call it backfilling - not pagination. --- docs/python_architecture.rst | 2 +- docs/server-server/specification.rst | 4 ++-- docs/versioning.rst | 8 +++---- experiments/test_messaging.py | 12 +++++----- synapse/federation/handler.py | 4 ++-- synapse/federation/persistence.py | 6 ++--- synapse/federation/replication.py | 18 +++++++-------- synapse/federation/transport.py | 34 ++++++++++++++-------------- synapse/storage/pdu.py | 20 ++++++++-------- 9 files changed, 54 insertions(+), 54 deletions(-) diff --git a/docs/python_architecture.rst b/docs/python_architecture.rst index eca36a190..8beaa615d 100644 --- a/docs/python_architecture.rst +++ b/docs/python_architecture.rst @@ -24,7 +24,7 @@ Where the bottom (the transport layer) is what talks to the internet via HTTP, a * duplicate pdu_id's - i.e., it makes sure we ignore them. * responding to requests for a given pdu_id * responding to requests for all metadata for a given context (i.e. room) - * handling incoming pagination requests + * handling incoming backfill requests So it has to parse incoming messages to discover which are metadata and which aren't, and has to correctly clobber existing metadata where appropriate. diff --git a/docs/server-server/specification.rst b/docs/server-server/specification.rst index a386bd3e7..f3c571aa8 100644 --- a/docs/server-server/specification.rst +++ b/docs/server-server/specification.rst @@ -155,9 +155,9 @@ To fetch all the state of a given context: PDUs that encode the state. -To paginate events on a given context: +To backfill events on a given context: - GET .../paginate/:context/ + GET .../backfill/:context/ Query args: v, limit Response: JSON encoding of a single Transaction containing multiple PDUs diff --git a/docs/versioning.rst b/docs/versioning.rst index 2f94bb6ef..ffda60633 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -1,11 +1,11 @@ -Versioning is, like, hard for paginating backwards because of the number of Home Servers involved. +Versioning is, like, hard for backfilling backwards because of the number of Home Servers involved. -The way we solve this is by doing versioning as an acyclic directed graph of PDUs. For pagination purposes, this is done on a per context basis. +The way we solve this is by doing versioning as an acyclic directed graph of PDUs. For backfilling purposes, this is done on a per context basis. When we send a PDU we include all PDUs that have been received for that context that hasn't been subsequently listed in a later PDU. The trivial case is a simple list of PDUs, e.g. A <- B <- C. However, if two servers send out a PDU at the same to, both B and C would point at A - a later PDU would then list both B and C. Problems with opaque version strings: - How do you do clustering without mandating that a cluster can only have one transaction in flight to a given remote home server at a time. - If you have multiple transactions sent at once, then you might drop one transaction, receive anotherwith a version that is later than the dropped transaction and which point ARGH WE LOST A TRANSACTION. - - How do you do pagination? A version string defines a point in a stream w.r.t. a single home server, not a point in the context. + If you have multiple transactions sent at once, then you might drop one transaction, receive another with a version that is later than the dropped transaction and which point ARGH WE LOST A TRANSACTION. + - How do you do backfilling? A version string defines a point in a stream w.r.t. a single home server, not a point in the context. We only need to store the ends of the directed graph, we DO NOT need to do the whole one table of nodes and one of edges. diff --git a/experiments/test_messaging.py b/experiments/test_messaging.py index f4ae71bfc..3ff7ab820 100644 --- a/experiments/test_messaging.py +++ b/experiments/test_messaging.py @@ -104,12 +104,12 @@ class InputOutput(object): #self.print_line("OK.") return - m = re.match("^paginate (\S+)$", line) + m = re.match("^backfill (\S+)$", line) if m: - # we want to paginate a room + # we want to backfill a room room_name, = m.groups() - self.print_line("paginate %s" % room_name) - self.server.paginate(room_name) + self.print_line("backfill %s" % room_name) + self.server.backfill(room_name) return self.print_line("Unrecognized command") @@ -307,7 +307,7 @@ class HomeServer(ReplicationHandler): except Exception as e: logger.exception(e) - def paginate(self, room_name, limit=5): + def backfill(self, room_name, limit=5): room = self.joined_rooms.get(room_name) if not room: @@ -315,7 +315,7 @@ class HomeServer(ReplicationHandler): dest = room.oldest_server - return self.replication_layer.paginate(dest, room_name, limit) + return self.replication_layer.backfill(dest, room_name, limit) def _get_room_remote_servers(self, room_name): return [i for i in self.joined_rooms.setdefault(room_name,).servers] diff --git a/synapse/federation/handler.py b/synapse/federation/handler.py index d361f0aaf..580e591ac 100644 --- a/synapse/federation/handler.py +++ b/synapse/federation/handler.py @@ -75,8 +75,8 @@ class FederationEventHandler(object): @log_function @defer.inlineCallbacks def backfill(self, room_id, limit): - # TODO: Work out which destinations to ask for pagination - # self.replication_layer.paginate(dest, room_id, limit) + # TODO: Work out which destinations to ask for backfill + # self.replication_layer.backfill(dest, room_id, limit) pass @log_function diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 372245712..e0e4de4e8 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -114,14 +114,14 @@ class PduActions(object): @defer.inlineCallbacks @log_function - def paginate(self, context, pdu_list, limit): + def backfill(self, context, pdu_list, limit): """ For a given list of PDU id and origins return the proceeding `limit` `Pdu`s in the given `context`. Returns: Deferred: Results in a list of `Pdu`s. """ - results = yield self.store.get_pagination( + results = yield self.store.get_backfill( context, pdu_list, limit ) @@ -131,7 +131,7 @@ class PduActions(object): def is_new(self, pdu): """ When we receive a `Pdu` from a remote home server, we want to figure out whether it is `new`, i.e. it is not some historic PDU that - we haven't seen simply because we haven't paginated back that far. + we haven't seen simply because we haven't backfilled back that far. Returns: Deferred: Results in a `bool` diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 01020566c..bc9df2f21 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -118,7 +118,7 @@ class ReplicationLayer(object): *Note:* The home server should always call `send_pdu` even if it knows that it does not need to be replicated to other home servers. This is in case e.g. someone else joins via a remote home server and then - paginates. + backfills. TODO: Figure out when we should actually resolve the deferred. @@ -179,13 +179,13 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def paginate(self, dest, context, limit): + def backfill(self, dest, context, limit): """Requests some more historic PDUs for the given context from the given destination server. Args: dest (str): The remote home server to ask. - context (str): The context to paginate back on. + context (str): The context to backfill. limit (int): The maximum number of PDUs to return. Returns: @@ -193,16 +193,16 @@ class ReplicationLayer(object): """ extremities = yield self.store.get_oldest_pdus_in_context(context) - logger.debug("paginate extrem=%s", extremities) + logger.debug("backfill extrem=%s", extremities) # If there are no extremeties then we've (probably) reached the start. if not extremities: return - transaction_data = yield self.transport_layer.paginate( + transaction_data = yield self.transport_layer.backfill( dest, context, extremities, limit) - logger.debug("paginate transaction_data=%s", repr(transaction_data)) + logger.debug("backfill transaction_data=%s", repr(transaction_data)) transaction = Transaction(**transaction_data) @@ -281,9 +281,9 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def on_paginate_request(self, context, versions, limit): + def on_backfill_request(self, context, versions, limit): - pdus = yield self.pdu_actions.paginate(context, versions, limit) + pdus = yield self.pdu_actions.backfill(context, versions, limit) defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) @@ -427,7 +427,7 @@ class ReplicationLayer(object): # Get missing pdus if necessary. is_new = yield self.pdu_actions.is_new(pdu) if is_new and not pdu.outlier: - # We only paginate backwards to the min depth. + # We only backfill backwards to the min depth. min_depth = yield self.store.get_min_depth_for_context(pdu.context) if min_depth and pdu.depth > min_depth: diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 69166036f..e09dfc267 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -112,7 +112,7 @@ class TransportLayer(object): return self._do_request_for_transaction(destination, subpath) @log_function - def paginate(self, dest, context, pdu_tuples, limit): + def backfill(self, dest, context, pdu_tuples, limit): """ Requests `limit` previous PDUs in a given context before list of PDUs. @@ -126,14 +126,14 @@ class TransportLayer(object): Deferred: Results in a dict received from the remote homeserver. """ logger.debug( - "paginate dest=%s, context=%s, pdu_tuples=%s, limit=%s", + "backfill dest=%s, context=%s, pdu_tuples=%s, limit=%s", dest, context, repr(pdu_tuples), str(limit) ) if not pdu_tuples: return - subpath = "/paginate/%s/" % context + subpath = "/backfill/%s/" % context args = {"v": ["%s,%s" % (i, o) for i, o in pdu_tuples]} args["limit"] = limit @@ -251,8 +251,8 @@ class TransportLayer(object): self.server.register_path( "GET", - re.compile("^" + PREFIX + "/paginate/([^/]*)/$"), - lambda request, context: self._on_paginate_request( + re.compile("^" + PREFIX + "/backfill/([^/]*)/$"), + lambda request, context: self._on_backfill_request( context, request.args["v"], request.args["limit"] ) @@ -352,7 +352,7 @@ class TransportLayer(object): defer.returnValue(data) @log_function - def _on_paginate_request(self, context, v_list, limits): + def _on_backfill_request(self, context, v_list, limits): if not limits: return defer.succeed( (400, {"error": "Did not include limit param"}) @@ -362,7 +362,7 @@ class TransportLayer(object): versions = [v.split(",", 1) for v in v_list] - return self.request_handler.on_paginate_request( + return self.request_handler.on_backfill_request( context, versions, limit) @@ -371,14 +371,14 @@ class TransportReceivedHandler(object): """ def on_incoming_transaction(self, transaction): """ Called on PUT /send/, or on response to a request - that we sent (e.g. a pagination request) + that we sent (e.g. a backfill request) Args: transaction (synapse.transaction.Transaction): The transaction that was sent to us. Returns: - twisted.internet.defer.Deferred: A deferred that get's fired when + twisted.internet.defer.Deferred: A deferred that gets fired when the transaction has finished being processed. The result should be a tuple in the form of @@ -438,14 +438,14 @@ class TransportRequestHandler(object): def on_context_state_request(self, context): """ Called on GET /state// - Get's hit when someone wants all the *current* state for a given + Gets hit when someone wants all the *current* state for a given contexts. Args: context (str): The name of the context that we're interested in. Returns: - twisted.internet.defer.Deferred: A deferred that get's fired when + twisted.internet.defer.Deferred: A deferred that gets fired when the transaction has finished being processed. The result should be a tuple in the form of @@ -457,20 +457,20 @@ class TransportRequestHandler(object): """ pass - def on_paginate_request(self, context, versions, limit): - """ Called on GET /paginate//?v=...&limit=... + def on_backfill_request(self, context, versions, limit): + """ Called on GET /backfill//?v=...&limit=... - Get's hit when we want to paginate backwards on a given context from + Gets hit when we want to backfill backwards on a given context from the given point. Args: - context (str): The context to paginate on - versions (list): A list of 2-tuple's representing where to paginate + context (str): The context to backfill + versions (list): A list of 2-tuples representing where to backfill from, in the form `(pdu_id, origin)` limit (int): How many pdus to return. Returns: - Deferred: Resultsin a tuple in the form of + Deferred: Results in a tuple in the form of `(response_code, respond_body)`, where `response_body` is a python dict that will get serialized to JSON. diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 202d7f6cb..13adc581e 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -168,7 +168,7 @@ class PduStore(SQLBaseStore): return self._get_pdu_tuples(txn, txn.fetchall()) - def get_pagination(self, context, pdu_list, limit): + def get_backfill(self, context, pdu_list, limit): """Get a list of Pdus for a given topic that occured before (and including) the pdus in pdu_list. Return a list of max size `limit`. @@ -182,12 +182,12 @@ class PduStore(SQLBaseStore): list: A list of PduTuples """ return self._db_pool.runInteraction( - self._get_paginate, context, pdu_list, limit + self._get_backfill, context, pdu_list, limit ) - def _get_paginate(self, txn, context, pdu_list, limit): + def _get_backfill(self, txn, context, pdu_list, limit): logger.debug( - "paginate: %s, %s, %s", + "backfill: %s, %s, %s", context, repr(pdu_list), limit ) @@ -213,7 +213,7 @@ class PduStore(SQLBaseStore): new_front = [] for pdu_id, origin in front: logger.debug( - "_paginate_interaction: i=%s, o=%s", + "_backfill_interaction: i=%s, o=%s", pdu_id, origin ) @@ -224,7 +224,7 @@ class PduStore(SQLBaseStore): for row in txn.fetchall(): logger.debug( - "_paginate_interaction: got i=%s, o=%s", + "_backfill_interaction: got i=%s, o=%s", *row ) new_front.append(row) @@ -262,7 +262,7 @@ class PduStore(SQLBaseStore): def update_min_depth_for_context(self, context, depth): """Update the minimum `depth` of the given context, which is the line - where we stop paginating backwards on. + on which we stop backfilling backwards. Args: context (str) @@ -320,9 +320,9 @@ class PduStore(SQLBaseStore): return [(row[0], row[1], row[2]) for row in results] def get_oldest_pdus_in_context(self, context): - """Get a list of Pdus that we paginated beyond yet (and haven't seen). - This list is used when we want to paginate backwards and is the list we - send to the remote server. + """Get a list of Pdus that we haven't backfilled beyond yet (and haven't + seen). This list is used when we want to backfill backwards and is the + list we send to the remote server. Args: txn From baf04be5cfa6337dcc6041cdb67023aa7f950ee1 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 13 Aug 2014 18:15:23 +0100 Subject: [PATCH 06/17] Set datastore's .hs field in SQLBaseStore rather than in the toplevel DataStore mixed-in result class --- synapse/storage/__init__.py | 1 - synapse/storage/_base.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 3c27428c0..5d5b5f7c4 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -44,7 +44,6 @@ class DataStore(RoomDataStore, RoomMemberStore, MessageStore, RoomStore, def __init__(self, hs): super(DataStore, self).__init__(hs) self.event_factory = hs.get_event_factory() - self.hs = hs def persist_event(self, event): if event.type == MessageEvent.TYPE: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 65f691ead..1b98bdfce 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -27,6 +27,7 @@ logger = logging.getLogger(__name__) class SQLBaseStore(object): def __init__(self, hs): + self.hs = hs self._db_pool = hs.get_db_pool() def cursor_to_dict(self, cursor): From fc778e2bce34825b068e2a90eac09c05d8cce747 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 13 Aug 2014 18:26:42 +0100 Subject: [PATCH 07/17] =?UTF-8?q?Move=20MockClock=20into=20tests.utils=20s?= =?UTF-8?q?o=20we=20can=20re=C3=BCse=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/federation/test_federation.py | 12 +----------- tests/utils.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index a3bcb5ede..ec39c7ee3 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -20,7 +20,7 @@ from twisted.trial import unittest from mock import Mock import logging -from ..utils import MockHttpServer +from ..utils import MockHttpServer, MockClock from synapse.server import HomeServer from synapse.federation import initialize_http_replication @@ -48,16 +48,6 @@ def make_pdu(prev_pdus=[], **kwargs): return PduTuple(PduEntry(**pdu_fields), prev_pdus) -class MockClock(object): - now = 1000 - - def time(self): - return self.now - - def time_msec(self): - return self.time() * 1000 - - class FederationTestCase(unittest.TestCase): def setUp(self): self.mock_http_server = MockHttpServer() diff --git a/tests/utils.py b/tests/utils.py index 578866b4f..e397712d8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -95,6 +95,16 @@ class MockHttpServer(HttpServer): self.callbacks.append((method, path_pattern, callback)) +class MockClock(object): + now = 1000 + + def time(self): + return self.now + + def time_msec(self): + return self.time() * 1000 + + class MemoryDataStore(object): class RoomMember(namedtuple( From eef58a299efdb91855f0f42d2b315f80d4733e22 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 13 Aug 2014 19:07:22 +0100 Subject: [PATCH 08/17] Don't mock out presence_handler's internal start/stop methods in presencelike unit test; it's rude --- tests/handlers/test_presencelike.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index f244ab600..54b92ba8e 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -63,6 +63,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): db_pool=None, datastore=Mock(spec=[ "set_presence_state", + "is_presence_visible", "set_profile_displayname", ]), @@ -83,6 +84,10 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): return defer.succeed("Frank") self.datastore.get_profile_displayname = get_profile_displayname + def is_presence_visible(*args, **kwargs): + return defer.succeed(False) + self.datastore.is_presence_visible = is_presence_visible + def get_profile_avatar_url(user_localpart): return defer.succeed("http://foo") self.datastore.get_profile_avatar_url = get_profile_avatar_url @@ -96,14 +101,9 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): self.handlers = hs.get_handlers() - self.mock_start = Mock() - self.mock_stop = Mock() - self.mock_update_client = Mock() self.mock_update_client.return_value = defer.succeed(None) - self.handlers.presence_handler.start_polling_presence = self.mock_start - self.handlers.presence_handler.stop_polling_presence = self.mock_stop self.handlers.presence_handler.push_update_to_clients = ( self.mock_update_client) @@ -132,10 +132,6 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): mocked_set.assert_called_with("apple", {"state": UNAVAILABLE, "status_msg": "Away"}) - self.mock_start.assert_called_with(self.u_apple, - state={"state": UNAVAILABLE, "status_msg": "Away", - "displayname": "Frank", - "avatar_url": "http://foo"}) @defer.inlineCallbacks def test_push_local(self): From a6a9b71da0f68c948a9cfd6f5e552a70068fb4e3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 13 Aug 2014 19:17:30 +0100 Subject: [PATCH 09/17] Allow advancing the MockClock's time --- tests/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/utils.py b/tests/utils.py index e397712d8..20a63316f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -104,6 +104,10 @@ class MockClock(object): def time_msec(self): return self.time() * 1000 + # For unit testing + def advance_time(self, secs): + self.now += secs + class MemoryDataStore(object): From d05aa651f80b604428c003a13a03c4f6f61c317d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 13 Aug 2014 19:18:55 +0100 Subject: [PATCH 10/17] An initial hack at storing presence state-change mtimes in database and presenting age durations to clients/federation events --- synapse/handlers/presence.py | 41 ++++++++++++++++++++++++----- synapse/storage/_base.py | 1 + synapse/storage/presence.py | 5 ++-- synapse/storage/schema/presence.sql | 1 + tests/handlers/test_presence.py | 36 +++++++++++++++++++------ tests/handlers/test_presencelike.py | 34 +++++++++++++++++------- tests/rest/test_presence.py | 12 +++++++-- 7 files changed, 101 insertions(+), 29 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 1c24efd45..8bdb0fe5c 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -56,6 +56,8 @@ class PresenceHandler(BaseHandler): self.homeserver = hs + self.clock = hs.get_clock() + distributor = hs.get_distributor() distributor.observe("registered_user", self.registered_user) @@ -168,14 +170,15 @@ class PresenceHandler(BaseHandler): state = yield self.store.get_presence_state( target_user.localpart ) - defer.returnValue(state) else: raise SynapseError(404, "Presence information not visible") else: # TODO(paul): Have remote server send us permissions set - defer.returnValue( - self._get_or_offline_usercache(target_user).get_state() - ) + state = self._get_or_offline_usercache(target_user).get_state() + + if "mtime" in state: + state["mtime_age"] = self.clock.time_msec() - state.pop("mtime") + defer.returnValue(state) @defer.inlineCallbacks def set_state(self, target_user, auth_user, state): @@ -209,6 +212,8 @@ class PresenceHandler(BaseHandler): ), ]) + state["mtime"] = self.clock.time_msec() + now_online = state["state"] != PresenceState.OFFLINE was_polling = target_user in self._user_cachemap @@ -361,6 +366,8 @@ class PresenceHandler(BaseHandler): observed_user = self.hs.parse_userid(p.pop("observed_user_id")) p["observed_user"] = observed_user p.update(self._get_or_offline_usercache(observed_user).get_state()) + if "mtime" in p: + p["mtime_age"] = self.clock.time_msec() - p.pop("mtime") defer.returnValue(presence) @@ -546,10 +553,15 @@ class PresenceHandler(BaseHandler): def _push_presence_remote(self, user, destination, state=None): if state is None: state = yield self.store.get_presence_state(user.localpart) + yield self.distributor.fire( "collect_presencelike_data", user, state ) + if "mtime" in state: + state = dict(state) + state["mtime_age"] = self.clock.time_msec() - state.pop("mtime") + yield self.federation.send_edu( destination=destination, edu_type="m.presence", @@ -585,6 +597,9 @@ class PresenceHandler(BaseHandler): state = dict(push) del state["user_id"] + if "mtime_age" in state: + state["mtime"] = self.clock.time_msec() - state.pop("mtime_age") + statuscache = self._get_or_make_usercache(user) self._user_cachemap_latest_serial += 1 @@ -631,9 +646,14 @@ class PresenceHandler(BaseHandler): def push_update_to_clients(self, observer_user, observed_user, statuscache): + state = statuscache.make_event(user=observed_user, clock=self.clock) + self.notifier.on_new_user_event( observer_user.to_string(), - event_data=statuscache.make_event(user=observed_user), + event_data=statuscache.make_event( + user=observed_user, + clock=self.clock + ), stream_type=PresenceStreamData, store_id=statuscache.serial ) @@ -652,8 +672,10 @@ class PresenceStreamData(StreamData): if from_key < cachemap[k].serial <= to_key] if updates: + clock = self.presence.clock + latest_serial = max([x[1].serial for x in updates]) - data = [x[1].make_event(user=x[0]) for x in updates] + data = [x[1].make_event(user=x[0], clock=clock) for x in updates] return ((data, latest_serial)) else: return (([], self.presence._user_cachemap_latest_serial)) @@ -674,6 +696,8 @@ class UserPresenceCache(object): self.serial = None def update(self, state, serial): + assert("mtime_age" not in state) + self.state.update(state) # Delete keys that are now 'None' for k in self.state.keys(): @@ -691,8 +715,11 @@ class UserPresenceCache(object): # clone it so caller can't break our cache return dict(self.state) - def make_event(self, user): + def make_event(self, user, clock): content = self.get_state() content["user_id"] = user.to_string() + if "mtime" in content: + content["mtime_age"] = clock.time_msec() - content.pop("mtime") + return {"type": "m.presence", "content": content} diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1b98bdfce..bf1800f4b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -29,6 +29,7 @@ class SQLBaseStore(object): def __init__(self, hs): self.hs = hs self._db_pool = hs.get_db_pool() + self._clock = hs.get_clock() def cursor_to_dict(self, cursor): """Converts a SQL cursor into an list of dicts. diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 6f5b042c2..23b6d1694 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -35,7 +35,7 @@ class PresenceStore(SQLBaseStore): return self._simple_select_one( table="presence", keyvalues={"user_id": user_localpart}, - retcols=["state", "status_msg"], + retcols=["state", "status_msg", "mtime"], ) def set_presence_state(self, user_localpart, new_state): @@ -43,7 +43,8 @@ class PresenceStore(SQLBaseStore): table="presence", keyvalues={"user_id": user_localpart}, updatevalues={"state": new_state["state"], - "status_msg": new_state["status_msg"]}, + "status_msg": new_state["status_msg"], + "mtime": self._clock.time_msec()}, retcols=["state"], ) diff --git a/synapse/storage/schema/presence.sql b/synapse/storage/schema/presence.sql index b22e3ba86..b1081d3aa 100644 --- a/synapse/storage/schema/presence.sql +++ b/synapse/storage/schema/presence.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id INTEGER NOT NULL, state INTEGER, status_msg TEXT, + mtime INTEGER, -- miliseconds since last state change FOREIGN KEY(user_id) REFERENCES users(id) ); diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 2299a2a7b..b365741d9 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -20,6 +20,8 @@ from twisted.internet import defer from mock import Mock, call, ANY import logging +from ..utils import MockClock + from synapse.server import HomeServer from synapse.api.constants import PresenceState from synapse.api.errors import SynapseError @@ -55,6 +57,7 @@ class PresenceStateTestCase(unittest.TestCase): def setUp(self): hs = HomeServer("test", + clock=MockClock(), db_pool=None, datastore=Mock(spec=[ "get_presence_state", @@ -154,7 +157,11 @@ class PresenceStateTestCase(unittest.TestCase): mocked_set.assert_called_with("apple", {"state": UNAVAILABLE, "status_msg": "Away"}) self.mock_start.assert_called_with(self.u_apple, - state={"state": UNAVAILABLE, "status_msg": "Away"}) + state={ + "state": UNAVAILABLE, + "status_msg": "Away", + "mtime": 1000000, # MockClock + }) yield self.handler.set_state( target_user=self.u_apple, auth_user=self.u_apple, @@ -386,7 +393,10 @@ class PresencePushTestCase(unittest.TestCase): self.replication.send_edu = Mock() self.replication.send_edu.return_value = defer.succeed((200, "OK")) + self.clock = MockClock() + hs = HomeServer("test", + clock=self.clock, db_pool=None, datastore=Mock(spec=[ "set_presence_state", @@ -519,13 +529,18 @@ class PresencePushTestCase(unittest.TestCase): yield self.handler.set_state(self.u_banana, self.u_banana, {"state": ONLINE}) + self.clock.advance_time(2) + presence = yield self.handler.get_presence_list( observer_user=self.u_apple, accepted=True) self.assertEquals([ - {"observed_user": self.u_banana, "state": ONLINE}, - {"observed_user": self.u_clementine, "state": OFFLINE}], - presence) + {"observed_user": self.u_banana, + "state": ONLINE, + "mtime_age": 2000}, + {"observed_user": self.u_clementine, + "state": OFFLINE}, + ], presence) self.mock_update_client.assert_has_calls([ call(observer_user=self.u_banana, @@ -555,7 +570,8 @@ class PresencePushTestCase(unittest.TestCase): content={ "push": [ {"user_id": "@apple:test", - "state": "online"}, + "state": "online", + "mtime_age": 0}, ], }), call( @@ -564,7 +580,8 @@ class PresencePushTestCase(unittest.TestCase): content={ "push": [ {"user_id": "@apple:test", - "state": "online"}, + "state": "online", + "mtime_age": 0}, ], }) ], any_order=True) @@ -582,7 +599,8 @@ class PresencePushTestCase(unittest.TestCase): "remote", "m.presence", { "push": [ {"user_id": "@potato:remote", - "state": "online"}, + "state": "online", + "mtime_age": 1000}, ], } ) @@ -596,9 +614,11 @@ class PresencePushTestCase(unittest.TestCase): statuscache=ANY), ], any_order=True) + self.clock.advance_time(2) + state = yield self.handler.get_state(self.u_potato, self.u_apple) - self.assertEquals({"state": ONLINE}, state) + self.assertEquals({"state": ONLINE, "mtime_age": 3000}, state) @defer.inlineCallbacks def test_join_room_local(self): diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 54b92ba8e..6eeb1bb52 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -22,6 +22,8 @@ from twisted.internet import defer from mock import Mock, call, ANY import logging +from ..utils import MockClock + from synapse.server import HomeServer from synapse.api.constants import PresenceState from synapse.handlers.presence import PresenceHandler @@ -60,6 +62,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): def setUp(self): hs = HomeServer("test", + clock=MockClock(), db_pool=None, datastore=Mock(spec=[ "set_presence_state", @@ -156,10 +159,14 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): observer_user=self.u_apple, accepted=True) self.assertEquals([ - {"observed_user": self.u_banana, "state": ONLINE, - "displayname": "Frank", "avatar_url": "http://foo"}, - {"observed_user": self.u_clementine, "state": OFFLINE}], - presence) + {"observed_user": self.u_banana, + "state": ONLINE, + "mtime_age": 0, + "displayname": "Frank", + "avatar_url": "http://foo"}, + {"observed_user": self.u_clementine, + "state": OFFLINE}], + presence) self.mock_update_client.assert_has_calls([ call(observer_user=self.u_apple, @@ -171,9 +178,12 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): ], any_order=True) statuscache = self.mock_update_client.call_args[1]["statuscache"] - self.assertEquals({"state": ONLINE, - "displayname": "Frank", - "avatar_url": "http://foo"}, statuscache.state) + self.assertEquals({ + "state": ONLINE, + "mtime": 1000000, # MockClock + "displayname": "Frank", + "avatar_url": "http://foo", + }, statuscache.state) self.mock_update_client.reset_mock() @@ -193,9 +203,12 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): ], any_order=True) statuscache = self.mock_update_client.call_args[1]["statuscache"] - self.assertEquals({"state": ONLINE, - "displayname": "I am an Apple", - "avatar_url": "http://foo"}, statuscache.state) + self.assertEquals({ + "state": ONLINE, + "mtime": 1000000, # MockClock + "displayname": "I am an Apple", + "avatar_url": "http://foo", + }, statuscache.state) @defer.inlineCallbacks def test_push_remote(self): @@ -220,6 +233,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): "push": [ {"user_id": "@apple:test", "state": "online", + "mtime_age": 0, "displayname": "Frank", "avatar_url": "http://foo"}, ], diff --git a/tests/rest/test_presence.py b/tests/rest/test_presence.py index 7c54e067c..f013abbee 100644 --- a/tests/rest/test_presence.py +++ b/tests/rest/test_presence.py @@ -234,7 +234,11 @@ class PresenceEventStreamTestCase(unittest.TestCase): # I'll already get my own presence state change self.assertEquals({"start": "0", "end": "1", "chunk": [ {"type": "m.presence", - "content": {"user_id": "@apple:test", "state": ONLINE}}, + "content": { + "user_id": "@apple:test", + "state": ONLINE, + "mtime_age": 0, + }}, ]}, response) self.mock_datastore.set_presence_state.return_value = defer.succeed( @@ -251,5 +255,9 @@ class PresenceEventStreamTestCase(unittest.TestCase): self.assertEquals(200, code) self.assertEquals({"start": "1", "end": "2", "chunk": [ {"type": "m.presence", - "content": {"user_id": "@banana:test", "state": ONLINE}}, + "content": { + "user_id": "@banana:test", + "state": ONLINE, + "mtime_age": 0, + }}, ]}, response) From 4e21bfd2dbe2c3b48f3b6a74ad65abf44f71f9ea Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 13 Aug 2014 21:19:21 +0100 Subject: [PATCH 11/17] Consistent capitalisation of 'Matrix' as a proper noun in README; 80 col wrap --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 53a8ca5f5..6ee107d21 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ VoIP[1]. The basics you need to know to get up and running are: - Matrix user IDs look like ``@matthew:matrix.org`` (although in the future you will normally refer to yourself and others using a 3PID: email - address, phone number, etc rather than manipulating matrix user IDs) + address, phone number, etc rather than manipulating Matrix user IDs) The overall architecture is:: @@ -147,7 +147,7 @@ Setting up Federation In order for other homeservers to send messages to your server, it will need to be publicly visible on the internet, and they will need to know its host name. -You have two choices here, which will influence the form of your matrix user +You have two choices here, which will influence the form of your Matrix user IDs: 1) Use the machine's own hostname as available on public DNS in the form of its @@ -231,14 +231,15 @@ synapse sandbox running on localhost) Logging In To An Existing Account --------------------------------- -Just enter the ``@localpart:my.domain.here`` matrix user ID and password into the form and click the Login button. +Just enter the ``@localpart:my.domain.here`` Matrix user ID and password into +the form and click the Login button. Identity Servers ================ The job of authenticating 3PIDs and tracking which 3PIDs are associated with a -given matrix user is very security-sensitive, as there is obvious risk of spam +given Matrix user is very security-sensitive, as there is obvious risk of spam if it is too easy to sign up for Matrix accounts or harvest 3PID data. Meanwhile the job of publishing the end-to-end encryption public keys for Matrix users is also very security-sensitive for similar reasons. From aebe5ce08a01fb8067e7354434572a93e88256ec Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 13 Aug 2014 20:53:38 +0100 Subject: [PATCH 12/17] fix whitespace --- synapse/api/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 84bc0398f..8d2ba242e 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -14,6 +14,7 @@ # limitations under the License. """This module contains classes for authenticating the user.""" + from twisted.internet import defer from synapse.api.constants import Membership From 6c2db18be150eb6410f1e3a148057b81dcae8093 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 14 Aug 2014 02:13:14 +0100 Subject: [PATCH 13/17] completely change the CSS to be an entirely 'position: absolute' layout rather than top-to-bottom. makes the overscroll much more predictable and sane and not dependent on CSS expressions. --- webclient/app.css | 47 ++++++++++++++++--------------- webclient/app.js | 7 ++++- webclient/login/login.html | 2 ++ webclient/room/room-controller.js | 6 +++- webclient/room/room.html | 9 +++--- webclient/rooms/rooms.html | 4 ++- 6 files changed, 46 insertions(+), 29 deletions(-) diff --git a/webclient/app.css b/webclient/app.css index 15b6c9130..0111b78e0 100644 --- a/webclient/app.css +++ b/webclient/app.css @@ -11,21 +11,33 @@ h1 { /*** Overall page layout ***/ .page { - max-width: 1280px; + position: absolute; + top: 80px; + bottom: 100px; + left: 0px; + right: 0px; + margin: 20px; + margin: 20px; +} + +.wrapper { margin: auto; - margin-bottom: 80px ! important; - padding-left: 20px; - padding-right: 20px; + max-width: 1280px; + height: 100%; } .roomName { + max-width: 1280px; + width: 100%; text-align: right; + top: -40px; + position: absolute; font-size: 16pt; margin-bottom: 10px; } .controlPanel { - position: fixed; + position: absolute; bottom: 0px; width: 100%; background-color: #f8f8f8; @@ -70,8 +82,9 @@ h1 { .userAvatar { width: 80px; - height: 80px; + height: 100px; position: relative; + background-color: #000; } .userAvatar .userAvatarImage { @@ -81,7 +94,7 @@ h1 { .userAvatar .userAvatarGradient { position: absolute; - bottom: 0px; + bottom: 20px; } .userAvatar .userName { @@ -91,7 +104,6 @@ h1 { bottom: 0px; font-size: 8pt; word-wrap: break-word; - word-break: break-all; } .userPresence { @@ -110,27 +122,18 @@ h1 { background-color: #FFCC00; } -/*** Room page ***/ - -/* Limit the height of the page content to 100% of the viewport height minus the - height of the header and the footer. - The two divs containing the messages list and the users list will then scroll- - overflow separetely. - */ -.room .page { - height: calc(100vh - 220px); -} - /*** Message table ***/ .messageTableWrapper { - width: auto; height: 100%; margin-right: 140px; overflow-y: auto; + width: auto; } .messageTable { + margin: auto; + max-width: 1280px; width: 100%; border-collapse: collapse; } @@ -180,6 +183,8 @@ h1 { height: 32px; display: inline-table; max-width: 90%; + word-wrap: break-word; + word-break: break-all; } .emote { @@ -217,12 +222,10 @@ h1 { /******************************/ .header { - margin-top: 12px ! important; padding-left: 20px; padding-right: 20px; max-width: 1280px; margin: auto; - height: 60px; } .header-buttons { diff --git a/webclient/app.js b/webclient/app.js index 2133a98cb..651aeeaa7 100644 --- a/webclient/app.js +++ b/webclient/app.js @@ -70,4 +70,9 @@ matrixWebClient $timeout(function() { element[0].focus() }, 0); } }; - }]); + }]) + .filter('to_trusted', ['$sce', function($sce){ + return function(text) { + return $sce.trustAsHtml(text); + }; + }]); \ No newline at end of file diff --git a/webclient/login/login.html b/webclient/login/login.html index 0d3e8c57f..508ff5e4b 100644 --- a/webclient/login/login.html +++ b/webclient/login/login.html @@ -1,5 +1,6 @@ diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index a30f46baf..470f41521 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -42,6 +42,8 @@ angular.module('RoomController', []) console.log("Got response from "+$scope.state.events_from+" to "+response.data.end); $scope.state.events_from = response.data.end; + $scope.feedback = ""; + for (var i = 0; i < response.data.chunk.length; i++) { var chunk = response.data.chunk[i]; if (chunk.room_id == $scope.room_id && chunk.type == "m.room.message") { @@ -68,10 +70,12 @@ angular.module('RoomController', []) $timeout(shortPoll, 0); } }, function(response) { - $scope.feedback = "Can't stream: " + JSON.stringify(response); + $scope.feedback = "Can't stream: " + response.data; + if (response.status == 403) { $scope.stopPoll = true; } + if ($scope.stopPoll) { console.log("Stopping polling."); } diff --git a/webclient/room/room.html b/webclient/room/room.html index 87d3458af..51af54e7b 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -1,6 +1,7 @@
+
{{ room_alias || room_id }} @@ -12,7 +13,8 @@ -
{{ info.displayname || name }}
+ +
@@ -44,6 +46,7 @@
+
@@ -53,7 +56,7 @@ {{ state.user_id }} - + @@ -85,7 +88,5 @@
- - diff --git a/webclient/rooms/rooms.html b/webclient/rooms/rooms.html index f134e5ee8..d303e143b 100644 --- a/webclient/rooms/rooms.html +++ b/webclient/rooms/rooms.html @@ -1,7 +1,8 @@
- +
+
@@ -77,4 +78,5 @@ {{ feedback }}
+
From d7dcef7ff4e7f2eb73ab2205f1f9c0d97aba3457 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 14 Aug 2014 02:21:49 +0100 Subject: [PATCH 14/17] config css --- webclient/app.css | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/webclient/app.css b/webclient/app.css index 0111b78e0..65049c95c 100644 --- a/webclient/app.css +++ b/webclient/app.css @@ -232,6 +232,18 @@ h1 { float: right; } +.config { + position: absolute; + z-index: 100; + top: 100px; + left: 50%; + width: 400px; + margin-left: -200px; + text-align: center; + padding: 20px; + background-color: #aaa; +} + .text_entry_section { position: fixed; bottom: 0; From a4da962babcb865861af6bfcd4e318dc386d7c5e Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 14 Aug 2014 02:59:54 +0100 Subject: [PATCH 15/17] fix http client GET parameters; somehow missing named param. how could this have ever worked!? --- synapse/http/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 5c73d62cf..36ba2c659 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -32,7 +32,7 @@ import urllib logger = logging.getLogger(__name__) - +# FIXME: SURELY these should be killed?! _destination_mappings = { "red": "localhost:8080", "blue": "localhost:8081", @@ -147,7 +147,7 @@ class TwistedHttpClient(HttpClient): destination.encode("ascii"), "GET", path.encode("ascii"), - query_bytes + query_bytes=query_bytes ) body = yield readBody(response) From 9391be0f5d2c04c13467de6c7ae81c4d6cc4c0a5 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 14 Aug 2014 03:35:09 +0100 Subject: [PATCH 16/17] fix emote presentation --- webclient/room/room.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webclient/room/room.html b/webclient/room/room.html index 51af54e7b..8fc7d5d36 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -33,7 +33,7 @@
- {{ msg.content.msgtype === "m.emote" ? ("* " + (members[msg.user_id].displayname || msg.user_id) + " ") : "" }} + {{ msg.content.msgtype === "m.emote" ? ("* " + (members[msg.user_id].displayname || msg.user_id) + " " + msg.content.body) : "" }} {{ msg.content.msgtype === "m.text" ? msg.content.body : "" }} {{ msg.content.body }}
From e7736668ba0ab2f7954717b6d99dc2f7c69d1845 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 14 Aug 2014 03:36:03 +0100 Subject: [PATCH 17/17] grammar fix --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6ee107d21..8131172d8 100644 --- a/README.rst +++ b/README.rst @@ -40,8 +40,8 @@ To get up and running: About Matrix ============ -Matrix specifies a set of pragmatic RESTful HTTP JSON APIs for VoIP and IM as an -open standard, providing: +Matrix specifies a set of pragmatic RESTful HTTP JSON APIs as an open standard, +which handle: - Creating and managing fully distributed chat rooms with no single points of control or failure