Merge branch 'develop' into pushers

CHANGES.rst

@@ -1,7 +1,21 @@
+Changes in synapse 0.6.1 (2015-01-07)
+=====================================
+
+ * Major optimizations to improve performance of initial sync and event sending
+   in large rooms (by up to 10x)
+ * Media repository now includes a Content-Length header on media downloads.
+ * Improve quality of thumbnails by changing resizing algorithm.
+
 Changes in synapse 0.6.0 (2014-12-16)
 =====================================
 
  * Add new API for media upload and download that supports thumbnailing.
+ * Replicate media uploads over multiple homeservers so media is always served
+   to clients from their local homeserver.  This obsoletes the
+   --content-addr parameter and confusion over accessing content directly
+   from remote homeservers.
+ * Implement exponential backoff when retrying federation requests when
+   sending to remote homeservers which are offline.
  * Implement typing notifications.
  * Fix bugs where we sent events with invalid signatures due to bugs where
    we incorrectly persisted events.

README.rst

@@ -108,6 +108,18 @@ To install the synapse homeserver run::
 This installs synapse, along with the libraries it uses, into
 ``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX.
 
+Your python may not give priority to locally installed libraries over system
+libraries, in which case you must add your local packages to your python path::
+
+    $ # on Linux:
+    $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages:$PYTHONPATH
+
+    $ # on OSX:
+    $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages:$PYTHONPATH
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server.  See docs/turn-howto.rst for details.
+
 Troubleshooting Installation
 ----------------------------
 
@@ -239,6 +251,11 @@ Upgrading an existing homeserver
 IMPORTANT: Before upgrading an existing homeserver to a new version, please
 refer to UPGRADE.rst for any additional instructions.
 
+Otherwise, simply re-install the new codebase over the current one - e.g.
+by ``pip install --user --process-dependency-links
+https://github.com/matrix-org/synapse/tarball/master``
+if using pip, or by ``git pull`` if running off a git working copy.
+
 
 Setting up Federation
 =====================

UPGRADE.rst

@@ -1,6 +1,10 @@
 Upgrading to v0.6.0
 ===================
 
+To pull in new dependencies, run::
+
+    python setup.py develop --user
+
 This update includes a change to the database schema. To upgrade you first need
 to upgrade the database by running::
 

VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1b

docs/media_repository.rst

@@ -1,6 +1,8 @@
-Media Repository
+Media Repository 
 ================
 
+*Synapse implementation-specific details for the media repository*
+
 The media repository is where attachments and avatar photos are stored.
 It stores attachment content and thumbnails for media uploaded by local users.
 It caches attachment content and thumbnails for media uploaded by remote users.

graph/graph2.py

@@ -23,14 +23,27 @@ import argparse
 from synapse.events import FrozenEvent
 
 
-def make_graph(db_name, room_id, file_prefix):
+def make_graph(db_name, room_id, file_prefix, limit):
     conn = sqlite3.connect(db_name)
 
-    c = conn.execute(
+    sql = (
-        "SELECT json FROM event_json where room_id = ?",
+        "SELECT json FROM event_json as j "
-        (room_id,)
+        "INNER JOIN events as e ON e.event_id = j.event_id "
+        "WHERE j.room_id = ?"
     )
 
+    args = [room_id]
+
+    if limit:
+        sql += (
+            " ORDER BY topological_ordering DESC, stream_ordering DESC "
+            "LIMIT ?"
+        )
+
+        args.append(limit)
+
+    c = conn.execute(sql, args)
+
     events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()]
 
     events.sort(key=lambda e: e.depth)
@@ -128,11 +141,16 @@ if __name__ == "__main__":
     )
     parser.add_argument(
         "-p", "--prefix", dest="prefix",
-        help="String to prefix output files with"
+        help="String to prefix output files with",
+        default="graph_output"
+    )
+    parser.add_argument(
+        "-l", "--limit",
+        help="Only retrieve the last N events.",
     )
     parser.add_argument('db')
     parser.add_argument('room')
 
     args = parser.parse_args()
 
-    make_graph(args.db, args.room, args.prefix)
+    make_graph(args.db, args.room, args.prefix, args.limit)

scripts/copyrighter-sql.pl

@@ -0,0 +1,33 @@
+#!/usr/bin/perl -pi
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$copyright = <<EOT;
+/* Copyright 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+EOT
+
+s/^(# -\*- coding: utf-8 -\*-\n)?/$1$copyright/ if ($. == 1);

scripts/copyrighter.pl

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 $copyright = <<EOT;
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

setup.py

@@ -32,7 +32,7 @@ setup(
     description="Reference Synapse Home Server",
     install_requires=[
         "syutil==0.0.2",
-        "matrix_angular_sdk==0.5.3b",
+        "matrix_angular_sdk==0.6.0",
         "Twisted>=14.0.0",
         "service_identity>=1.0.0",
         "pyopenssl>=0.14",
@@ -47,7 +47,7 @@ setup(
     dependency_links=[
         "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2",
         "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0",
-        "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.3b/#egg=matrix_angular_sdk-0.5.3b",
+        "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0",
     ],
     setup_requires=[
         "setuptools_trial",

synapse/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" This is a reference implementation of a synapse home server.
+""" This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.6.0"
+__version__ = "0.6.1b"

synapse/api/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/auth.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/constants.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/errors.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/ratelimiting.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/api/urls.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/app/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/app/homeserver.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@ from synapse.storage import prepare_database, UpgradeDatabaseException
 
 from synapse.server import HomeServer
 
+from synapse.python_dependencies import check_requirements
+
 from twisted.internet import reactor
 from twisted.enterprise import adbapi
 from twisted.web.resource import Resource
@@ -39,6 +41,8 @@ from synapse.util.logcontext import LoggingContext
 from daemonize import Daemonize
 import twisted.manhole.telnet
 
+import synapse
+
 import logging
 import os
 import re
@@ -198,7 +202,10 @@ def setup():
 
     config.setup_logging()
 
+    check_requirements()
+
     logger.info("Server hostname: %s", config.server_name)
+    logger.info("Server version: %s", synapse.__version__)
 
     if re.search(":[0-9]+$", config.server_name):
         domain_with_port = config.server_name
@@ -234,7 +241,7 @@ def setup():
     except UpgradeDatabaseException:
         sys.stderr.write(
             "\nFailed to upgrade database.\n"
-            "Have you followed any instructions in UPGRADES.rst?\n"
+            "Have you checked for version specific instructions in UPGRADES.rst?\n"
         )
         sys.exit(1)
 
@@ -279,6 +286,7 @@ def run():
 
 def main():
     with LoggingContext("main"):
+        check_requirements()
         setup()
 
 

synapse/app/synctl.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/captcha.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/database.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/email.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/homeserver.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/logger.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -66,7 +66,10 @@ class LoggingConfig(Config):
 
             formatter = logging.Formatter(log_format)
             if self.log_file:
-                handler = logging.FileHandler(self.log_file)
+                # TODO: Customisable file size / backup count
+                handler = logging.handlers.RotatingFileHandler(
+                    self.log_file, maxBytes=(1000 * 1000 * 100), backupCount=3
+                )
             else:
                 handler = logging.StreamHandler()
             handler.setFormatter(formatter)

synapse/config/ratelimiting.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/repository.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014, 2015 matrix.org
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,7 +37,7 @@ class ContentRepositoryConfig(Config):
         super(ContentRepositoryConfig, cls).add_arguments(parser)
         db_group = parser.add_argument_group("content_repository")
         db_group.add_argument(
-            "--max-upload-size", default="1M"
+            "--max-upload-size", default="10M"
         )
         db_group.add_argument(
             "--media-store-path", default=cls.default_path("media_store")

synapse/config/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,8 +47,12 @@ class ServerConfig(Config):
     def add_arguments(cls, parser):
         super(ServerConfig, cls).add_arguments(parser)
         server_group = parser.add_argument_group("server")
-        server_group.add_argument("-H", "--server-name", default="localhost",
+        server_group.add_argument(
-                                  help="The name of the server")
+            "-H", "--server-name", default="localhost",
+            help="The domain name of the server, with optional explicit port. "
+                 "This is used by remote servers to connect to this server, "
+                 "e.g. matrix.org, localhost:8080, etc."
+        )
         server_group.add_argument("--signing-key-path",
                                   help="The signing key to sign messages with")
         server_group.add_argument("-p", "--bind-port", metavar="PORT",

synapse/config/tls.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/config/voip.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/context_factory.py

@@ -1,4 +1,4 @@
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/event_signing.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/keyclient.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/crypto/keyring.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/events/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,12 +15,10 @@
 
 from synapse.util.frozenutils import freeze, unfreeze
 
-import copy
-
 
 class _EventInternalMetadata(object):
     def __init__(self, internal_metadata_dict):
-        self.__dict__ = copy.deepcopy(internal_metadata_dict)
+        self.__dict__ = internal_metadata_dict
 
     def get_dict(self):
         return dict(self.__dict__)
@@ -49,10 +47,10 @@ def _event_dict_property(key):
 class EventBase(object):
     def __init__(self, event_dict, signatures={}, unsigned={},
                  internal_metadata_dict={}):
-        self.signatures = copy.deepcopy(signatures)
+        self.signatures = signatures
-        self.unsigned = copy.deepcopy(unsigned)
+        self.unsigned = unsigned
 
-        self._event_dict = copy.deepcopy(event_dict)
+        self._event_dict = event_dict
 
         self.internal_metadata = _EventInternalMetadata(
             internal_metadata_dict
@@ -112,10 +110,16 @@ class EventBase(object):
 
 class FrozenEvent(EventBase):
     def __init__(self, event_dict, internal_metadata_dict={}):
-        event_dict = copy.deepcopy(event_dict)
+        event_dict = dict(event_dict)
 
-        signatures = copy.deepcopy(event_dict.pop("signatures", {}))
+        # Signatures is a dict of dicts, and this is faster than doing a
-        unsigned = copy.deepcopy(event_dict.pop("unsigned", {}))
+        # copy.deepcopy
+        signatures = {
+            name: {sig_id: sig for sig_id, sig in sigs.items()}
+            for name, sigs in event_dict.pop("signatures", {}).items()
+        }
+
+        unsigned = dict(event_dict.pop("unsigned", {}))
 
         frozen_dict = freeze(event_dict)
 

synapse/events/builder.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/events/snapshot.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/events/utils.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -89,16 +89,24 @@ def prune_event(event):
     return type(event)(allowed_fields)
 
 
-def serialize_event(hs, e):
+def serialize_event(hs, e, client_event=True):
     # FIXME(erikj): To handle the case of presence events and the like
     if not isinstance(e, EventBase):
         return e
 
     # Should this strip out None's?
     d = {k: v for k, v in e.get_dict().items()}
+
+    if not client_event:
+        # set the age and keep all other keys
+        if "age_ts" in d["unsigned"]:
+            now = int(hs.get_clock().time_msec())
+            d["unsigned"]["age"] = now - d["unsigned"]["age_ts"]
+        return d
+
     if "age_ts" in d["unsigned"]:
         now = int(hs.get_clock().time_msec())
-        d["unsigned"]["age"] = now - d["unsigned"]["age_ts"]
+        d["age"] = now - d["unsigned"]["age_ts"]
         del d["unsigned"]["age_ts"]
 
     d["user_id"] = d.pop("sender", None)
@@ -126,5 +134,8 @@ def serialize_event(hs, e):
     del d["prev_events"]
     del d["hashes"]
     del d["signatures"]
+    d.pop("depth", None)
+    d.pop("unsigned", None)
+    d.pop("origin", None)
 
     return d

synapse/events/validator.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/persistence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/replication.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -256,31 +256,35 @@ class ReplicationLayer(object):
 
     @defer.inlineCallbacks
     @log_function
-    def get_state_for_context(self, destination, context, event_id=None):
+    def get_state_for_context(self, destination, context, event_id):
         """Requests all of the `current` state PDUs for a given context from
         a remote home server.
 
         Args:
             destination (str): The remote homeserver to query for the state.
             context (str): The context we're interested in.
+            event_id (str): The id of the event we want the state at.
 
         Returns:
             Deferred: Results in a list of PDUs.
         """
 
-        transaction_data = yield self.transport_layer.get_context_state(
+        result = yield self.transport_layer.get_context_state(
             destination,
             context,
             event_id=event_id,
         )
 
-        transaction = Transaction(**transaction_data)
         pdus = [
-            self.event_from_pdu_json(p, outlier=True)
+            self.event_from_pdu_json(p, outlier=True) for p in result["pdus"]
-            for p in transaction.pdus
         ]
 
-        defer.returnValue(pdus)
+        auth_chain = [
+            self.event_from_pdu_json(p, outlier=True)
+            for p in result.get("auth_chain", [])
+        ]
+
+        defer.returnValue((pdus, auth_chain))
 
     @defer.inlineCallbacks
     @log_function
@@ -383,10 +387,16 @@ class ReplicationLayer(object):
                 context,
                 event_id,
             )
+            auth_chain = yield self.store.get_auth_chain(
+                [pdu.event_id for pdu in pdus]
+            )
         else:
             raise NotImplementedError("Specify an event")
 
-        defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict()))
+        defer.returnValue((200, {
+            "pdus": [pdu.get_pdu_json() for pdu in pdus],
+            "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain],
+        }))
 
     @defer.inlineCallbacks
     @log_function
@@ -562,8 +572,8 @@ class ReplicationLayer(object):
 
         already_seen = (
             existing and (
-                not existing.internal_metadata.outlier
+                not existing.internal_metadata.is_outlier()
-                or pdu.internal_metadata.outlier
+                or pdu.internal_metadata.is_outlier()
             )
         )
         if already_seen:
@@ -573,6 +583,8 @@ class ReplicationLayer(object):
 
         state = None
 
+        auth_chain = []
+
         # We need to make sure we have all the auth events.
         # for e_id, _ in pdu.auth_events:
         #     exists = yield self._get_persisted_pdu(
@@ -604,7 +616,7 @@ class ReplicationLayer(object):
         #             )
 
         # Get missing pdus if necessary.
-        if not pdu.internal_metadata.outlier:
+        if not pdu.internal_metadata.is_outlier():
             # We only backfill backwards to the min depth.
             min_depth = yield self.handler.get_min_depth_for_context(
                 pdu.room_id
@@ -645,7 +657,7 @@ class ReplicationLayer(object):
                     "_handle_new_pdu getting state for %s",
                     pdu.room_id
                 )
-                state = yield self.get_state_for_context(
+                state, auth_chain = yield self.get_state_for_context(
                     origin, pdu.room_id, pdu.event_id,
                 )
 
@@ -655,6 +667,7 @@ class ReplicationLayer(object):
                 pdu,
                 backfilled=backfilled,
                 state=state,
+                auth_chain=auth_chain,
             )
         else:
             ret = None
@@ -717,6 +730,7 @@ class _TransactionQueue(object):
 
         destinations = set(destinations)
         destinations.discard(self.server_name)
+        destinations.discard("localhost")
 
         logger.debug("Sending to: %s", str(destinations))
 
@@ -801,6 +815,8 @@ class _TransactionQueue(object):
             else:
                 logger.info("TX [%s] is ready for retry", destination)
 
+        logger.info("TX [%s] _attempt_new_transaction", destination)
+        
         if destination in self.pending_transactions:
             # XXX: pending_transactions can get stuck on by a never-ending
             # request at which point pending_pdus_by_dest just keeps growing.
@@ -813,6 +829,9 @@ class _TransactionQueue(object):
         pending_edus = self.pending_edus_by_dest.pop(destination, [])
         pending_failures = self.pending_failures_by_dest.pop(destination, [])
 
+        if pending_pdus:
+            logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", destination, len(pending_pdus))
+
         if not pending_pdus and not pending_edus and not pending_failures:
             return
 

synapse/federation/transport.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/federation/units.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/admin.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/directory.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -155,4 +155,4 @@ class DirectoryHandler(BaseHandler):
             "room_id": room_id,
             "sender": user_id,
             "content": {"aliases": aliases},
-        })
+        }, ratelimit=False)

synapse/handlers/events.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,7 +46,8 @@ class EventStreamHandler(BaseHandler):
 
     @defer.inlineCallbacks
     @log_function
-    def get_stream(self, auth_user_id, pagin_config, timeout=0):
+    def get_stream(self, auth_user_id, pagin_config, timeout=0,
+                   as_client_event=True):
         auth_user = self.hs.parse_userid(auth_user_id)
 
         try:
@@ -78,7 +79,9 @@ class EventStreamHandler(BaseHandler):
                     auth_user, room_ids, pagin_config, timeout
                 )
 
-            chunks = [self.hs.serialize_event(e) for e in events]
+            chunks = [
+                self.hs.serialize_event(e, as_client_event) for e in events
+            ]
 
             chunk = {
                 "chunk": chunks,

synapse/handlers/federation.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -91,11 +91,12 @@ class FederationHandler(BaseHandler):
 
         yield run_on_reactor()
 
-        yield self.replication_layer.send_pdu(event, destinations)
+        self.replication_layer.send_pdu(event, destinations)
 
     @log_function
     @defer.inlineCallbacks
-    def on_receive_pdu(self, origin, pdu, backfilled, state=None):
+    def on_receive_pdu(self, origin, pdu, backfilled, state=None,
+                       auth_chain=None):
         """ Called by the ReplicationLayer when we have a new pdu. We need to
         do auth checks and put it through the StateHandler.
         """
@@ -150,40 +151,41 @@ class FederationHandler(BaseHandler):
         if not is_in_room and not event.internal_metadata.outlier:
             logger.debug("Got event for room we're not in.")
 
-            replication_layer = self.replication_layer
+            replication = self.replication_layer
-            auth_chain = yield replication_layer.get_event_auth(
-                origin,
-                context=event.room_id,
-                event_id=event.event_id,
-            )
-
-            for e in auth_chain:
-                e.internal_metadata.outlier = True
-                try:
-                    yield self._handle_new_event(e, fetch_missing=False)
-                except:
-                    logger.exception(
-                        "Failed to parse auth event %s",
-                        e.event_id,
-                    )
 
             if not state:
-                state = yield replication_layer.get_state_for_context(
+                state, auth_chain = yield replication.get_state_for_context(
+                    origin, context=event.room_id, event_id=event.event_id,
+                )
+
+            if not auth_chain:
+                auth_chain = yield replication.get_event_auth(
                     origin,
                     context=event.room_id,
                     event_id=event.event_id,
                 )
 
+            for e in auth_chain:
+                e.internal_metadata.outlier = True
+                try:
+                    yield self._handle_new_event(e, fetch_auth_from=origin)
+                except:
+                    logger.exception(
+                        "Failed to handle auth event %s",
+                        e.event_id,
+                    )
+
             current_state = state
 
         if state:
             for e in state:
+                logging.info("A :) %r", e)
                 e.internal_metadata.outlier = True
                 try:
                     yield self._handle_new_event(e)
                 except:
                     logger.exception(
-                        "Failed to parse state event %s",
+                        "Failed to handle state event %s",
                         e.event_id,
                     )
 
@@ -288,7 +290,7 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def on_event_auth(self, event_id):
-        auth = yield self.store.get_auth_chain(event_id)
+        auth = yield self.store.get_auth_chain([event_id])
 
         for event in auth:
             event.signatures.update(
@@ -391,10 +393,10 @@ class FederationHandler(BaseHandler):
             for e in auth_chain:
                 e.internal_metadata.outlier = True
                 try:
-                    yield self._handle_new_event(e, fetch_missing=False)
+                    yield self._handle_new_event(e)
                 except:
                     logger.exception(
-                        "Failed to parse auth event %s",
+                        "Failed to handle auth event %s",
                         e.event_id,
                     )
 
@@ -403,12 +405,11 @@ class FederationHandler(BaseHandler):
                 e.internal_metadata.outlier = True
                 try:
                     yield self._handle_new_event(
-                        e,
+                        e, fetch_auth_from=target_host
-                        fetch_missing=True
                     )
                 except:
                     logger.exception(
-                        "Failed to parse state event %s",
+                        "Failed to handle state event %s",
                         e.event_id,
                     )
 
@@ -526,9 +527,12 @@ class FederationHandler(BaseHandler):
             event.signatures,
         )
 
-        yield self.replication_layer.send_pdu(new_pdu, destinations)
+        self.replication_layer.send_pdu(new_pdu, destinations)
 
-        auth_chain = yield self.store.get_auth_chain(event.event_id)
+        state_ids = [e.event_id for e in context.current_state.values()]
+        auth_chain = yield self.store.get_auth_chain(set(
+            [event.event_id] + state_ids
+        ))
 
         defer.returnValue({
             "state": context.current_state.values(),
@@ -613,13 +617,13 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     @log_function
-    def on_backfill_request(self, origin, context, pdu_list, limit):
+    def on_backfill_request(self, origin, room_id, pdu_list, limit):
-        in_room = yield self.auth.check_host_in_room(context, origin)
+        in_room = yield self.auth.check_host_in_room(room_id, origin)
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
         events = yield self.store.get_backfill_events(
-            context,
+            room_id,
             pdu_list,
             limit
         )
@@ -678,7 +682,7 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _handle_new_event(self, event, state=None, backfilled=False,
-                          current_state=None, fetch_missing=True):
+                          current_state=None, fetch_auth_from=None):
 
         logger.debug(
             "_handle_new_event: Before annotate: %s, sigs: %s",
@@ -699,11 +703,20 @@ class FederationHandler(BaseHandler):
         known_ids = set(
             [s.event_id for s in context.auth_events.values()]
         )
+
         for e_id, _ in event.auth_events:
             if e_id not in known_ids:
-                e = yield self.store.get_event(
+                e = yield self.store.get_event(e_id, allow_none=True)
-                    e_id, allow_none=True,
+
-                )
+                if not e and fetch_auth_from is not None:
+                    # Grab the auth_chain over federation if we are missing
+                    # auth events.
+                    auth_chain = yield self.replication_layer.get_event_auth(
+                        fetch_auth_from, event.event_id, event.room_id
+                    )
+                    for auth_event in auth_chain:
+                        yield self._handle_new_event(auth_event)
+                    e = yield self.store.get_event(e_id, allow_none=True)
 
                 if not e:
                     # TODO: Do some conflict res to make sure that we're
@@ -713,7 +726,7 @@ class FederationHandler(BaseHandler):
                         event.event_id, e_id, known_ids,
                     )
                     # FIXME: How does raising AuthError work with federation?
-                    raise AuthError(403, "Auth events are stale")
+                    raise AuthError(403, "Cannot find auth event")
 
                 context.auth_events[(e.type, e.state_key)] = e
 

synapse/handlers/login.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/message.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import RoomError
 from synapse.streams.config import PaginationConfig
 from synapse.events.validator import EventValidator
+from synapse.util.logcontext import PreserveLoggingContext
 
 from ._base import BaseHandler
 
@@ -66,7 +67,7 @@ class MessageHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def get_messages(self, user_id=None, room_id=None, pagin_config=None,
-                     feedback=False):
+                     feedback=False, as_client_event=True):
         """Get messages in a room.
 
         Args:
@@ -75,6 +76,7 @@ class MessageHandler(BaseHandler):
             pagin_config (synapse.api.streams.PaginationConfig): The pagination
             config rules to apply, if any.
             feedback (bool): True to get compressed feedback with the messages
+            as_client_event (bool): True to get events in client-server format.
         Returns:
             dict: Pagination API results
         """
@@ -98,7 +100,9 @@ class MessageHandler(BaseHandler):
         )
 
         chunk = {
-            "chunk": [self.hs.serialize_event(e) for e in events],
+            "chunk": [
+                self.hs.serialize_event(e, as_client_event) for e in events
+            ],
             "start": pagin_config.from_token.to_string(),
             "end": next_token.to_string(),
         }
@@ -106,7 +110,7 @@ class MessageHandler(BaseHandler):
         defer.returnValue(chunk)
 
     @defer.inlineCallbacks
-    def create_and_send_event(self, event_dict):
+    def create_and_send_event(self, event_dict, ratelimit=True):
         """ Given a dict from a client, create and handle a new event.
 
         Creates an FrozenEvent object, filling out auth_events, prev_events,
@@ -123,7 +127,8 @@ class MessageHandler(BaseHandler):
 
         self.validator.validate_new(builder)
 
-        self.ratelimit(builder.user_id)
+        if ratelimit:
+            self.ratelimit(builder.user_id)
         # TODO(paul): Why does 'event' not have a 'user' object?
         user = self.hs.parse_userid(builder.user_id)
         assert self.hs.is_mine(user), "User must be our own: %s" % (user,)
@@ -152,6 +157,11 @@ class MessageHandler(BaseHandler):
                 context=context,
             )
 
+        if event.type == EventTypes.Message:
+            presence = self.hs.get_handlers().presence_handler
+            with PreserveLoggingContext():
+                presence.bump_presence_active_time(user)
+
         defer.returnValue(event)
 
     @defer.inlineCallbacks
@@ -204,7 +214,7 @@ class MessageHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def snapshot_all_rooms(self, user_id=None, pagin_config=None,
-                           feedback=False):
+                           feedback=False, as_client_event=True):
         """Retrieve a snapshot of all rooms the user is invited or has joined.
 
         This snapshot may include messages for all rooms where the user is
@@ -215,6 +225,7 @@ class MessageHandler(BaseHandler):
             pagin_config (synapse.api.streams.PaginationConfig): The pagination
             config used to determine how many messages *PER ROOM* to return.
             feedback (bool): True to get feedback along with these messages.
+            as_client_event (bool): True to get events in client-server format.
         Returns:
             A list of dicts with "room_id" and "membership" keys for all rooms
             the user is currently invited or joined in on. Rooms where the user
@@ -256,7 +267,7 @@ class MessageHandler(BaseHandler):
             }
 
             if event.membership == Membership.INVITE:
-                d["inviter"] = event.user_id
+                d["inviter"] = event.sender
 
             rooms_ret.append(d)
 
@@ -273,7 +284,10 @@ class MessageHandler(BaseHandler):
                 end_token = now_token.copy_and_replace("room_key", token[1])
 
                 d["messages"] = {
-                    "chunk": [self.hs.serialize_event(m) for m in messages],
+                    "chunk": [
+                        self.hs.serialize_event(m, as_client_event)
+                        for m in messages
+                    ],
                     "start": start_token.to_string(),
                     "end": end_token.to_string(),
                 }

synapse/handlers/presence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/profile.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError, AuthError, CodeMessageException
-from synapse.api.constants import Membership
+from synapse.api.constants import EventTypes, Membership
 from synapse.util.logcontext import PreserveLoggingContext
 
 from ._base import BaseHandler
@@ -194,6 +194,8 @@ class ProfileHandler(BaseHandler):
         if not self.hs.is_mine(user):
             return
 
+        self.ratelimit(user.to_string())
+
         joins = yield self.store.get_rooms_for_user_where_membership_is(
             user.to_string(),
             [Membership.JOIN],
@@ -201,7 +203,7 @@ class ProfileHandler(BaseHandler):
 
         for j in joins:
             content = {
-                "membership": j.content["membership"],
+                "membership": Membership.JOIN,
             }
 
             yield self.distributor.fire(
@@ -210,9 +212,9 @@ class ProfileHandler(BaseHandler):
 
             msg_handler = self.hs.get_handlers().message_handler
             yield msg_handler.create_and_send_event({
-                "type": j.type,
+                "type": EventTypes.Member,
                 "room_id": j.room_id,
-                "state_key": j.state_key,
+                "state_key": user.to_string(),
                 "content": content,
-                "sender": j.state_key,
+                "sender": user.to_string()
-            })
+            }, ratelimit=False)

synapse/handlers/register.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/handlers/room.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -130,6 +130,7 @@ class RoomCreationHandler(BaseHandler):
                 "type": EventTypes.Name,
                 "room_id": room_id,
                 "sender": user_id,
+                "state_key": "",
                 "content": {"name": name},
             })
 
@@ -139,6 +140,7 @@ class RoomCreationHandler(BaseHandler):
                 "type": EventTypes.Topic,
                 "room_id": room_id,
                 "sender": user_id,
+                "state_key": "",
                 "content": {"topic": topic},
             })
 
@@ -147,7 +149,7 @@ class RoomCreationHandler(BaseHandler):
                 "type": EventTypes.Member,
                 "state_key": invitee,
                 "room_id": room_id,
-                "user_id": user_id,
+                "sender": user_id,
                 "content": {"membership": Membership.INVITE},
             })
 
@@ -243,14 +245,12 @@ class RoomMemberHandler(BaseHandler):
         self.distributor.declare("user_left_room")
 
     @defer.inlineCallbacks
-    def get_room_members(self, room_id, membership=Membership.JOIN):
+    def get_room_members(self, room_id):
         hs = self.hs
 
-        memberships = yield self.store.get_room_members(
+        users = yield self.store.get_users_in_room(room_id)
-            room_id=room_id, membership=membership
-        )
 
-        defer.returnValue([hs.parse_userid(m.user_id) for m in memberships])
+        defer.returnValue([hs.parse_userid(u) for u in users])
 
     @defer.inlineCallbacks
     def fetch_room_distributions_into(self, room_id, localusers=None,
@@ -390,6 +390,11 @@ class RoomMemberHandler(BaseHandler):
 
         host = hosts[0]
 
+        # If event doesn't include a display name, add one.
+        yield self.distributor.fire(
+            "collect_presencelike_data", joinee, content
+        )
+
         content.update({"membership": Membership.JOIN})
         builder = self.event_builder_factory.new({
             "type": EventTypes.Member,
@@ -420,10 +425,22 @@ class RoomMemberHandler(BaseHandler):
             event.room_id,
             self.hs.hostname
         )
+        if not is_host_in_room:
+            # is *anyone* in the room?
+            room_member_keys = [
+                v for (k, v) in context.current_state.keys() if (
+                    k == "m.room.member"
+                )
+            ]
+            if len(room_member_keys) == 0:
+                # has the room been created so we can join it?
+                create_event = context.current_state.get(("m.room.create", ""))
+                if create_event:
+                    is_host_in_room = True
 
         if is_host_in_room:
             should_do_dance = False
-        elif room_host:
+        elif room_host:  # TODO: Shouldn't this be remote_room_host?
             should_do_dance = True
         else:
             # TODO(markjh): get prev_state from snapshot
@@ -437,7 +454,8 @@ class RoomMemberHandler(BaseHandler):
                 should_do_dance = not self.hs.is_mine(inviter)
                 room_host = inviter.domain
             else:
-                should_do_dance = False
+                # return the same error as join_room_alias does
+                raise SynapseError(404, "No known servers")
 
         if should_do_dance:
             handler = self.hs.get_handlers().federation_handler
@@ -524,11 +542,10 @@ class RoomListHandler(BaseHandler):
     def get_public_room_list(self):
         chunk = yield self.store.get_rooms(is_public=True)
         for room in chunk:
-            joined_members = yield self.store.get_room_members(
+            joined_users = yield self.store.get_users_in_room(
                 room_id=room["room_id"],
-                membership=Membership.JOIN
             )
-            room["num_joined_members"] = len(joined_members)
+            room["num_joined_members"] = len(joined_users)
         # FIXME (erikj): START is no longer a valid value
         defer.returnValue({"start": "START", "end": "END", "chunk": chunk})
 

synapse/handlers/typing.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -83,9 +83,15 @@ class TypingNotificationHandler(BaseHandler):
         if member in self._member_typing_timer:
             self.clock.cancel_call_later(self._member_typing_timer[member])
 
+        def _cb():
+            logger.debug(
+                "%s has timed out in %s", target_user.to_string(), room_id
+            )
+            self._stopped_typing(member)
+
         self._member_typing_until[member] = until
         self._member_typing_timer[member] = self.clock.call_later(
-            timeout / 1000, lambda: self._stopped_typing(member)
+            timeout / 1000.0, _cb
         )
 
         if was_present:

synapse/http/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/http/agent_name.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse import __version__
+
+AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii")

synapse/http/client.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 
+from synapse.http.agent_name import AGENT_NAME
 from twisted.internet import defer, reactor
 from twisted.web.client import (
     Agent, readBody, FileBodyProducer, PartialDownloadError
@@ -51,7 +52,8 @@ class SimpleHttpClient(object):
             "POST",
             uri.encode("ascii"),
             headers=Headers({
-                "Content-Type": ["application/x-www-form-urlencoded"]
+                b"Content-Type": [b"application/x-www-form-urlencoded"],
+                b"User-Agent": [AGENT_NAME],
             }),
             bodyProducer=FileBodyProducer(StringIO(query_bytes))
         )
@@ -105,6 +107,9 @@ class SimpleHttpClient(object):
         response = yield self.agent.request(
             "GET",
             uri.encode("ascii"),
+            headers=Headers({
+                b"User-Agent": [AGENT_NAME],
+            })
         )
 
         body = yield readBody(response)
@@ -127,7 +132,8 @@ class CaptchaServerHttpClient(SimpleHttpClient):
             url.encode("ascii"),
             bodyProducer=FileBodyProducer(StringIO(query_bytes)),
             headers=Headers({
-                "Content-Type": ["application/x-www-form-urlencoded"]
+                b"Content-Type": [b"application/x-www-form-urlencoded"],
+                b"User-Agent": [AGENT_NAME],
             })
         )
 

synapse/http/endpoint.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/http/matrixfederationclient.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ from twisted.web.client import readBody, _AgentBase, _URI
 from twisted.web.http_headers import Headers
 from twisted.web._newclient import ResponseDone
 
+from synapse.http.agent_name import AGENT_NAME
 from synapse.http.endpoint import matrix_federation_endpoint
 from synapse.util.async import sleep
 from synapse.util.logcontext import PreserveLoggingContext
@@ -71,6 +72,7 @@ class MatrixFederationHttpClient(object):
             requests.
     """
 
+
     def __init__(self, hs):
         self.hs = hs
         self.signing_key = hs.config.signing_key[0]
@@ -83,7 +85,7 @@ class MatrixFederationHttpClient(object):
                         query_bytes=b"", retry_on_dns_fail=True):
         """ Creates and sends a request to the given url
         """
-        headers_dict[b"User-Agent"] = [b"Synapse"]
+        headers_dict[b"User-Agent"] = [AGENT_NAME]
         headers_dict[b"Host"] = [destination]
 
         url_bytes = urlparse.urlunparse(

synapse/http/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,14 +14,16 @@
 # limitations under the License.
 
 
-from syutil.jsonutil import (
+from synapse.http.agent_name import AGENT_NAME
-    encode_canonical_json, encode_pretty_printed_json
-)
 from synapse.api.errors import (
     cs_exception, SynapseError, CodeMessageException
 )
 from synapse.util.logcontext import LoggingContext
 
+from syutil.jsonutil import (
+    encode_canonical_json, encode_pretty_printed_json
+)
+
 from twisted.internet import defer, reactor
 from twisted.web import server, resource
 from twisted.web.server import NOT_DONE_YET
@@ -230,6 +232,8 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
 
     request.setResponseCode(code, message=response_code_message)
     request.setHeader(b"Content-Type", b"application/json")
+    request.setHeader(b"Server", AGENT_NAME)
+    request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
 
     if send_cors:
         request.setHeader("Access-Control-Allow-Origin", "*")

synapse/http/server_key_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/media/v0/content_repository.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/media/v1/__init__.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import PIL.Image
+
+# check for JPEG support.
+try:
+    PIL.Image._getdecoder("rgb", "jpeg", None)
+except IOError as e:
+    if str(e).startswith("decoder jpeg not available"):
+        raise Exception(
+            "FATAL: jpeg codec not supported. Install pillow correctly! "
+            " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
+            " pip install pillow --user'"
+        )
+except Exception:
+    # any other exception is fine
+    pass
+
+
+# check for PNG support.
+try:
+    PIL.Image._getdecoder("rgb", "zip", None)
+except IOError as e:
+    if str(e).startswith("decoder zip not available"):
+        raise Exception(
+            "FATAL: zip codec not supported. Install pillow correctly! "
+            " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
+            " pip install pillow --user'"
+        )
+except Exception:
+    # any other exception is fine
+    pass

synapse/media/v1/base_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -139,6 +139,7 @@ class BaseMediaResource(Resource):
             @download.addBoth
             def callback(media_info):
                 del self.downloads[key]
+                return media_info
         return download
 
     @defer.inlineCallbacks
@@ -201,7 +202,8 @@ class BaseMediaResource(Resource):
         defer.returnValue(media_info)
 
     @defer.inlineCallbacks
-    def _respond_with_file(self, request, media_type, file_path):
+    def _respond_with_file(self, request, media_type, file_path,
+                           file_size=None):
         logger.debug("Responding with %r", file_path)
 
         if os.path.isfile(file_path):
@@ -215,13 +217,20 @@ class BaseMediaResource(Resource):
             request.setHeader(
                 b"Cache-Control", b"public,max-age=86400,s-maxage=86400"
             )
+            if file_size is None:
+                stat = os.stat(file_path)
+                file_size = stat.st_size
+
+            request.setHeader(
+                b"Content-Length", b"%d" % (file_size,)
+            )
 
             with open(file_path, "rb") as f:
                 yield FileSender().beginFileTransfer(f, request)
 
             request.finish()
         else:
-            self._respond_404()
+            self._respond_404(request)
 
     def _get_thumbnail_requirements(self, media_type):
         if media_type == "image/jpeg":

synapse/media/v1/download_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,23 +46,29 @@ class DownloadResource(BaseMediaResource):
     def _respond_local_file(self, request, media_id):
         media_info = yield self.store.get_local_media(media_id)
         if not media_info:
-            self._respond_404()
+            self._respond_404(request)
             return
 
         media_type = media_info["media_type"]
+        media_length = media_info["media_length"]
         file_path = self.filepaths.local_media_filepath(media_id)
 
-        yield self._respond_with_file(request, media_type, file_path)
+        yield self._respond_with_file(
+            request, media_type, file_path, media_length
+        )
 
     @defer.inlineCallbacks
     def _respond_remote_file(self, request, server_name, media_id):
         media_info = yield self._get_remote_media(server_name, media_id)
 
         media_type = media_info["media_type"]
+        media_length = media_info["media_length"]
         filesystem_id = media_info["filesystem_id"]
 
         file_path = self.filepaths.remote_media_filepath(
             server_name, filesystem_id
         )
 
-        yield self._respond_with_file(request, media_type, file_path)
+        yield self._respond_with_file(
+            request, media_type, file_path, media_length
+        )

synapse/media/v1/filepath.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/media/v1/media_repository.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/media/v1/thumbnail_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -100,11 +100,12 @@ class ThumbnailResource(BaseMediaResource):
             t_type = thumbnail_info["thumbnail_type"]
             t_method = thumbnail_info["thumbnail_method"]
             file_id = thumbnail_info["filesystem_id"]
+            t_length = thumbnail_info["thumbnail_length"]
 
             file_path = self.filepaths.remote_media_thumbnail(
                 server_name, file_id, t_width, t_height, t_type, t_method,
             )
-            yield self._respond_with_file(request, t_type, file_path)
+            yield self._respond_with_file(request, t_type, file_path, t_length)
         else:
             yield self._respond_default_thumbnail(
                 request, media_info, width, height, method, m_type,
@@ -139,11 +140,12 @@ class ThumbnailResource(BaseMediaResource):
         t_height = thumbnail_info["thumbnail_height"]
         t_type = thumbnail_info["thumbnail_type"]
         t_method = thumbnail_info["thumbnail_method"]
+        t_length = thumbnail_info["thumbnail_length"]
 
         file_path = self.filepaths.default_thumbnail(
             top_level_type, sub_type, t_width, t_height, t_type, t_method,
         )
-        yield self.respond_with_file(request, t_type, file_path)
+        yield self.respond_with_file(request, t_type, file_path, t_length)
 
     def _select_thumbnail(self, desired_width, desired_height, desired_method,
                           desired_type, thumbnail_infos):
@@ -165,18 +167,27 @@ class ThumbnailResource(BaseMediaResource):
                         aspect_quality, size_quality, type_quality,
                         length_quality, info
                     ))
-            return min(info_list)[-1]
+            if info_list:
+                return min(info_list)[-1]
         else:
             info_list = []
+            info_list2 = []
             for info in thumbnail_infos:
                 t_w = info["thumbnail_width"]
                 t_h = info["thumbnail_height"]
                 t_method = info["thumbnail_method"]
+                size_quality = abs((d_w - t_w) * (d_h - t_h))
+                type_quality = desired_type != info["thumbnail_type"]
+                length_quality = info["thumbnail_length"]
                 if t_method == "scale" and (t_w >= d_w or t_h >= d_h):
-                    size_quality = abs((d_w - t_w) * (d_h - t_h))
-                    type_quality = desired_type != info["thumbnail_type"]
-                    length_quality = info["thumbnail_length"]
                     info_list.append((
                         size_quality, type_quality, length_quality, info
                     ))
-            return min(info_list)[-1]
+                elif t_method == "scale":
+                    info_list2.append((
+                        size_quality, type_quality, length_quality, info
+                    ))
+            if info_list:
+                return min(info_list)[-1]
+            else:
+                return min(info_list2)[-1]

synapse/media/v1/thumbnailer.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,7 +48,7 @@ class Thumbnailer(object):
 
     def scale(self, output_path, width, height, output_type):
         """Rescales the image to the given dimensions"""
-        scaled = self.image.resize((width, height), Image.BILINEAR)
+        scaled = self.image.resize((width, height), Image.ANTIALIAS)
         return self.save_image(scaled, output_type, output_path)
 
     def crop(self, output_path, width, height, output_type):
@@ -65,7 +65,7 @@ class Thumbnailer(object):
         if width * self.height > height * self.width:
             scaled_height = (width * self.height) // self.width
             scaled_image = self.image.resize(
-                (width, scaled_height), Image.BILINEAR
+                (width, scaled_height), Image.ANTIALIAS
             )
             crop_top = (scaled_height - height) // 2
             crop_bottom = height + crop_top
@@ -73,7 +73,7 @@ class Thumbnailer(object):
         else:
             scaled_width = (height * self.width) // self.height
             scaled_image = self.image.resize(
-                (scaled_width, height), Image.BILINEAR
+                (scaled_width, height), Image.ANTIALIAS
             )
             crop_left = (scaled_width - width) // 2
             crop_right = width + crop_left
@@ -82,7 +82,7 @@ class Thumbnailer(object):
 
     def save_image(self, output_image, output_type, output_path):
         output_bytes_io = BytesIO()
-        output_image.save(output_bytes_io, self.FORMATS[output_type])
+        output_image.save(output_bytes_io, self.FORMATS[output_type], quality=70)
         output_bytes = output_bytes_io.getvalue()
         with open(output_path, "wb") as output_file:
             output_file.write(output_bytes)

synapse/media/v1/upload_resource.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/notifier.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/python_dependencies.py

@@ -0,0 +1,80 @@
+import logging
+from distutils.version import LooseVersion
+
+logger = logging.getLogger(__name__)
+
+REQUIREMENTS = {
+    "syutil==0.0.2": ["syutil"],
+    "matrix_angular_sdk==0.6.0": ["syweb==0.6.0"],
+    "Twisted>=14.0.0": ["twisted>=14.0.0"],
+    "service_identity>=1.0.0": ["service_identity>=1.0.0"],
+    "pyopenssl>=0.14": ["OpenSSL>=0.14"],
+    "pyyaml": ["yaml"],
+    "pyasn1": ["pyasn1"],
+    "pynacl": ["nacl"],
+    "daemonize": ["daemonize"],
+    "py-bcrypt": ["bcrypt"],
+    "frozendict>=0.4": ["frozendict"],
+    "pillow": ["PIL"],
+}
+
+
+class MissingRequirementError(Exception):
+    pass
+
+
+def check_requirements():
+    """Checks that all the modules needed by synapse have been correctly
+    installed and are at the correct version"""
+    for dependency, module_requirements in REQUIREMENTS.items():
+        for module_requirement in module_requirements:
+            if ">=" in module_requirement:
+                module_name, required_version = module_requirement.split(">=")
+                version_test = ">="
+            elif "==" in module_requirement:
+                module_name, required_version = module_requirement.split("==")
+                version_test = "=="
+            else:
+                module_name = module_requirement
+                version_test = None
+
+            try:
+                module = __import__(module_name)
+            except ImportError:
+                logging.exception(
+                    "Can't import %r which is part of %r",
+                    module_name, dependency
+                )
+                raise MissingRequirementError(
+                    "Can't import %r which is part of %r"
+                    % (module_name, dependency)
+                )
+            version = getattr(module, "__version__", None)
+            file_path = getattr(module, "__file__", None)
+            logger.info(
+                "Using %r version %r from %r to satisfy %r",
+                module_name, version, file_path, dependency
+            )
+
+            if version_test == ">=":
+                if version is None:
+                    raise MissingRequirementError(
+                        "Version of %r isn't set as __version__ of module %r"
+                        % (dependency, module_name)
+                    )
+                if LooseVersion(version) < LooseVersion(required_version):
+                    raise MissingRequirementError(
+                        "Version of %r in %r is too old. %r < %r"
+                        % (dependency, file_path, version, required_version)
+                    )
+            elif version_test == "==":
+                if version is None:
+                    raise MissingRequirementError(
+                        "Version of %r isn't set as __version__ of module %r"
+                        % (dependency, module_name)
+                    )
+                if LooseVersion(version) != LooseVersion(required_version):
+                    raise MissingRequirementError(
+                        "Unexpected version of %r in %r. %r != %r"
+                        % (dependency, file_path, version, required_version)
+                    )

synapse/rest/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/admin.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/directory.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/events.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -44,8 +44,11 @@ class EventStreamRestServlet(RestServlet):
                 except ValueError:
                     raise SynapseError(400, "timeout must be in milliseconds.")
 
+            as_client_event = "raw" not in request.args
+
             chunk = yield handler.get_stream(
-                auth_user.to_string(), pagin_config, timeout=timeout
+                auth_user.to_string(), pagin_config, timeout=timeout,
+                as_client_event=as_client_event
             )
         except:
             logger.exception("Event stream failed")

synapse/rest/initial_sync.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,12 +27,15 @@ class InitialSyncRestServlet(RestServlet):
     def on_GET(self, request):
         user = yield self.auth.get_user_by_req(request)
         with_feedback = "feedback" in request.args
+        as_client_event = "raw" not in request.args
         pagination_config = PaginationConfig.from_request(request)
         handler = self.handlers.message_handler
         content = yield handler.snapshot_all_rooms(
             user_id=user.to_string(),
             pagin_config=pagination_config,
-            feedback=with_feedback)
+            feedback=with_feedback,
+            as_client_event=as_client_event
+        )
 
         defer.returnValue((200, content))
 

synapse/rest/login.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/presence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/profile.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/register.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/room.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -314,12 +314,15 @@ class RoomMessageListRestServlet(RestServlet):
             request, default_limit=10,
         )
         with_feedback = "feedback" in request.args
+        as_client_event = "raw" not in request.args
         handler = self.handlers.message_handler
         msgs = yield handler.get_messages(
             room_id=room_id,
             user_id=user.to_string(),
             pagin_config=pagination_config,
-            feedback=with_feedback)
+            feedback=with_feedback,
+            as_client_event=as_client_event
+        )
 
         defer.returnValue((200, msgs))
 

synapse/rest/transactions.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/rest/voip.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/server.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -151,8 +151,8 @@ class BaseHomeServer(object):
         object."""
         return EventID.from_string(s)
 
-    def serialize_event(self, e):
+    def serialize_event(self, e, as_client_event=True):
-        return serialize_event(self, e)
+        return serialize_event(self, e, as_client_event)
 
     def get_ip_from_request(self, request):
         # May be an X-Forwarding-For header depending on config

synapse/state.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -68,7 +68,7 @@ SCHEMAS = [
 
 # Remember to update this number every time an incompatible change is made to
 # database schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 10
+SCHEMA_VERSION = 11
 
 
 class _RollbackButIsFineException(Exception):
@@ -146,9 +146,7 @@ class DataStore(RoomMemberStore, RoomStore,
         elif event.type == EventTypes.Redaction:
             self._store_redaction(txn, event)
 
-        outlier = False
+        outlier = event.internal_metadata.is_outlier()
-        if hasattr(event.internal_metadata, "outlier"):
-            outlier = event.internal_metadata.outlier
 
         event_dict = {
             k: v

synapse/storage/_base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -64,7 +64,7 @@ class LoggingTransaction(object):
             # Don't let logging failures stop SQL from working
             pass
 
-        start = time.clock() * 1000
+        start = time.time() * 1000
         try:
             return self.txn.execute(
                 sql, *args, **kwargs
@@ -73,7 +73,7 @@ class LoggingTransaction(object):
                 logger.exception("[SQL FAIL] {%s}", self.name)
                 raise
         finally:
-            end = time.clock() * 1000
+            end = time.time() * 1000
             sql_logger.debug("[SQL time] {%s} %f", self.name, end - start)
 
 
@@ -93,7 +93,7 @@ class SQLBaseStore(object):
         def inner_func(txn, *args, **kwargs):
             with LoggingContext("runInteraction") as context:
                 current_context.copy_to(context)
-                start = time.clock() * 1000
+                start = time.time() * 1000
                 txn_id = SQLBaseStore._TXN_ID
 
                 # We don't really need these to be unique, so lets stop it from
@@ -109,7 +109,7 @@ class SQLBaseStore(object):
                     logger.exception("[TXN FAIL] {%s}", name)
                     raise
                 finally:
-                    end = time.clock() * 1000
+                    end = time.time() * 1000
                     transaction_logger.debug(
                         "[TXN END] {%s} %f",
                         name, end - start
@@ -479,23 +479,31 @@ class SQLBaseStore(object):
 
         return self.runInteraction("_simple_max_id", func)
 
-    def _get_events(self, event_ids):
+    def _get_events(self, event_ids, check_redacted=True,
+                    get_prev_content=False):
         return self.runInteraction(
-            "_get_events", self._get_events_txn, event_ids
+            "_get_events", self._get_events_txn, event_ids,
+            check_redacted=check_redacted, get_prev_content=get_prev_content,
         )
 
-    def _get_events_txn(self, txn, event_ids):
+    def _get_events_txn(self, txn, event_ids, check_redacted=True,
-        events = []
+                        get_prev_content=False):
-        for e_id in event_ids:
+        if not event_ids:
-            ev = self._get_event_txn(txn, e_id)
+            return []
 
-            if ev:
+        events = [
-                events.append(ev)
+            self._get_event_txn(
+                txn, event_id,
+                check_redacted=check_redacted,
+                get_prev_content=get_prev_content
+            )
+            for event_id in event_ids
+        ]
 
-        return events
+        return [e for e in events if e]
 
     def _get_event_txn(self, txn, event_id, check_redacted=True,
-                       get_prev_content=True):
+                       get_prev_content=False):
         sql = (
             "SELECT internal_metadata, json, r.event_id FROM event_json as e "
             "LEFT JOIN redactions as r ON e.event_id = r.redacts "
@@ -512,6 +520,14 @@ class SQLBaseStore(object):
 
         internal_metadata, js, redacted = res
 
+        return self._get_event_from_row_txn(
+            txn, internal_metadata, js, redacted,
+            check_redacted=check_redacted,
+            get_prev_content=get_prev_content,
+        )
+
+    def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted,
+                                check_redacted=True, get_prev_content=False):
         d = json.loads(js)
         internal_metadata = json.loads(internal_metadata)
 
@@ -533,11 +549,13 @@ class SQLBaseStore(object):
                 ev.unsigned["redacted_because"] = because
 
         if get_prev_content and "replaces_state" in ev.unsigned:
-            ev.unsigned["prev_content"] = self._get_event_txn(
+            prev = self._get_event_txn(
                 txn,
                 ev.unsigned["replaces_state"],
                 get_prev_content=False,
-            ).get_dict()["content"]
+            )
+            if prev:
+                ev.unsigned["prev_content"] = prev.get_dict()["content"]
 
         return ev
 

synapse/storage/directory.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/event_federation.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,39 +32,33 @@ class EventFederationStore(SQLBaseStore):
     and backfilling from another server respectively.
     """
 
-    def get_auth_chain(self, event_id):
+    def get_auth_chain(self, event_ids):
         return self.runInteraction(
             "get_auth_chain",
             self._get_auth_chain_txn,
-            event_id
+            event_ids
         )
 
-    def _get_auth_chain_txn(self, txn, event_id):
+    def _get_auth_chain_txn(self, txn, event_ids):
-        results = self._get_auth_chain_ids_txn(txn, event_id)
+        results = self._get_auth_chain_ids_txn(txn, event_ids)
 
-        sql = "SELECT * FROM events WHERE event_id = ?"
+        return self._get_events_txn(txn, results)
-        rows = []
-        for ev_id in results:
-            c = txn.execute(sql, (ev_id,))
-            rows.extend(self.cursor_to_dict(c))
 
-        return self._parse_events_txn(txn, rows)
+    def get_auth_chain_ids(self, event_ids):
-
-    def get_auth_chain_ids(self, event_id):
         return self.runInteraction(
             "get_auth_chain_ids",
             self._get_auth_chain_ids_txn,
-            event_id
+            event_ids
         )
 
-    def _get_auth_chain_ids_txn(self, txn, event_id):
+    def _get_auth_chain_ids_txn(self, txn, event_ids):
         results = set()
 
         base_sql = (
             "SELECT auth_id FROM event_auth WHERE %s"
         )
 
-        front = set([event_id])
+        front = set(event_ids)
         while front:
             sql = base_sql % (
                 " OR ".join(["event_id=?"] * len(front)),

synapse/storage/feedback.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/keys.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/media_repository.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

synapse/storage/presence.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.