Merge branch 'develop' into pushers

Conflicts:
	synapse/rest/__init__.py

contrib/cmdclient/console.py

@@ -0,0 +1,747 @@
+#!/usr/bin/env python
+
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Starts a synapse client console. """
+
+from twisted.internet import reactor, defer, threads
+from http import TwistedHttpClient
+
+import argparse
+import cmd
+import getpass
+import json
+import shlex
+import sys
+import time
+import urllib
+import urlparse
+
+import nacl.signing
+import nacl.encoding
+
+from syutil.crypto.jsonsign import verify_signed_json, SignatureVerifyException
+
+CONFIG_JSON = "cmdclient_config.json"
+
+TRUSTED_ID_SERVERS = [
+    'localhost:8001'
+]
+
+class SynapseCmd(cmd.Cmd):
+
+    """Basic synapse command-line processor.
+
+    This processes commands from the user and calls the relevant HTTP methods.
+    """
+
+    def __init__(self, http_client, server_url, identity_server_url, username, token):
+        cmd.Cmd.__init__(self)
+        self.http_client = http_client
+        self.http_client.verbose = True
+        self.config = {
+            "url": server_url,
+            "identityServerUrl": identity_server_url,
+            "user": username,
+            "token": token,
+            "verbose": "on",
+            "complete_usernames": "on",
+            "send_delivery_receipts": "on"
+        }
+        self.path_prefix = "/_matrix/client/api/v1"
+        self.event_stream_token = "END"
+        self.prompt = ">>> "
+
+    def do_EOF(self, line):  # allows CTRL+D quitting
+        return True
+
+    def emptyline(self):
+        pass  # else it repeats the previous command
+
+    def _usr(self):
+        return self.config["user"]
+
+    def _tok(self):
+        return self.config["token"]
+
+    def _url(self):
+        return self.config["url"] + self.path_prefix
+
+    def _identityServerUrl(self):
+        return self.config["identityServerUrl"]
+
+    def _is_on(self, config_name):
+        if config_name in self.config:
+            return self.config[config_name] == "on"
+        return False
+
+    def _domain(self):
+        if "user" not in self.config or not self.config["user"]:
+            return None
+        return self.config["user"].split(":")[1]
+
+    def do_config(self, line):
+        """ Show the config for this client: "config"
+        Edit a key value mapping: "config key value" e.g. "config token 1234"
+        Config variables:
+            user: The username to auth with.
+            token: The access token to auth with.
+            url: The url of the server.
+            verbose: [on|off] The verbosity of requests/responses.
+            complete_usernames: [on|off] Auto complete partial usernames by
+            assuming they are on the same homeserver as you.
+            E.g. name >> @name:yourhost
+            send_delivery_receipts: [on|off] Automatically send receipts to
+            messages when performing a 'stream' command.
+        Additional key/values can be added and can be substituted into requests
+        by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'.
+        """
+        if len(line) == 0:
+            print json.dumps(self.config, indent=4)
+            return
+
+        try:
+            args = self._parse(line, ["key", "val"], force_keys=True)
+
+            # make sure restricted config values are checked
+            config_rules = [  # key, valid_values
+                ("verbose", ["on", "off"]),
+                ("complete_usernames", ["on", "off"]),
+                ("send_delivery_receipts", ["on", "off"])
+            ]
+            for key, valid_vals in config_rules:
+                if key == args["key"] and args["val"] not in valid_vals:
+                    print "%s value must be one of %s" % (args["key"],
+                                                          valid_vals)
+                    return
+
+            # toggle the http client verbosity
+            if args["key"] == "verbose":
+                self.http_client.verbose = "on" == args["val"]
+
+            # assign the new config
+            self.config[args["key"]] = args["val"]
+            print json.dumps(self.config, indent=4)
+
+            save_config(self.config)
+        except Exception as e:
+            print e
+
+    def do_register(self, line):
+        """Registers for a new account: "register <userid> <noupdate>"
+        <userid> : The desired user ID
+        <noupdate> : Do not automatically clobber config values.
+        """
+        args = self._parse(line, ["userid", "noupdate"])
+
+        password = None
+        pwd = None
+        pwd2 = "_"
+        while pwd != pwd2:
+            pwd = getpass.getpass("Type a password for this user: ")
+            pwd2 = getpass.getpass("Retype the password: ")
+            if pwd != pwd2 or len(pwd) == 0:
+                print "Password mismatch."
+                pwd = None
+            else:
+                password = pwd
+
+        body = {
+            "type": "m.login.password"
+        }
+        if "userid" in args:
+            body["user"] = args["userid"]
+        if password:
+            body["password"] = password
+
+        reactor.callFromThread(self._do_register, body,
+                               "noupdate" not in args)
+
+    @defer.inlineCallbacks
+    def _do_register(self, data, update_config):
+        # check the registration flows
+        url = self._url() + "/register"
+        json_res = yield self.http_client.do_request("GET", url)
+        print json.dumps(json_res, indent=4)
+
+        passwordFlow = None
+        for flow in json_res["flows"]:
+            if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
+                print "Unable to register: Home server requires captcha."
+                return
+            if flow["type"] == "m.login.password" and "stages" not in flow:
+                passwordFlow = flow
+                break
+
+        if not passwordFlow:
+            return
+
+        json_res = yield self.http_client.do_request("POST", url, data=data)
+        print json.dumps(json_res, indent=4)
+        if update_config and "user_id" in json_res:
+            self.config["user"] = json_res["user_id"]
+            self.config["token"] = json_res["access_token"]
+            save_config(self.config)
+
+    def do_login(self, line):
+        """Login as a specific user: "login @bob:localhost"
+        You MAY be prompted for a password, or instructed to visit a URL.
+        """
+        try:
+            args = self._parse(line, ["user_id"], force_keys=True)
+            can_login = threads.blockingCallFromThread(
+                reactor,
+                self._check_can_login)
+            if can_login:
+                p = getpass.getpass("Enter your password: ")
+                user = args["user_id"]
+                if self._is_on("complete_usernames") and not user.startswith("@"):
+                    domain = self._domain()
+                    if domain:
+                        user = "@" + user + ":" + domain
+                
+                reactor.callFromThread(self._do_login, user, p)
+                #print " got %s " % p
+        except Exception as e:
+            print e
+
+    @defer.inlineCallbacks
+    def _do_login(self, user, password):
+        path = "/login"
+        data = {
+            "user": user,
+            "password": password,
+            "type": "m.login.password"
+        }
+        url = self._url() + path
+        json_res = yield self.http_client.do_request("POST", url, data=data)
+        print json_res
+
+        if "access_token" in json_res:
+            self.config["user"] = user
+            self.config["token"] = json_res["access_token"]
+            save_config(self.config)
+            print "Login successful."
+
+    @defer.inlineCallbacks
+    def _check_can_login(self):
+        path = "/login"
+        # ALWAYS check that the home server can handle the login request before
+        # submitting!
+        url = self._url() + path
+        json_res = yield self.http_client.do_request("GET", url)
+        print json_res
+
+        if "flows" not in json_res:
+            print "Failed to find any login flows."
+            defer.returnValue(False)
+
+        flow = json_res["flows"][0] # assume first is the one we want.
+        if ("type" not in flow or "m.login.password" != flow["type"] or
+                "stages" in flow):
+            fallback_url = self._url() + "/login/fallback"
+            print ("Unable to login via the command line client. Please visit "
+                "%s to login." % fallback_url)
+            defer.returnValue(False)
+        defer.returnValue(True)
+
+    def do_emailrequest(self, line):
+        """Requests the association of a third party identifier
+        <address> The email address)
+        <clientSecret> A string of characters generated when requesting an email that you'll supply in subsequent calls to identify yourself
+        <sendAttempt> The number of times the user has requested an email. Leave this the same between requests to retry the request at the transport level. Increment it to request that the email be sent again.
+        """
+        args = self._parse(line, ['address', 'clientSecret', 'sendAttempt'])
+
+        postArgs = {'email': args['address'], 'clientSecret': args['clientSecret'], 'sendAttempt': args['sendAttempt']}
+
+        reactor.callFromThread(self._do_emailrequest, postArgs)
+
+    @defer.inlineCallbacks
+    def _do_emailrequest(self, args):
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/requestToken"
+
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
+        if 'sid' in json_res:
+            print "Token sent. Your session ID is %s" % (json_res['sid'])
+
+    def do_emailvalidate(self, line):
+        """Validate and associate a third party ID
+        <sid> The session ID (sid) given to you in the response to requestToken
+        <token> The token sent to your third party identifier address
+        <clientSecret> The same clientSecret you supplied in requestToken
+        """
+        args = self._parse(line, ['sid', 'token', 'clientSecret'])
+
+        postArgs = { 'sid' : args['sid'], 'token' : args['token'], 'clientSecret': args['clientSecret'] }
+
+        reactor.callFromThread(self._do_emailvalidate, postArgs)
+
+    @defer.inlineCallbacks
+    def _do_emailvalidate(self, args):
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/submitToken"
+
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
+
+    def do_3pidbind(self, line):
+        """Validate and associate a third party ID
+        <sid> The session ID (sid) given to you in the response to requestToken
+        <clientSecret> The same clientSecret you supplied in requestToken
+        """
+        args = self._parse(line, ['sid', 'clientSecret'])
+
+        postArgs = { 'sid' : args['sid'], 'clientSecret': args['clientSecret'] }
+        postArgs['mxid'] = self.config["user"]
+
+        reactor.callFromThread(self._do_3pidbind, postArgs)
+
+    @defer.inlineCallbacks
+    def _do_3pidbind(self, args):
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/3pid/bind"
+
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
+
+    def do_join(self, line):
+        """Joins a room: "join <roomid>" """
+        try:
+            args = self._parse(line, ["roomid"], force_keys=True)
+            self._do_membership_change(args["roomid"], "join", self._usr())
+        except Exception as e:
+            print e
+
+    def do_joinalias(self, line):
+        try:
+            args = self._parse(line, ["roomname"], force_keys=True)
+            path = "/join/%s" % urllib.quote(args["roomname"])
+            reactor.callFromThread(self._run_and_pprint, "POST", path, {})
+        except Exception as e:
+            print e
+
+    def do_topic(self, line):
+        """"topic [set|get] <roomid> [<newtopic>]"
+        Set the topic for a room: topic set <roomid> <newtopic>
+        Get the topic for a room: topic get <roomid>
+        """
+        try:
+            args = self._parse(line, ["action", "roomid", "topic"])
+            if "action" not in args or "roomid" not in args:
+                print "Must specify set|get and a room ID."
+                return
+            if args["action"].lower() not in ["set", "get"]:
+                print "Must specify set|get, not %s" % args["action"]
+                return
+
+            path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
+
+            if args["action"].lower() == "set":
+                if "topic" not in args:
+                    print "Must specify a new topic."
+                    return
+                body = {
+                    "topic": args["topic"]
+                }
+                reactor.callFromThread(self._run_and_pprint, "PUT", path, body)
+            elif args["action"].lower() == "get":
+                reactor.callFromThread(self._run_and_pprint, "GET", path)
+        except Exception as e:
+            print e
+
+    def do_invite(self, line):
+        """Invite a user to a room: "invite <userid> <roomid>" """
+        try:
+            args = self._parse(line, ["userid", "roomid"], force_keys=True)
+
+            user_id = args["userid"]
+
+            reactor.callFromThread(self._do_invite, args["roomid"], user_id)
+        except Exception as e:
+            print e
+
+    @defer.inlineCallbacks
+    def _do_invite(self, roomid, userstring):
+        if (not userstring.startswith('@') and
+                    self._is_on("complete_usernames")):
+            url = self._identityServerUrl()+"/_matrix/identity/api/v1/lookup"
+
+            json_res = yield self.http_client.do_request("GET", url, qparams={'medium':'email','address':userstring})
+
+            mxid = None
+
+            if 'mxid' in json_res and 'signatures' in json_res:
+                url = self._identityServerUrl()+"/_matrix/identity/api/v1/pubkey/ed25519"
+
+                pubKey = None
+                pubKeyObj = yield self.http_client.do_request("GET", url)
+                if 'public_key' in pubKeyObj:
+                    pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
+                else:
+                    print "No public key found in pubkey response!"
+
+                sigValid = False
+
+                if pubKey:
+                    for signame in json_res['signatures']:
+                        if signame not in TRUSTED_ID_SERVERS:
+                            print "Ignoring signature from untrusted server %s" % (signame)
+                        else:
+                            try:
+                                verify_signed_json(json_res, signame, pubKey)
+                                sigValid = True
+                                print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)
+                                break
+                            except SignatureVerifyException as e:
+                                print "Invalid signature from %s" % (signame)
+                                print e
+
+                if sigValid:
+                    print "Resolved 3pid %s to %s" % (userstring, json_res['mxid'])
+                    mxid = json_res['mxid']
+                else:
+                    print "Got association for %s but couldn't verify signature" % (userstring)
+
+            if not mxid:
+                mxid = "@" + userstring + ":" + self._domain()
+
+            self._do_membership_change(roomid, "invite", mxid)
+
+    def do_leave(self, line):
+        """Leaves a room: "leave <roomid>" """
+        try:
+            args = self._parse(line, ["roomid"], force_keys=True)
+            self._do_membership_change(args["roomid"], "leave", self._usr())
+        except Exception as e:
+            print e
+
+    def do_send(self, line):
+        """Sends a message. "send <roomid> <body>" """
+        args = self._parse(line, ["roomid", "body"])
+        txn_id = "txn%s" % int(time.time())
+        path = "/rooms/%s/send/m.room.message/%s" % (urllib.quote(args["roomid"]),
+                                             txn_id)
+        body_json = {
+            "msgtype": "m.text",
+            "body": args["body"]
+        }
+        reactor.callFromThread(self._run_and_pprint, "PUT", path, body_json)
+
+    def do_list(self, line):
+        """List data about a room.
+        "list members <roomid> [query]" - List all the members in this room.
+        "list messages <roomid> [query]" - List all the messages in this room.
+
+        Where [query] will be directly applied as query parameters, allowing
+        you to use the pagination API. E.g. the last 3 messages in this room:
+        "list messages <roomid> from=END&to=START&limit=3"
+        """
+        args = self._parse(line, ["type", "roomid", "qp"])
+        if not "type" in args or not "roomid" in args:
+            print "Must specify type and room ID."
+            return
+        if args["type"] not in ["members", "messages"]:
+            print "Unrecognised type: %s" % args["type"]
+            return
+        room_id = args["roomid"]
+        path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
+
+        qp = {"access_token": self._tok()}
+        if "qp" in args:
+            for key_value_str in args["qp"].split("&"):
+                try:
+                    key_value = key_value_str.split("=")
+                    qp[key_value[0]] = key_value[1]
+                except:
+                    print "Bad query param: %s" % key_value
+                    return
+
+        reactor.callFromThread(self._run_and_pprint, "GET", path,
+                               query_params=qp)
+
+    def do_create(self, line):
+        """Creates a room.
+        "create [public|private] <roomname>" - Create a room <roomname> with the
+                                             specified visibility.
+        "create <roomname>" - Create a room <roomname> with default visibility.
+        "create [public|private]" - Create a room with specified visibility.
+        "create" - Create a room with default visibility.
+        """
+        args = self._parse(line, ["vis", "roomname"])
+        # fixup args depending on which were set
+        body = {}
+        if "vis" in args and args["vis"] in ["public", "private"]:
+            body["visibility"] = args["vis"]
+
+        if "roomname" in args:
+            room_name = args["roomname"]
+            body["room_alias_name"] = room_name
+        elif "vis" in args and args["vis"] not in ["public", "private"]:
+            room_name = args["vis"]
+            body["room_alias_name"] = room_name
+
+        reactor.callFromThread(self._run_and_pprint, "POST", "/createRoom", body)
+
+    def do_raw(self, line):
+        """Directly send a JSON object: "raw <method> <path> <data> <notoken>"
+        <method>: Required. One of "PUT", "GET", "POST", "xPUT", "xGET",
+        "xPOST". Methods with 'x' prefixed will not automatically append the
+        access token.
+        <path>: Required. E.g. "/events"
+        <data>: Optional. E.g. "{ "msgtype":"custom.text", "body":"abc123"}"
+        """
+        args = self._parse(line, ["method", "path", "data"])
+        # sanity check
+        if "method" not in args or "path" not in args:
+            print "Must specify path and method."
+            return
+
+        args["method"] = args["method"].upper()
+        valid_methods = ["PUT", "GET", "POST", "DELETE",
+                         "XPUT", "XGET", "XPOST", "XDELETE"]
+        if args["method"] not in valid_methods:
+            print "Unsupported method: %s" % args["method"]
+            return
+
+        if "data" not in args:
+            args["data"] = None
+        else:
+            try:
+                args["data"] = json.loads(args["data"])
+            except Exception as e:
+                print "Data is not valid JSON. %s" % e
+                return
+
+        qp = {"access_token": self._tok()}
+        if args["method"].startswith("X"):
+            qp = {}  # remove access token
+            args["method"] = args["method"][1:]  # snip the X
+        else:
+            # append any query params the user has set
+            try:
+                parsed_url = urlparse.urlparse(args["path"])
+                qp.update(urlparse.parse_qs(parsed_url.query))
+                args["path"] = parsed_url.path
+            except:
+                pass
+
+        reactor.callFromThread(self._run_and_pprint, args["method"],
+                                                     args["path"],
+                                                     args["data"],
+                                                     query_params=qp)
+
+    def do_stream(self, line):
+        """Stream data from the server: "stream <longpoll timeout ms>" """
+        args = self._parse(line, ["timeout"])
+        timeout = 5000
+        if "timeout" in args:
+            try:
+                timeout = int(args["timeout"])
+            except ValueError:
+                print "Timeout must be in milliseconds."
+                return
+        reactor.callFromThread(self._do_event_stream, timeout)
+
+    @defer.inlineCallbacks
+    def _do_event_stream(self, timeout):
+        res = yield self.http_client.get_json(
+                self._url() + "/events",
+                {
+                    "access_token": self._tok(),
+                    "timeout": str(timeout),
+                    "from": self.event_stream_token
+                })
+        print json.dumps(res, indent=4)
+
+        if "chunk" in res:
+            for event in res["chunk"]:
+                if (event["type"] == "m.room.message" and
+                        self._is_on("send_delivery_receipts") and
+                        event["user_id"] != self._usr()):  # not sent by us
+                    self._send_receipt(event, "d")
+
+        # update the position in the stram
+        if "end" in res:
+            self.event_stream_token = res["end"]
+
+    def _send_receipt(self, event, feedback_type):
+        path = ("/rooms/%s/messages/%s/%s/feedback/%s/%s" %
+               (urllib.quote(event["room_id"]), event["user_id"], event["msg_id"],
+                self._usr(), feedback_type))
+        data = {}
+        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data,
+                               alt_text="Sent receipt for %s" % event["msg_id"])
+
+    def _do_membership_change(self, roomid, membership, userid):
+        path = "/rooms/%s/state/m.room.member/%s" % (urllib.quote(roomid), urllib.quote(userid))
+        data = {
+            "membership": membership
+        }
+        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
+
+    def do_displayname(self, line):
+        """Get or set my displayname: "displayname [new_name]" """
+        args = self._parse(line, ["name"])
+        path = "/profile/%s/displayname" % (self.config["user"])
+
+        if "name" in args:
+            data = {"displayname": args["name"]}
+            reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
+        else:
+            reactor.callFromThread(self._run_and_pprint, "GET", path)
+
+    def _do_presence_state(self, state, line):
+        args = self._parse(line, ["msgstring"])
+        path = "/presence/%s/status" % (self.config["user"])
+        data = {"state": state}
+        if "msgstring" in args:
+            data["status_msg"] = args["msgstring"]
+
+        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
+
+    def do_offline(self, line):
+        """Set my presence state to OFFLINE"""
+        self._do_presence_state(0, line)
+
+    def do_away(self, line):
+        """Set my presence state to AWAY"""
+        self._do_presence_state(1, line)
+
+    def do_online(self, line):
+        """Set my presence state to ONLINE"""
+        self._do_presence_state(2, line)
+
+    def _parse(self, line, keys, force_keys=False):
+        """ Parses the given line.
+
+        Args:
+            line : The line to parse
+            keys : A list of keys to map onto the args
+            force_keys : True to enforce that the line has a value for every key
+        Returns:
+            A dict of key:arg
+        """
+        line_args = shlex.split(line)
+        if force_keys and len(line_args) != len(keys):
+            raise IndexError("Must specify all args: %s" % keys)
+
+        # do $ substitutions
+        for i, arg in enumerate(line_args):
+            for config_key in self.config:
+                if ("$" + config_key) in arg:
+                    arg = arg.replace("$" + config_key,
+                                      self.config[config_key])
+            line_args[i] = arg
+
+        return dict(zip(keys, line_args))
+
+    @defer.inlineCallbacks
+    def _run_and_pprint(self, method, path, data=None,
+                        query_params={"access_token": None}, alt_text=None):
+        """ Runs an HTTP request and pretty prints the output.
+
+        Args:
+            method: HTTP method
+            path: Relative path
+            data: Raw JSON data if any
+            query_params: dict of query parameters to add to the url
+        """
+        url = self._url() + path
+        if "access_token" in query_params:
+            query_params["access_token"] = self._tok()
+
+        json_res = yield self.http_client.do_request(method, url,
+                                                    data=data,
+                                                    qparams=query_params)
+        if alt_text:
+            print alt_text
+        else:
+            print json.dumps(json_res, indent=4)
+
+
+def save_config(config):
+    with open(CONFIG_JSON, 'w') as out:
+        json.dump(config, out)
+
+
+def main(server_url, identity_server_url, username, token, config_path):
+    print "Synapse command line client"
+    print "==========================="
+    print "Server: %s" % server_url
+    print "Type 'help' to get started."
+    print "Close this console with CTRL+C then CTRL+D."
+    if not username or not token:
+        print "-  'register <username>' - Register an account"
+        print "-  'stream' - Connect to the event stream"
+        print "-  'create <roomid>' - Create a room"
+        print "-  'send <roomid> <message>' - Send a message"
+    http_client = TwistedHttpClient()
+
+    # the command line client
+    syn_cmd = SynapseCmd(http_client, server_url, identity_server_url, username, token)
+
+    # load synapse.json config from a previous session
+    global CONFIG_JSON
+    CONFIG_JSON = config_path  # bit cheeky, but just overwrite the global
+    try:
+        with open(config_path, 'r') as config:
+            syn_cmd.config = json.load(config)
+            try:
+                http_client.verbose = "on" == syn_cmd.config["verbose"]
+            except:
+                pass
+            print "Loaded config from %s" % config_path
+    except:
+        pass
+
+    # Twisted-specific: Runs the command processor in Twisted's event loop
+    # to maintain a single thread for both commands and event processing.
+    # If using another HTTP client, just call syn_cmd.cmdloop()
+    reactor.callInThread(syn_cmd.cmdloop)
+    reactor.run()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser("Starts a synapse client.")
+    parser.add_argument(
+        "-s", "--server", dest="server", default="http://localhost:8008",
+        help="The URL of the home server to talk to.")
+    parser.add_argument(
+        "-i", "--identity-server", dest="identityserver", default="http://localhost:8090",
+        help="The URL of the identity server to talk to.")
+    parser.add_argument(
+        "-u", "--username", dest="username",
+        help="Your username on the server.")
+    parser.add_argument(
+        "-t", "--token", dest="token",
+        help="Your access token.")
+    parser.add_argument(
+        "-c", "--config", dest="config", default=CONFIG_JSON,
+        help="The location of the config.json file to read from.")
+    args = parser.parse_args()
+
+    if not args.server:
+        print "You must supply a server URL to communicate with."
+        parser.print_help()
+        sys.exit(1)
+
+    server = args.server
+    if not server.startswith("http://"):
+        server = "http://" + args.server
+
+    main(server, args.identityserver, args.username, args.token, args.config)

contrib/cmdclient/http.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.web.client import Agent, readBody
+from twisted.web.http_headers import Headers
+from twisted.internet import defer, reactor
+
+from pprint import pformat
+
+import json
+import urllib
+
+
+class HttpClient(object):
+    """ Interface for talking json over http
+    """
+
+    def put_json(self, url, data):
+        """ Sends the specifed json data using PUT
+
+        Args:
+            url (str): The URL to PUT data to.
+            data (dict): A dict containing the data that will be used as
+                the request body. This will be encoded as JSON.
+
+        Returns:
+            Deferred: Succeeds when we get *any* HTTP response.
+
+            The result of the deferred is a tuple of `(code, response)`,
+            where `response` is a dict representing the decoded JSON body.
+        """
+        pass
+
+    def get_json(self, url, args=None):
+        """ Get's some json from the given host homeserver and path
+
+        Args:
+            url (str): The URL to GET data from.
+            args (dict): A dictionary used to create query strings, defaults to
+                None.
+                **Note**: The value of each key is assumed to be an iterable
+                and *not* a string.
+
+        Returns:
+            Deferred: Succeeds when we get *any* HTTP response.
+
+            The result of the deferred is a tuple of `(code, response)`,
+            where `response` is a dict representing the decoded JSON body.
+        """
+        pass
+
+
+class TwistedHttpClient(HttpClient):
+    """ Wrapper around the twisted HTTP client api.
+
+    Attributes:
+        agent (twisted.web.client.Agent): The twisted Agent used to send the
+            requests.
+    """
+
+    def __init__(self):
+        self.agent = Agent(reactor)
+
+    @defer.inlineCallbacks
+    def put_json(self, url, data):
+        response = yield self._create_put_request(
+            url,
+            data,
+            headers_dict={"Content-Type": ["application/json"]}
+        )
+        body = yield readBody(response)
+        defer.returnValue((response.code, body))
+
+    @defer.inlineCallbacks
+    def get_json(self, url, args=None):
+        if args:
+            # generates a list of strings of form "k=v".
+            qs = urllib.urlencode(args, True)
+            url = "%s?%s" % (url, qs)
+        response = yield self._create_get_request(url)
+        body = yield readBody(response)
+        defer.returnValue(json.loads(body))
+
+    def _create_put_request(self, url, json_data, headers_dict={}):
+        """ Wrapper of _create_request to issue a PUT request
+        """
+
+        if "Content-Type" not in headers_dict:
+            raise defer.error(
+                RuntimeError("Must include Content-Type header for PUTs"))
+
+        return self._create_request(
+            "PUT",
+            url,
+            producer=_JsonProducer(json_data),
+            headers_dict=headers_dict
+        )
+
+    def _create_get_request(self, url, headers_dict={}):
+        """ Wrapper of _create_request to issue a GET request
+        """
+        return self._create_request(
+            "GET",
+            url,
+            headers_dict=headers_dict
+        )
+
+    @defer.inlineCallbacks
+    def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
+        if qparams:
+            url = "%s?%s" % (url, urllib.urlencode(qparams, True))
+
+        if jsonreq:
+            prod = _JsonProducer(data)
+            headers['Content-Type'] = ["application/json"];
+        else:
+            prod = _RawProducer(data)
+
+        if method in ["POST", "PUT"]:
+            response = yield self._create_request(method, url,
+                    producer=prod,
+                    headers_dict=headers)
+        else:
+            response = yield self._create_request(method, url)
+
+        body = yield readBody(response)
+        defer.returnValue(json.loads(body))
+
+    @defer.inlineCallbacks
+    def _create_request(self, method, url, producer=None, headers_dict={}):
+        """ Creates and sends a request to the given url
+        """
+        headers_dict["User-Agent"] = ["Synapse Cmd Client"]
+
+        retries_left = 5
+        print "%s to %s with headers %s" % (method, url, headers_dict)
+        if self.verbose and producer:
+            if "password" in producer.data:
+                temp = producer.data["password"]
+                producer.data["password"] = "[REDACTED]"
+                print json.dumps(producer.data, indent=4)
+                producer.data["password"] = temp
+            else:
+                print json.dumps(producer.data, indent=4)
+
+        while True:
+            try:
+                response = yield self.agent.request(
+                    method,
+                    url.encode("UTF8"),
+                    Headers(headers_dict),
+                    producer
+                )
+                break
+            except Exception as e:
+                print "uh oh: %s" % e
+                if retries_left:
+                    yield self.sleep(2 ** (5 - retries_left))
+                    retries_left -= 1
+                else:
+                    raise e
+
+        if self.verbose:
+            print "Status %s %s" % (response.code, response.phrase)
+            print pformat(list(response.headers.getAllRawHeaders()))
+        defer.returnValue(response)
+
+    def sleep(self, seconds):
+        d = defer.Deferred()
+        reactor.callLater(seconds, d.callback, seconds)
+        return d
+
+class _RawProducer(object):
+    def __init__(self, data):
+        self.data = data
+        self.body = data
+        self.length = len(self.body)
+
+    def startProducing(self, consumer):
+        consumer.write(self.body)
+        return defer.succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass
+
+class _JsonProducer(object):
+    """ Used by the twisted http client to create the HTTP body from json
+    """
+    def __init__(self, jsn):
+        self.data = jsn
+        self.body = json.dumps(jsn).encode("utf8")
+        self.length = len(self.body)
+
+    def startProducing(self, consumer):
+        consumer.write(self.body)
+        return defer.succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass

contrib/experiments/cursesio.py

@@ -0,0 +1,168 @@
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import curses
+import curses.wrapper
+from curses.ascii import isprint
+
+from twisted.internet import reactor
+
+
+class CursesStdIO():
+    def __init__(self, stdscr, callback=None):
+        self.statusText = "Synapse test app -"
+        self.searchText = ''
+        self.stdscr = stdscr
+
+        self.logLine = ''
+
+        self.callback = callback
+
+        self._setup()
+
+    def _setup(self):
+        self.stdscr.nodelay(1)  # Make non blocking
+
+        self.rows, self.cols = self.stdscr.getmaxyx()
+        self.lines = []
+
+        curses.use_default_colors()
+
+        self.paintStatus(self.statusText)
+        self.stdscr.refresh()
+
+    def set_callback(self, callback):
+        self.callback = callback
+
+    def fileno(self):
+        """ We want to select on FD 0 """
+        return 0
+
+    def connectionLost(self, reason):
+        self.close()
+
+    def print_line(self, text):
+        """ add a line to the internal list of lines"""
+
+        self.lines.append(text)
+        self.redraw()
+
+    def print_log(self, text):
+        self.logLine = text
+        self.redraw()
+
+    def redraw(self):
+        """ method for redisplaying lines
+            based on internal list of lines """
+
+        self.stdscr.clear()
+        self.paintStatus(self.statusText)
+        i = 0
+        index = len(self.lines) - 1
+        while i < (self.rows - 3) and index >= 0:
+            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index],
+                               curses.A_NORMAL)
+            i = i + 1
+            index = index - 1
+
+        self.printLogLine(self.logLine)
+
+        self.stdscr.refresh()
+
+    def paintStatus(self, text):
+        if len(text) > self.cols:
+            raise RuntimeError("TextTooLongError")
+
+        self.stdscr.addstr(
+            self.rows - 2, 0,
+            text + ' ' * (self.cols - len(text)),
+            curses.A_STANDOUT)
+
+    def printLogLine(self, text):
+        self.stdscr.addstr(
+            0, 0,
+            text + ' ' * (self.cols - len(text)),
+            curses.A_STANDOUT)
+
+    def doRead(self):
+        """ Input is ready! """
+        curses.noecho()
+        c = self.stdscr.getch()  # read a character
+
+        if c == curses.KEY_BACKSPACE:
+            self.searchText = self.searchText[:-1]
+
+        elif c == curses.KEY_ENTER or c == 10:
+            text = self.searchText
+            self.searchText = ''
+
+            self.print_line(">> %s" % text)
+
+            try:
+                if self.callback:
+                    self.callback.on_line(text)
+            except Exception as e:
+                self.print_line(str(e))
+
+            self.stdscr.refresh()
+
+        elif isprint(c):
+            if len(self.searchText) == self.cols - 2:
+                return
+            self.searchText = self.searchText + chr(c)
+
+        self.stdscr.addstr(self.rows - 1, 0,
+                           self.searchText + (' ' * (
+                           self.cols - len(self.searchText) - 2)))
+
+        self.paintStatus(self.statusText + ' %d' % len(self.searchText))
+        self.stdscr.move(self.rows - 1, len(self.searchText))
+        self.stdscr.refresh()
+
+    def logPrefix(self):
+        return "CursesStdIO"
+
+    def close(self):
+        """ clean up """
+
+        curses.nocbreak()
+        self.stdscr.keypad(0)
+        curses.echo()
+        curses.endwin()
+
+
+class Callback(object):
+
+    def __init__(self, stdio):
+        self.stdio = stdio
+
+    def on_line(self, text):
+        self.stdio.print_line(text)
+
+
+def main(stdscr):
+    screen = CursesStdIO(stdscr)   # create Screen object
+
+    callback = Callback(screen)
+
+    screen.set_callback(callback)
+
+    stdscr.refresh()
+    reactor.addReader(screen)
+    reactor.run()
+    screen.close()
+
+
+if __name__ == '__main__':
+    curses.wrapper(main)

contrib/experiments/test_messaging.py

@@ -0,0 +1,394 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+""" This is an example of using the server to server implementation to do a
+basic chat style thing. It accepts commands from stdin and outputs to stdout.
+
+It assumes that ucids are of the form <user>@<domain>, and uses <domain> as
+the address of the remote home server to hit.
+
+Usage:
+    python test_messaging.py <port>
+
+Currently assumes the local address is localhost:<port>
+
+"""
+
+
+from synapse.federation import (
+    ReplicationHandler
+)
+
+from synapse.federation.units import Pdu
+
+from synapse.util import origin_from_ucid
+
+from synapse.app.homeserver import SynapseHomeServer
+
+#from synapse.util.logutils import log_function
+
+from twisted.internet import reactor, defer
+from twisted.python import log
+
+import argparse
+import json
+import logging
+import os
+import re
+
+import cursesio
+import curses.wrapper
+
+
+logger = logging.getLogger("example")
+
+
+def excpetion_errback(failure):
+    logging.exception(failure)
+
+
+class InputOutput(object):
+    """ This is responsible for basic I/O so that a user can interact with
+    the example app.
+    """
+
+    def __init__(self, screen, user):
+        self.screen = screen
+        self.user = user
+
+    def set_home_server(self, server):
+        self.server = server
+
+    def on_line(self, line):
+        """ This is where we process commands.
+        """
+
+        try:
+            m = re.match("^join (\S+)$", line)
+            if m:
+                # The `sender` wants to join a room.
+                room_name, = m.groups()
+                self.print_line("%s joining %s" % (self.user, room_name))
+                self.server.join_room(room_name, self.user, self.user)
+                #self.print_line("OK.")
+                return
+
+            m = re.match("^invite (\S+) (\S+)$", line)
+            if m:
+                # `sender` wants to invite someone to a room
+                room_name, invitee = m.groups()
+                self.print_line("%s invited to %s" % (invitee, room_name))
+                self.server.invite_to_room(room_name, self.user, invitee)
+                #self.print_line("OK.")
+                return
+
+            m = re.match("^send (\S+) (.*)$", line)
+            if m:
+                # `sender` wants to message a room
+                room_name, body = m.groups()
+                self.print_line("%s send to %s" % (self.user, room_name))
+                self.server.send_message(room_name, self.user, body)
+                #self.print_line("OK.")
+                return
+
+            m = re.match("^backfill (\S+)$", line)
+            if m:
+                # we want to backfill a room
+                room_name, = m.groups()
+                self.print_line("backfill %s" % room_name)
+                self.server.backfill(room_name)
+                return
+
+            self.print_line("Unrecognized command")
+
+        except Exception as e:
+            logger.exception(e)
+
+    def print_line(self, text):
+        self.screen.print_line(text)
+
+    def print_log(self, text):
+        self.screen.print_log(text)
+
+
+class IOLoggerHandler(logging.Handler):
+
+    def __init__(self, io):
+        logging.Handler.__init__(self)
+        self.io = io
+
+    def emit(self, record):
+        if record.levelno < logging.WARN:
+            return
+
+        msg = self.format(record)
+        self.io.print_log(msg)
+
+
+class Room(object):
+    """ Used to store (in memory) the current membership state of a room, and
+    which home servers we should send PDUs associated with the room to.
+    """
+    def __init__(self, room_name):
+        self.room_name = room_name
+        self.invited = set()
+        self.participants = set()
+        self.servers = set()
+
+        self.oldest_server = None
+
+        self.have_got_metadata = False
+
+    def add_participant(self, participant):
+        """ Someone has joined the room
+        """
+        self.participants.add(participant)
+        self.invited.discard(participant)
+
+        server = origin_from_ucid(participant)
+        self.servers.add(server)
+
+        if not self.oldest_server:
+            self.oldest_server = server
+
+    def add_invited(self, invitee):
+        """ Someone has been invited to the room
+        """
+        self.invited.add(invitee)
+        self.servers.add(origin_from_ucid(invitee))
+
+
+class HomeServer(ReplicationHandler):
+    """ A very basic home server implentation that allows people to join a
+    room and then invite other people.
+    """
+    def __init__(self, server_name, replication_layer, output):
+        self.server_name = server_name
+        self.replication_layer = replication_layer
+        self.replication_layer.set_handler(self)
+
+        self.joined_rooms = {}
+
+        self.output = output
+
+    def on_receive_pdu(self, pdu):
+        """ We just received a PDU
+        """
+        pdu_type = pdu.pdu_type
+
+        if pdu_type == "sy.room.message":
+            self._on_message(pdu)
+        elif pdu_type == "sy.room.member" and "membership" in pdu.content:
+            if pdu.content["membership"] == "join":
+                self._on_join(pdu.context, pdu.state_key)
+            elif pdu.content["membership"] == "invite":
+                self._on_invite(pdu.origin, pdu.context, pdu.state_key)
+        else:
+            self.output.print_line("#%s (unrec) %s = %s" %
+                (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
+            )
+
+    #def on_state_change(self, pdu):
+        ##self.output.print_line("#%s (state) %s *** %s" %
+                ##(pdu.context, pdu.state_key, pdu.pdu_type)
+            ##)
+
+        #if "joinee" in pdu.content:
+            #self._on_join(pdu.context, pdu.content["joinee"])
+        #elif "invitee" in pdu.content:
+            #self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
+
+    def _on_message(self, pdu):
+        """ We received a message
+        """
+        self.output.print_line("#%s %s %s" %
+                (pdu.context, pdu.content["sender"], pdu.content["body"])
+            )
+
+    def _on_join(self, context, joinee):
+        """ Someone has joined a room, either a remote user or a local user
+        """
+        room = self._get_or_create_room(context)
+        room.add_participant(joinee)
+
+        self.output.print_line("#%s %s %s" %
+                (context, joinee, "*** JOINED")
+            )
+
+    def _on_invite(self, origin, context, invitee):
+        """ Someone has been invited
+        """
+        room = self._get_or_create_room(context)
+        room.add_invited(invitee)
+
+        self.output.print_line("#%s %s %s" %
+                (context, invitee, "*** INVITED")
+            )
+
+        if not room.have_got_metadata and origin is not self.server_name:
+            logger.debug("Get room state")
+            self.replication_layer.get_state_for_context(origin, context)
+            room.have_got_metadata = True
+
+    @defer.inlineCallbacks
+    def send_message(self, room_name, sender, body):
+        """ Send a message to a room!
+        """
+        destinations = yield self.get_servers_for_context(room_name)
+
+        try:
+            yield self.replication_layer.send_pdu(
+                Pdu.create_new(
+                    context=room_name,
+                    pdu_type="sy.room.message",
+                    content={"sender": sender, "body": body},
+                    origin=self.server_name,
+                    destinations=destinations,
+                )
+            )
+        except Exception as e:
+            logger.exception(e)
+
+    @defer.inlineCallbacks
+    def join_room(self, room_name, sender, joinee):
+        """ Join a room!
+        """
+        self._on_join(room_name, joinee)
+
+        destinations = yield self.get_servers_for_context(room_name)
+
+        try:
+            pdu = Pdu.create_new(
+                    context=room_name,
+                    pdu_type="sy.room.member",
+                    is_state=True,
+                    state_key=joinee,
+                    content={"membership": "join"},
+                    origin=self.server_name,
+                    destinations=destinations,
+                )
+            yield self.replication_layer.send_pdu(pdu)
+        except Exception as e:
+            logger.exception(e)
+
+    @defer.inlineCallbacks
+    def invite_to_room(self, room_name, sender, invitee):
+        """ Invite someone to a room!
+        """
+        self._on_invite(self.server_name, room_name, invitee)
+
+        destinations = yield self.get_servers_for_context(room_name)
+
+        try:
+            yield self.replication_layer.send_pdu(
+                Pdu.create_new(
+                    context=room_name,
+                    is_state=True,
+                    pdu_type="sy.room.member",
+                    state_key=invitee,
+                    content={"membership": "invite"},
+                    origin=self.server_name,
+                    destinations=destinations,
+                )
+            )
+        except Exception as e:
+            logger.exception(e)
+
+    def backfill(self, room_name, limit=5):
+        room = self.joined_rooms.get(room_name)
+
+        if not room:
+            return
+
+        dest = room.oldest_server
+
+        return self.replication_layer.backfill(dest, room_name, limit)
+
+    def _get_room_remote_servers(self, room_name):
+        return [i for i in self.joined_rooms.setdefault(room_name,).servers]
+
+    def _get_or_create_room(self, room_name):
+        return self.joined_rooms.setdefault(room_name, Room(room_name))
+
+    def get_servers_for_context(self, context):
+        return defer.succeed(
+                self.joined_rooms.setdefault(context, Room(context)).servers
+            )
+
+
+def main(stdscr):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('user', type=str)
+    parser.add_argument('-v', '--verbose', action='count')
+    args = parser.parse_args()
+
+    user = args.user
+    server_name = origin_from_ucid(user)
+
+    ## Set up logging ##
+
+    root_logger = logging.getLogger()
+
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(lineno)d - '
+            '%(levelname)s - %(message)s')
+    if not os.path.exists("logs"):
+        os.makedirs("logs")
+    fh = logging.FileHandler("logs/%s" % user)
+    fh.setFormatter(formatter)
+
+    root_logger.addHandler(fh)
+    root_logger.setLevel(logging.DEBUG)
+
+    # Hack: The only way to get it to stop logging to sys.stderr :(
+    log.theLogPublisher.observers = []
+    observer = log.PythonLoggingObserver()
+    observer.start()
+
+    ## Set up synapse server
+
+    curses_stdio = cursesio.CursesStdIO(stdscr)
+    input_output = InputOutput(curses_stdio, user)
+
+    curses_stdio.set_callback(input_output)
+
+    app_hs = SynapseHomeServer(server_name, db_name="dbs/%s" % user)
+    replication = app_hs.get_replication_layer()
+
+    hs = HomeServer(server_name, replication, curses_stdio)
+
+    input_output.set_home_server(hs)
+
+    ## Add input_output logger
+    io_logger = IOLoggerHandler(input_output)
+    io_logger.setFormatter(formatter)
+    root_logger.addHandler(io_logger)
+
+    ## Start! ##
+
+    try:
+        port = int(server_name.split(":")[1])
+    except:
+        port = 12345
+
+    app_hs.get_http_server().start_listening(port)
+
+    reactor.addReader(curses_stdio)
+
+    reactor.run()
+
+
+if __name__ == "__main__":
+    curses.wrapper(main)

contrib/graph/graph.py

@@ -0,0 +1,151 @@
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sqlite3
+import pydot
+import cgi
+import json
+import datetime
+import argparse
+import urllib2
+
+
+def make_name(pdu_id, origin):
+    return "%s@%s" % (pdu_id, origin)
+
+
+def make_graph(pdus, room, filename_prefix):
+    pdu_map = {}
+    node_map = {}
+
+    origins = set()
+    colors = set(("red", "green", "blue", "yellow", "purple"))
+
+    for pdu in pdus:
+        origins.add(pdu.get("origin"))
+
+    color_map = {color: color for color in colors if color in origins}
+    colors -= set(color_map.values())
+
+    color_map[None] = "black"
+
+    for o in origins:
+        if o in color_map:
+            continue
+        try:
+            c = colors.pop()
+            color_map[o] = c
+        except:
+            print "Run out of colours!"
+            color_map[o] = "black"
+
+    graph = pydot.Dot(graph_name="Test")
+
+    for pdu in pdus:
+        name = make_name(pdu.get("pdu_id"), pdu.get("origin"))
+        pdu_map[name] = pdu
+
+        t = datetime.datetime.fromtimestamp(
+            float(pdu["ts"]) / 1000
+        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+
+        label = (
+            "<"
+            "<b>%(name)s </b><br/>"
+            "Type: <b>%(type)s </b><br/>"
+            "State key: <b>%(state_key)s </b><br/>"
+            "Content: <b>%(content)s </b><br/>"
+            "Time: <b>%(time)s </b><br/>"
+            "Depth: <b>%(depth)s </b><br/>"
+            ">"
+        ) % {
+            "name": name,
+            "type": pdu.get("pdu_type"),
+            "state_key": pdu.get("state_key"),
+            "content": cgi.escape(json.dumps(pdu.get("content")), quote=True),
+            "time": t,
+            "depth": pdu.get("depth"),
+        }
+
+        node = pydot.Node(
+            name=name,
+            label=label,
+            color=color_map[pdu.get("origin")]
+        )
+        node_map[name] = node
+        graph.add_node(node)
+
+    for pdu in pdus:
+        start_name = make_name(pdu.get("pdu_id"), pdu.get("origin"))
+        for i, o in pdu.get("prev_pdus", []):
+            end_name = make_name(i, o)
+
+            if end_name not in node_map:
+                print "%s not in nodes" % end_name
+                continue
+
+            edge = pydot.Edge(node_map[start_name], node_map[end_name])
+            graph.add_edge(edge)
+
+        # Add prev_state edges, if they exist
+        if pdu.get("prev_state_id") and pdu.get("prev_state_origin"):
+            prev_state_name = make_name(
+                pdu.get("prev_state_id"), pdu.get("prev_state_origin")
+            )
+
+            if prev_state_name in node_map:
+                state_edge = pydot.Edge(
+                    node_map[start_name], node_map[prev_state_name],
+                    style='dotted'
+                )
+                graph.add_edge(state_edge)
+
+    graph.write('%s.dot' % filename_prefix, format='raw', prog='dot')
+#    graph.write_png("%s.png" % filename_prefix, prog='dot')
+    graph.write_svg("%s.svg" % filename_prefix, prog='dot')
+
+
+def get_pdus(host, room):
+    transaction = json.loads(
+        urllib2.urlopen(
+            "http://%s/_matrix/federation/v1/context/%s/" % (host, room)
+        ).read()
+    )
+
+    return transaction["pdus"]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate a PDU graph for a given room by talking "
+                    "to the given homeserver to get the list of PDUs. \n"
+                    "Requires pydot."
+    )
+    parser.add_argument(
+        "-p", "--prefix", dest="prefix",
+        help="String to prefix output files with"
+    )
+    parser.add_argument('host')
+    parser.add_argument('room')
+
+    args = parser.parse_args()
+
+    host = args.host
+    room = args.room
+    prefix = args.prefix if args.prefix else "%s_graph" % (room)
+
+    pdus = get_pdus(host, room)
+
+    make_graph(pdus, room, prefix)

contrib/graph/graph2.py

@@ -0,0 +1,156 @@
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sqlite3
+import pydot
+import cgi
+import json
+import datetime
+import argparse
+
+from synapse.events import FrozenEvent
+
+
+def make_graph(db_name, room_id, file_prefix, limit):
+    conn = sqlite3.connect(db_name)
+
+    sql = (
+        "SELECT json FROM event_json as j "
+        "INNER JOIN events as e ON e.event_id = j.event_id "
+        "WHERE j.room_id = ?"
+    )
+
+    args = [room_id]
+
+    if limit:
+        sql += (
+            " ORDER BY topological_ordering DESC, stream_ordering DESC "
+            "LIMIT ?"
+        )
+
+        args.append(limit)
+
+    c = conn.execute(sql, args)
+
+    events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()]
+
+    events.sort(key=lambda e: e.depth)
+
+    node_map = {}
+    state_groups = {}
+
+    graph = pydot.Dot(graph_name="Test")
+
+    for event in events:
+        c = conn.execute(
+            "SELECT state_group FROM event_to_state_groups "
+            "WHERE event_id = ?",
+            (event.event_id,)
+        )
+
+        res = c.fetchone()
+        state_group = res[0] if res else None
+
+        if state_group is not None:
+            state_groups.setdefault(state_group, []).append(event.event_id)
+
+        t = datetime.datetime.fromtimestamp(
+            float(event.origin_server_ts) / 1000
+        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+
+        content = json.dumps(event.get_dict()["content"])
+
+        label = (
+            "<"
+            "<b>%(name)s </b><br/>"
+            "Type: <b>%(type)s </b><br/>"
+            "State key: <b>%(state_key)s </b><br/>"
+            "Content: <b>%(content)s </b><br/>"
+            "Time: <b>%(time)s </b><br/>"
+            "Depth: <b>%(depth)s </b><br/>"
+            "State group: %(state_group)s<br/>"
+            ">"
+        ) % {
+            "name": event.event_id,
+            "type": event.type,
+            "state_key": event.get("state_key", None),
+            "content": cgi.escape(content, quote=True),
+            "time": t,
+            "depth": event.depth,
+            "state_group": state_group,
+        }
+
+        node = pydot.Node(
+            name=event.event_id,
+            label=label,
+        )
+
+        node_map[event.event_id] = node
+        graph.add_node(node)
+
+    for event in events:
+        for prev_id, _ in event.prev_events:
+            try:
+                end_node = node_map[prev_id]
+            except:
+                end_node = pydot.Node(
+                    name=prev_id,
+                    label="<<b>%s</b>>" % (prev_id,),
+                )
+
+                node_map[prev_id] = end_node
+                graph.add_node(end_node)
+
+            edge = pydot.Edge(node_map[event.event_id], end_node)
+            graph.add_edge(edge)
+
+    for group, event_ids in state_groups.items():
+        if len(event_ids) <= 1:
+            continue
+
+        cluster = pydot.Cluster(
+            str(group),
+            label="<State Group: %s>" % (str(group),)
+        )
+
+        for event_id in event_ids:
+            cluster.add_node(node_map[event_id])
+
+        graph.add_subgraph(cluster)
+
+    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
+    graph.write_svg("%s.svg" % file_prefix, prog='dot')
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate a PDU graph for a given room by talking "
+                    "to the given homeserver to get the list of PDUs. \n"
+                    "Requires pydot."
+    )
+    parser.add_argument(
+        "-p", "--prefix", dest="prefix",
+        help="String to prefix output files with",
+        default="graph_output"
+    )
+    parser.add_argument(
+        "-l", "--limit",
+        help="Only retrieve the last N events.",
+    )
+    parser.add_argument('db')
+    parser.add_argument('room')
+
+    args = parser.parse_args()
+
+    make_graph(args.db, args.room, args.prefix, args.limit)