Merge remote-tracking branch 'origin/develop' into develop

docs/specification.rst

@@ -1127,6 +1127,23 @@ There are several APIs provided to ``GET`` events for a room:
   Example:
     TODO
 
+Redactions
+----------
+Since events are extensible it is possible for malicious users and/or servers to add
+keys that are, for example offensive or illegal. Since some events cannot be simply
+deleted, e.g. membership events, we instead 'redact' events. This involves removing
+all keys from an event that are not required by the protocol. This stripped down
+event is thereafter returned anytime a client or remote server requests it.
+
+Events that have been redacted include a ``redacted_because`` key whose value is the
+event that caused it to be redacted, which may include a reason.
+
+Redacting an event cannot be undone, allowing server owners to delete the offending
+content from the databases.
+
+Currently, only room admins can redact events by sending a ``m.room.redacted`` event,
+but server admins also need to be able to redact events by a similar mechanism.
+
 
 Room Events
 ===========
@@ -1321,6 +1338,22 @@ prefixed with ``m.``
     end-user). The ``target_event_id`` should reference the ``m.room.message``
     event being acknowledged. 
 
+``m.room.redaction``
+  Summary:
+    Indicates a previous event has been redacted.
+  Type:
+    Non-state event
+  JSON format:
+    ``{ "reason": "string" }``
+  Description:
+    Events can be redacted by either room or server admins. Redacting an event means that
+    all keys not required by the protocol are stripped off, allowing admins to remove
+    offensive or illegal content that may have been attached to any event. This cannot be
+    undone, allowing server owners to physically delete the offending data.
+    There is also a concept of a moderator hiding a non-state event, which can be undone,
+    but cannot be applied to state events.
+    The event that has been redacted is specified in the ``redacts`` event level key.
+
 m.room.message msgtypes
 -----------------------
 Each ``m.room.message`` MUST have a ``msgtype`` key which identifies the type

synapse/api/auth.py

@@ -206,6 +206,7 @@ class Auth(object):
 
         defer.returnValue(True)
 
+    @defer.inlineCallbacks
     def get_user_by_req(self, request):
         """ Get a registered user's ID.
 
@@ -218,7 +219,25 @@ class Auth(object):
         """
         # Can optionally look elsewhere in the request (e.g. headers)
         try:
-            return self.get_user_by_token(request.args["access_token"][0])
+            access_token = request.args["access_token"][0]
+            user_info = yield self.get_user_by_token(access_token)
+            user = user_info["user"]
+
+            ip_addr = self.hs.get_ip_from_request(request)
+            user_agent = request.requestHeaders.getRawHeaders(
+                "User-Agent",
+                default=[""]
+            )[0]
+            if user and access_token and ip_addr:
+                self.store.insert_client_ip(
+                    user=user,
+                    access_token=access_token,
+                    device_id=user_info["device_id"],
+                    ip=ip_addr,
+                    user_agent=user_agent
+                )
+
+            defer.returnValue(user)
         except KeyError:
             raise AuthError(403, "Missing access token.")
 
@@ -227,21 +246,32 @@ class Auth(object):
         """ Get a registered user's ID.
 
         Args:
-            token (str)- The access token to get the user by.
+            token (str): The access token to get the user by.
         Returns:
-            UserID : User ID object of the user who has that access token.
+            dict : dict that includes the user, device_id, and whether the
+                user is a server admin.
         Raises:
             AuthError if no user by that token exists or the token is invalid.
         """
         try:
-            user_id = yield self.store.get_user_by_token(token=token)
+            ret = yield self.store.get_user_by_token(token=token)
-            if not user_id:
+            if not ret:
                 raise StoreError()
-            defer.returnValue(self.hs.parse_userid(user_id))
+
+            user_info = {
+                "admin": bool(ret.get("admin", False)),
+                "device_id": ret.get("device_id"),
+                "user": self.hs.parse_userid(ret.get("name")),
+            }
+
+            defer.returnValue(user_info)
         except StoreError:
             raise AuthError(403, "Unrecognised access token.",
                             errcode=Codes.UNKNOWN_TOKEN)
 
+    def is_server_admin(self, user):
+        return self.store.is_server_admin(user)
+
     @defer.inlineCallbacks
     @log_function
     def _can_send_event(self, event):

synapse/handlers/__init__.py

@@ -25,6 +25,7 @@ from .profile import ProfileHandler
 from .presence import PresenceHandler
 from .directory import DirectoryHandler
 from .typing import TypingNotificationHandler
+from .admin import AdminHandler
 
 
 class Handlers(object):
@@ -49,3 +50,4 @@ class Handlers(object):
         self.login_handler = LoginHandler(hs)
         self.directory_handler = DirectoryHandler(hs)
         self.typing_notification_handler = TypingNotificationHandler(hs)
+        self.admin_handler = AdminHandler(hs)

synapse/handlers/admin.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from ._base import BaseHandler
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class AdminHandler(BaseHandler):
+
+    def __init__(self, hs):
+        super(AdminHandler, self).__init__(hs)
+
+    @defer.inlineCallbacks
+    def get_whois(self, user):
+        res = yield self.store.get_user_ip_and_agents(user)
+
+        d = {}
+        for r in res:
+            device = d.setdefault(r["device_id"], {})
+            session = device.setdefault(r["access_token"], [])
+            session.append({
+                "ip": r["ip"],
+                "user_agent": r["user_agent"],
+                "last_seen": r["last_seen"],
+            })
+
+        ret = {
+            "user_id": user.to_string(),
+            "devices": [
+                {
+                    "device_id": k,
+                    "sessions": [
+                        {
+                            # "access_token": x, TODO (erikj)
+                            "connections": y,
+                        }
+                        for x, y in v.items()
+                    ]
+                }
+                for k, v in d.items()
+            ],
+        }
+
+        defer.returnValue(ret)

synapse/rest/__init__.py

@@ -15,7 +15,8 @@
 
 
 from . import (
-    room, events, register, login, profile, presence, initial_sync, directory, voip
+    room, events, register, login, profile, presence, initial_sync, directory,
+    voip, admin,
 )
 
 
@@ -43,3 +44,4 @@ class RestServletFactory(object):
         initial_sync.register_servlets(hs, client_resource)
         directory.register_servlets(hs, client_resource)
         voip.register_servlets(hs, client_resource)
+        admin.register_servlets(hs, client_resource)

synapse/rest/admin.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.errors import AuthError, SynapseError
+from base import RestServlet, client_path_pattern
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class WhoisRestServlet(RestServlet):
+    PATTERN = client_path_pattern("/admin/whois/(?P<user_id>[^/]*)")
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, user_id):
+        target_user = self.hs.parse_userid(user_id)
+        auth_user = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(auth_user)
+
+        if not is_admin and target_user != auth_user:
+            raise AuthError(403, "You are not a server admin")
+
+        if not target_user.is_mine:
+            raise SynapseError(400, "Can only whois a local user")
+
+        ret = yield self.handlers.admin_handler.get_whois(target_user)
+
+        defer.returnValue((200, ret))
+
+
+def register_servlets(hs, http_server):
+    WhoisRestServlet(hs).register(http_server)

synapse/rest/register.py

@@ -195,13 +195,7 @@ class RegisterRestServlet(RestServlet):
             raise SynapseError(400, "Captcha response is required",
                                errcode=Codes.CAPTCHA_NEEDED)
 
-        # May be an X-Forwarding-For header depending on config
+        ip_addr = self.hs.get_ip_from_request(request)
-        ip_addr = request.getClientIP()
-        if self.hs.config.captcha_ip_origin_is_x_forwarded:
-            # use the header
-            if request.requestHeaders.hasHeader("X-Forwarded-For"):
-                ip_addr = request.requestHeaders.getRawHeaders(
-                    "X-Forwarded-For")[0]
 
         handler = self.handlers.registration_handler
         yield handler.check_recaptcha(

synapse/server.py

@@ -143,6 +143,18 @@ class BaseHomeServer(object):
     def serialize_event(self, e):
         return serialize_event(self, e)
 
+    def get_ip_from_request(self, request):
+        # May be an X-Forwarding-For header depending on config
+        ip_addr = request.getClientIP()
+        if self.config.captcha_ip_origin_is_x_forwarded:
+            # use the header
+            if request.requestHeaders.hasHeader("X-Forwarded-For"):
+                ip_addr = request.requestHeaders.getRawHeaders(
+                    "X-Forwarded-For"
+                )[0]
+
+        return ip_addr
+
 # Build magic accessors for every dependency
 for depname in BaseHomeServer.DEPENDENCIES:
     BaseHomeServer._make_dependency_method(depname)

synapse/storage/__init__.py

@@ -63,7 +63,7 @@ SCHEMAS = [
 
 # Remember to update this number every time an incompatible change is made to
 # database schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 4
+SCHEMA_VERSION = 5
 
 
 class _RollbackButIsFineException(Exception):
@@ -294,6 +294,28 @@ class DataStore(RoomMemberStore, RoomStore,
 
         defer.returnValue(self.min_token)
 
+    def insert_client_ip(self, user, access_token, device_id, ip, user_agent):
+        return self._simple_insert(
+            "user_ips",
+            {
+                "user": user.to_string(),
+                "access_token": access_token,
+                "device_id": device_id,
+                "ip": ip,
+                "user_agent": user_agent,
+                "last_seen": int(self._clock.time_msec()),
+            }
+        )
+
+    def get_user_ip_and_agents(self, user):
+        return self._simple_select_list(
+            table="user_ips",
+            keyvalues={"user": user.to_string()},
+            retcols=[
+                "device_id", "access_token", "ip", "user_agent", "last_seen"
+            ],
+        )
+
     def snapshot_room(self, room_id, user_id, state_type=None, state_key=None):
         """Snapshot the room for an update by a user
         Args:

synapse/storage/registration.py

@@ -88,27 +88,40 @@ class RegistrationStore(SQLBaseStore):
             query, user_id
         )
 
-    @defer.inlineCallbacks
     def get_user_by_token(self, token):
         """Get a user from the given access token.
 
         Args:
             token (str): The access token of a user.
         Returns:
-            str: The user ID of the user.
+            dict: Including the name (user_id), device_id and whether they are
+                an admin.
         Raises:
             StoreError if no user was found.
         """
-        user_id = yield self.runInteraction(self._query_for_auth,
+        return self.runInteraction(
-                                                     token)
+            self._query_for_auth,
-        defer.returnValue(user_id)
+            token
+        )
+
+    def is_server_admin(self, user):
+        return self._simple_select_one_onecol(
+            table="users",
+            keyvalues={"name": user.to_string()},
+            retcol="admin",
+        )
 
     def _query_for_auth(self, txn, token):
-        txn.execute("SELECT users.name FROM access_tokens LEFT JOIN users" +
+        sql = (
-                    " ON users.id = access_tokens.user_id WHERE token = ?",
+            "SELECT users.name, users.admin, access_tokens.device_id "
-                    [token])
+            "FROM users "
-        row = txn.fetchone()
+            "INNER JOIN access_tokens on users.id = access_tokens.user_id "
-        if row:
+            "WHERE token = ?"
-            return row[0]
+        )
+
+        cursor = txn.execute(sql, (token,))
+        rows = self.cursor_to_dict(cursor)
+        if rows:
+            return rows[0]
 
         raise StoreError(404, "Token not found.")

synapse/storage/schema/delta/v5.sql

@@ -0,0 +1,16 @@
+
+CREATE TABLE IF NOT EXISTS user_ips (
+    user TEXT NOT NULL,
+    access_token TEXT NOT NULL,
+    device_id TEXT,
+    ip TEXT NOT NULL,
+    user_agent TEXT NOT NULL,
+    last_seen INTEGER NOT NULL,
+    CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+
+ALTER TABLE users ADD COLUMN admin BOOL DEFAULT 0 NOT NULL;
+
+PRAGMA user_version = 5;

synapse/storage/schema/users.sql

@@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS users(
     name TEXT,
     password_hash TEXT,
     creation_ts INTEGER,
+    admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name) ON CONFLICT ROLLBACK
 );
 
@@ -29,3 +30,16 @@ CREATE TABLE IF NOT EXISTS access_tokens(
     FOREIGN KEY(user_id) REFERENCES users(id),
     UNIQUE(token) ON CONFLICT ROLLBACK
 );
+
+CREATE TABLE IF NOT EXISTS user_ips (
+    user TEXT NOT NULL,
+    access_token TEXT NOT NULL,
+    device_id TEXT,
+    ip TEXT NOT NULL,
+    user_agent TEXT NOT NULL,
+    last_seen INTEGER NOT NULL,
+    CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+

tests/rest/test_presence.py

@@ -51,10 +51,12 @@ class PresenceStateTestCase(unittest.TestCase):
             datastore=Mock(spec=[
                 "get_presence_state",
                 "set_presence_state",
+                "insert_client_ip",
             ]),
             http_client=None,
             resource_for_client=self.mock_resource,
             resource_for_federation=self.mock_resource,
+            config=Mock(),
         )
         hs.handlers = JustPresenceHandlers(hs)
 
@@ -65,7 +67,11 @@ class PresenceStateTestCase(unittest.TestCase):
         self.datastore.get_presence_list = get_presence_list
 
         def _get_user_by_token(token=None):
-            return hs.parse_userid(myid)
+            return {
+                "user": hs.parse_userid(myid),
+                "admin": False,
+                "device_id": None,
+            }
 
         hs.get_auth().get_user_by_token = _get_user_by_token
 
@@ -131,10 +137,12 @@ class PresenceListTestCase(unittest.TestCase):
                 "set_presence_list_accepted",
                 "del_presence_list",
                 "get_presence_list",
+                "insert_client_ip",
             ]),
             http_client=None,
             resource_for_client=self.mock_resource,
-            resource_for_federation=self.mock_resource
+            resource_for_federation=self.mock_resource,
+            config=Mock(),
         )
         hs.handlers = JustPresenceHandlers(hs)
 
@@ -147,7 +155,11 @@ class PresenceListTestCase(unittest.TestCase):
         self.datastore.has_presence_state = has_presence_state
 
         def _get_user_by_token(token=None):
-            return hs.parse_userid(myid)
+            return {
+                "user": hs.parse_userid(myid),
+                "admin": False,
+                "device_id": None,
+            }
 
         room_member_handler = hs.handlers.room_member_handler = Mock(
             spec=[

tests/rest/test_profile.py

@@ -50,10 +50,10 @@ class ProfileTestCase(unittest.TestCase):
             datastore=None,
         )
 
-        def _get_user_by_token(token=None):
+        def _get_user_by_req(request=None):
             return hs.parse_userid(myid)
 
-        hs.get_auth().get_user_by_token = _get_user_by_token
+        hs.get_auth().get_user_by_req = _get_user_by_req
 
         hs.get_handlers().profile_handler = self.mock_handler
 

tests/rest/test_rooms.py

@@ -69,7 +69,11 @@ class RoomPermissionsTestCase(RestTestCase):
         hs.get_handlers().federation_handler = Mock()
 
         def _get_user_by_token(token=None):
-            return hs.parse_userid(self.auth_user_id)
+            return {
+                "user": hs.parse_userid(self.auth_user_id),
+                "admin": False,
+                "device_id": None,
+            }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
         self.auth_user_id = self.rmcreator_id
@@ -425,7 +429,11 @@ class RoomsMemberListTestCase(RestTestCase):
         self.auth_user_id = self.user_id
 
         def _get_user_by_token(token=None):
-            return hs.parse_userid(self.auth_user_id)
+            return {
+                "user": hs.parse_userid(self.auth_user_id),
+                "admin": False,
+                "device_id": None,
+            }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
         synapse.rest.room.register_servlets(hs, self.mock_resource)
@@ -508,7 +516,11 @@ class RoomsCreateTestCase(RestTestCase):
         hs.get_handlers().federation_handler = Mock()
 
         def _get_user_by_token(token=None):
-            return hs.parse_userid(self.auth_user_id)
+            return {
+                "user": hs.parse_userid(self.auth_user_id),
+                "admin": False,
+                "device_id": None,
+            }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
         synapse.rest.room.register_servlets(hs, self.mock_resource)
@@ -605,7 +617,11 @@ class RoomTopicTestCase(RestTestCase):
         hs.get_handlers().federation_handler = Mock()
 
         def _get_user_by_token(token=None):
-            return hs.parse_userid(self.auth_user_id)
+            return {
+                "user": hs.parse_userid(self.auth_user_id),
+                "admin": False,
+                "device_id": None,
+            }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
         synapse.rest.room.register_servlets(hs, self.mock_resource)
@@ -715,7 +731,16 @@ class RoomMemberStateTestCase(RestTestCase):
         hs.get_handlers().federation_handler = Mock()
 
         def _get_user_by_token(token=None):
-            return hs.parse_userid(self.auth_user_id)
+            return {
+                "user": hs.parse_userid(self.auth_user_id),
+                "admin": False,
+                "device_id": None,
+            }
+            return {
+                "user": hs.parse_userid(self.auth_user_id),
+                "admin": False,
+                "device_id": None,
+            }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
         synapse.rest.room.register_servlets(hs, self.mock_resource)
@@ -847,7 +872,11 @@ class RoomMessagesTestCase(RestTestCase):
         hs.get_handlers().federation_handler = Mock()
 
         def _get_user_by_token(token=None):
-            return hs.parse_userid(self.auth_user_id)
+            return {
+                "user": hs.parse_userid(self.auth_user_id),
+                "admin": False,
+                "device_id": None,
+            }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
         synapse.rest.room.register_servlets(hs, self.mock_resource)

tests/storage/test_registration.py

@@ -53,7 +53,7 @@ class RegistrationStoreTestCase(unittest.TestCase):
         )
 
         self.assertEquals(
-            self.user_id,
+            {"admin": 0, "device_id": None, "name": self.user_id},
             (yield self.store.get_user_by_token(self.tokens[0]))
         )
 
@@ -63,7 +63,7 @@ class RegistrationStoreTestCase(unittest.TestCase):
         yield self.store.add_access_token_to_user(self.user_id, self.tokens[1])
 
         self.assertEquals(
-            self.user_id,
+            {"admin": 0, "device_id": None, "name": self.user_id},
             (yield self.store.get_user_by_token(self.tokens[1]))
         )
 

tests/utils.py

@@ -167,7 +167,11 @@ class MemoryDataStore(object):
 
     def get_user_by_token(self, token):
         try:
-            return self.tokens_to_users[token]
+            return {
+                "name": self.tokens_to_users[token],
+                "admin": 0,
+                "device_id": None,
+            }
         except:
             raise StoreError(400, "User does not exist.")
 
@@ -264,6 +268,9 @@ class MemoryDataStore(object):
     def get_ops_levels(self, room_id):
         return defer.succeed((5, 5, 5))
 
+    def insert_client_ip(self, user, device_id, access_token, ip, user_agent):
+        return defer.succeed(None)
+
 
 def _format_call(args, kwargs):
     return ", ".join(