From fac3c03087b14abf7b9857f937fd3311e0619a6f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Feb 2017 18:27:24 +0000 Subject: [PATCH 01/82] Be more agressive about purging old room event_push_actions --- synapse/storage/event_push_actions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 7de3e8c58..522d0114c 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -450,8 +450,12 @@ class EventPushActionsStore(SQLBaseStore): def _remove_old_push_actions_before_txn(self, txn, room_id, user_id, topological_ordering): """ - Purges old, stale push actions for a user and room before a given - topological_ordering + Purges old push actions for a user and room before a given + topological_ordering. + + We however keep a months worth of highlighted notifications, so that + users can still get a list of recent highlights. + Args: txn: The transcation room_id: Room ID to delete from @@ -475,7 +479,8 @@ class EventPushActionsStore(SQLBaseStore): txn.execute( "DELETE FROM event_push_actions " " WHERE user_id = ? AND room_id = ? AND " - " topological_ordering < ? AND stream_ordering < ?", + " topological_ordering < ?" + " AND ((stream_ordering < ? AND highlight = 1) or highlight = 0)", (user_id, room_id, topological_ordering, self.stream_ordering_month_ago) ) From 2849d3f29ddedaf35b0e080b02a40f8613fdba90 Mon Sep 17 00:00:00 2001 From: Morteza Araby Date: Thu, 2 Feb 2017 14:02:26 +0100 Subject: [PATCH 02/82] admin,storage: added more administrator functionalities administrators can now: - Set displayname of users - Update user avatars - Search for users by user_id - Browse all users in a paginated API - Reset user passwords - Deactivate users Helpers for doing paginated queries has also been added to storage Signed-off-by: Morteza Araby --- synapse/handlers/admin.py | 44 +++++- synapse/rest/client/v1/admin.py | 220 ++++++++++++++++++++++++++++++ synapse/rest/client/v1/profile.py | 6 +- synapse/storage/__init__.py | 76 +++++++++++ synapse/storage/_base.py | 159 +++++++++++++++++++++ 5 files changed, 502 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 084e33ca6..f36b358b4 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -19,7 +19,6 @@ from ._base import BaseHandler import logging - logger = logging.getLogger(__name__) @@ -54,3 +53,46 @@ class AdminHandler(BaseHandler): } defer.returnValue(ret) + + @defer.inlineCallbacks + def get_users(self): + """Function to reterive a list of users in users table. + + Args: + Returns: + defer.Deferred: resolves to list[dict[str, Any]] + """ + ret = yield self.store.get_users() + + defer.returnValue(ret) + + @defer.inlineCallbacks + def get_users_paginate(self, order, start, limit): + """Function to reterive a paginated list of users from + users list. This will return a json object, which contains + list of users and the total number of users in users table. + + Args: + order (str): column name to order the select by this column + start (int): start number to begin the query from + limit (int): number of rows to reterive + Returns: + defer.Deferred: resolves to json object {list[dict[str, Any]], count} + """ + ret = yield self.store.get_users_paginate(order, start, limit) + + defer.returnValue(ret) + + @defer.inlineCallbacks + def search_users(self, term): + """Function to search users list for one or more users with + the matched term. + + Args: + term (str): search term + Returns: + defer.Deferred: resolves to list[dict[str, Any]] + """ + ret = yield self.store.search_users(term) + + defer.returnValue(ret) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index af21661d7..29fcd7237 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.api.errors import AuthError, SynapseError from synapse.types import UserID +from synapse.http.servlet import parse_json_object_from_request from .base import ClientV1RestServlet, client_path_patterns @@ -25,6 +26,34 @@ import logging logger = logging.getLogger(__name__) +class UsersRestServlet(ClientV1RestServlet): + PATTERNS = client_path_patterns("/admin/users/(?P[^/]*)") + + def __init__(self, hs): + super(UsersRestServlet, self).__init__(hs) + self.handlers = hs.get_handlers() + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + target_user = UserID.from_string(user_id) + requester = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(requester.user) + + if not is_admin: + raise AuthError(403, "You are not a server admin") + + # To allow all users to get the users list + # if not is_admin and target_user != auth_user: + # raise AuthError(403, "You are not a server admin") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only users a local user") + + ret = yield self.handlers.admin_handler.get_users() + + defer.returnValue((200, ret)) + + class WhoisRestServlet(ClientV1RestServlet): PATTERNS = client_path_patterns("/admin/whois/(?P[^/]*)") @@ -128,8 +157,199 @@ class DeactivateAccountRestServlet(ClientV1RestServlet): defer.returnValue((200, {})) +class ResetPasswordRestServlet(ClientV1RestServlet): + """Post request to allow an administrator reset password for a user. + This need a user have a administrator access in Synapse. + Example: + http://localhost:8008/_matrix/client/api/v1/admin/reset_password/ + @user:to_reset_password?access_token=admin_access_token + JsonBodyToSend: + { + "new_password": "secret" + } + Returns: + 200 OK with empty object if success otherwise an error. + """ + PATTERNS = client_path_patterns("/admin/reset_password/(?P[^/]*)") + + def __init__(self, hs): + self.store = hs.get_datastore() + super(ResetPasswordRestServlet, self).__init__(hs) + self.hs = hs + self.auth = hs.get_auth() + self.auth_handler = hs.get_auth_handler() + + @defer.inlineCallbacks + def on_POST(self, request, target_user_id): + """Post request to allow an administrator reset password for a user. + This need a user have a administrator access in Synapse. + """ + UserID.from_string(target_user_id) + requester = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(requester.user) + + if not is_admin: + raise AuthError(403, "You are not a server admin") + + params = parse_json_object_from_request(request) + new_password = params['new_password'] + if not new_password: + raise SynapseError(400, "Missing 'new_password' arg") + + logger.info("new_password: %r", new_password) + + yield self.auth_handler.set_password( + target_user_id, new_password, requester + ) + defer.returnValue((200, {})) + + +class GetUsersPaginatedRestServlet(ClientV1RestServlet): + """Get request to get specific number of users from Synapse. + This need a user have a administrator access in Synapse. + Example: + http://localhost:8008/_matrix/client/api/v1/admin/users_paginate/ + @admin:user?access_token=admin_access_token&start=0&limit=10 + Returns: + 200 OK with json object {list[dict[str, Any]], count} or empty object. + """ + PATTERNS = client_path_patterns("/admin/users_paginate/(?P[^/]*)") + + def __init__(self, hs): + self.store = hs.get_datastore() + super(GetUsersPaginatedRestServlet, self).__init__(hs) + self.hs = hs + self.auth = hs.get_auth() + self.handlers = hs.get_handlers() + + @defer.inlineCallbacks + def on_GET(self, request, target_user_id): + """Get request to get specific number of users from Synapse. + This need a user have a administrator access in Synapse. + """ + target_user = UserID.from_string(target_user_id) + requester = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(requester.user) + + if not is_admin: + raise AuthError(403, "You are not a server admin") + + # To allow all users to get the users list + # if not is_admin and target_user != auth_user: + # raise AuthError(403, "You are not a server admin") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only users a local user") + + order = "name" # order by name in user table + start = request.args.get("start")[0] + limit = request.args.get("limit")[0] + if not limit: + raise SynapseError(400, "Missing 'limit' arg") + if not start: + raise SynapseError(400, "Missing 'start' arg") + logger.info("limit: %s, start: %s", limit, start) + + ret = yield self.handlers.admin_handler.get_users_paginate( + order, start, limit + ) + defer.returnValue((200, ret)) + + @defer.inlineCallbacks + def on_POST(self, request, target_user_id): + """Post request to get specific number of users from Synapse.. + This need a user have a administrator access in Synapse. + Example: + http://localhost:8008/_matrix/client/api/v1/admin/users_paginate/ + @admin:user?access_token=admin_access_token + JsonBodyToSend: + { + "start": "0", + "limit": "10 + } + Returns: + 200 OK with json object {list[dict[str, Any]], count} or empty object. + """ + UserID.from_string(target_user_id) + requester = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(requester.user) + + if not is_admin: + raise AuthError(403, "You are not a server admin") + + order = "name" # order by name in user table + params = parse_json_object_from_request(request) + limit = params['limit'] + start = params['start'] + if not limit: + raise SynapseError(400, "Missing 'limit' arg") + if not start: + raise SynapseError(400, "Missing 'start' arg") + logger.info("limit: %s, start: %s", limit, start) + + ret = yield self.handlers.admin_handler.get_users_paginate( + order, start, limit + ) + defer.returnValue((200, ret)) + + +class SearchUsersRestServlet(ClientV1RestServlet): + """Get request to search user table for specific users according to + search term. + This need a user have a administrator access in Synapse. + Example: + http://localhost:8008/_matrix/client/api/v1/admin/search_users/ + @admin:user?access_token=admin_access_token&term=alice + Returns: + 200 OK with json object {list[dict[str, Any]], count} or empty object. + """ + PATTERNS = client_path_patterns("/admin/search_users/(?P[^/]*)") + + def __init__(self, hs): + self.store = hs.get_datastore() + super(SearchUsersRestServlet, self).__init__(hs) + self.hs = hs + self.auth = hs.get_auth() + self.handlers = hs.get_handlers() + + @defer.inlineCallbacks + def on_GET(self, request, target_user_id): + """Get request to search user table for specific users according to + search term. + This need a user have a administrator access in Synapse. + """ + target_user = UserID.from_string(target_user_id) + requester = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(requester.user) + + if not is_admin: + raise AuthError(403, "You are not a server admin") + + # To allow all users to get the users list + # if not is_admin and target_user != auth_user: + # raise AuthError(403, "You are not a server admin") + + if not self.hs.is_mine(target_user): + raise SynapseError(400, "Can only users a local user") + + term = request.args.get("term")[0] + if not term: + raise SynapseError(400, "Missing 'term' arg") + + logger.info("term: %s ", term) + + ret = yield self.handlers.admin_handler.search_users( + term + ) + defer.returnValue((200, ret)) + + def register_servlets(hs, http_server): WhoisRestServlet(hs).register(http_server) PurgeMediaCacheRestServlet(hs).register(http_server) DeactivateAccountRestServlet(hs).register(http_server) PurgeHistoryRestServlet(hs).register(http_server) + UsersRestServlet(hs).register(http_server) + ResetPasswordRestServlet(hs).register(http_server) + GetUsersPaginatedRestServlet(hs).register(http_server) + SearchUsersRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 355e82474..1a5045c9e 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -46,6 +46,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): def on_PUT(self, request, user_id): requester = yield self.auth.get_user_by_req(request, allow_guest=True) user = UserID.from_string(user_id) + is_admin = yield self.auth.is_server_admin(requester.user) content = parse_json_object_from_request(request) @@ -55,7 +56,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, requester, new_name) + user, requester, new_name, is_admin) defer.returnValue((200, {})) @@ -88,6 +89,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): def on_PUT(self, request, user_id): requester = yield self.auth.get_user_by_req(request) user = UserID.from_string(user_id) + is_admin = yield self.auth.is_server_admin(requester.user) content = parse_json_object_from_request(request) try: @@ -96,7 +98,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_avatar_url( - user, requester, new_name) + user, requester, new_name, is_admin) defer.returnValue((200, {})) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b9968debe..d604e7668 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -297,6 +297,82 @@ class DataStore(RoomMemberStore, RoomStore, desc="get_user_ip_and_agents", ) + def get_users(self): + """Function to reterive a list of users in users table. + + Args: + Returns: + defer.Deferred: resolves to list[dict[str, Any]] + """ + return self._simple_select_list( + table="users", + keyvalues={}, + retcols=[ + "name", + "password_hash", + "is_guest", + "admin" + ], + desc="get_users", + ) + + def get_users_paginate(self, order, start, limit): + """Function to reterive a paginated list of users from + users list. This will return a json object, which contains + list of users and the total number of users in users table. + + Args: + order (str): column name to order the select by this column + start (int): start number to begin the query from + limit (int): number of rows to reterive + Returns: + defer.Deferred: resolves to json object {list[dict[str, Any]], count} + """ + is_guest = 0 + i_start = (int)(start) + i_limit = (int)(limit) + return self.get_user_list_paginate( + table="users", + keyvalues={ + "is_guest": is_guest + }, + pagevalues=[ + order, + i_limit, + i_start + ], + retcols=[ + "name", + "password_hash", + "is_guest", + "admin" + ], + desc="get_users_paginate", + ) + + def search_users(self, term): + """Function to search users list for one or more users with + the matched term. + + Args: + term (str): search term + col (str): column to query term should be matched to + Returns: + defer.Deferred: resolves to list[dict[str, Any]] + """ + return self._simple_search_list( + table="users", + term=term, + col="name", + retcols=[ + "name", + "password_hash", + "is_guest", + "admin" + ], + desc="search_users", + ) + def are_all_users_on_domain(txn, database_engine, domain): sql = database_engine.convert_param_style( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 05374682f..b0dc39119 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -934,6 +934,165 @@ class SQLBaseStore(object): else: return 0 + def _simple_select_list_paginate(self, table, keyvalues, pagevalues, retcols, + desc="_simple_select_list_paginate"): + """Executes a SELECT query on the named table with start and limit, + of row numbers, which may return zero or number of rows from start to limit, + returning the result as a list of dicts. + + Args: + table (str): the table name + keyvalues (dict[str, Any] | None): + column names and values to select the rows with, or None to not + apply a WHERE clause. + retcols (iterable[str]): the names of the columns to return + order (str): order the select by this column + start (int): start number to begin the query from + limit (int): number of rows to reterive + Returns: + defer.Deferred: resolves to list[dict[str, Any]] + """ + return self.runInteraction( + desc, + self._simple_select_list_paginate_txn, + table, keyvalues, pagevalues, retcols + ) + + @classmethod + def _simple_select_list_paginate_txn(cls, txn, table, keyvalues, pagevalues, retcols): + """Executes a SELECT query on the named table with start and limit, + of row numbers, which may return zero or number of rows from start to limit, + returning the result as a list of dicts. + + Args: + txn : Transaction object + table (str): the table name + keyvalues (dict[str, T] | None): + column names and values to select the rows with, or None to not + apply a WHERE clause. + pagevalues ([]): + order (str): order the select by this column + start (int): start number to begin the query from + limit (int): number of rows to reterive + retcols (iterable[str]): the names of the columns to return + Returns: + defer.Deferred: resolves to list[dict[str, Any]] + + """ + if keyvalues: + sql = "SELECT %s FROM %s WHERE %s ORDER BY %s" % ( + ", ".join(retcols), + table, + " AND ".join("%s = ?" % (k,) for k in keyvalues), + " ? ASC LIMIT ? OFFSET ?" + ) + txn.execute(sql, keyvalues.values() + pagevalues) + else: + sql = "SELECT %s FROM %s ORDER BY %s" % ( + ", ".join(retcols), + table, + " ? ASC LIMIT ? OFFSET ?" + ) + txn.execute(sql, pagevalues) + + return cls.cursor_to_dict(txn) + + @defer.inlineCallbacks + def get_user_list_paginate(self, table, keyvalues, pagevalues, retcols, + desc="get_user_list_paginate"): + """Get a list of users from start row to a limit number of rows. This will + return a json object with users and total number of users in users list. + + Args: + table (str): the table name + keyvalues (dict[str, Any] | None): + column names and values to select the rows with, or None to not + apply a WHERE clause. + pagevalues ([]): + order (str): order the select by this column + start (int): start number to begin the query from + limit (int): number of rows to reterive + retcols (iterable[str]): the names of the columns to return + Returns: + defer.Deferred: resolves to json object {list[dict[str, Any]], count} + """ + users = yield self.runInteraction( + desc, + self._simple_select_list_paginate_txn, + table, keyvalues, pagevalues, retcols + ) + count = yield self.runInteraction( + desc, + self.get_user_count_txn + ) + retval = { + "users": users, + "total": count + } + defer.returnValue(retval) + + def get_user_count_txn(self, txn): + """Get a total number of registerd users in the users list. + + Args: + txn : Transaction object + Returns: + defer.Deferred: resolves to int + """ + sql_count = "SELECT COUNT(*) FROM users WHERE is_guest = 0;" + txn.execute(sql_count) + count = txn.fetchone()[0] + defer.returnValue(count) + + def _simple_search_list(self, table, term, col, retcols, + desc="_simple_search_list"): + """Executes a SELECT query on the named table, which may return zero or + more rows, returning the result as a list of dicts. + + Args: + table (str): the table name + term (str | None): + term for searching the table matched to a column. + col (str): column to query term should be matched to + retcols (iterable[str]): the names of the columns to return + Returns: + defer.Deferred: resolves to list[dict[str, Any]] or None + """ + + return self.runInteraction( + desc, + self._simple_search_list_txn, + table, term, col, retcols + ) + + @classmethod + def _simple_search_list_txn(cls, txn, table, term, col, retcols): + """Executes a SELECT query on the named table, which may return zero or + more rows, returning the result as a list of dicts. + + Args: + txn : Transaction object + table (str): the table name + term (str | None): + term for searching the table matched to a column. + col (str): column to query term should be matched to + retcols (iterable[str]): the names of the columns to return + Returns: + defer.Deferred: resolves to list[dict[str, Any]] or None + """ + if term: + sql = "SELECT %s FROM %s WHERE %s LIKE ?" % ( + ", ".join(retcols), + table, + col + ) + termvalues = ["%%" + term + "%%"] + txn.execute(sql, termvalues) + else: + return 0 + + return cls.cursor_to_dict(txn) + class _RollbackButIsFineException(Exception): """ This exception is used to rollback a transaction without implying From 063a1251a965fc7b637a24bb87bb89678c137ecf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Feb 2017 11:36:08 +0000 Subject: [PATCH 03/82] Remove a few aspirational but unused constants from the Kegan era --- synapse/api/constants.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index a8123cddc..ca23c9c46 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -43,9 +43,6 @@ class JoinRules(object): class LoginType(object): PASSWORD = u"m.login.password" - OAUTH = u"m.login.oauth2" - EMAIL_CODE = u"m.login.email.code" - EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" DUMMY = u"m.login.dummy" From 52cd019a544b49469ce588c42a59100e6384d362 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Feb 2017 16:04:29 +0000 Subject: [PATCH 04/82] Make None check explicit --- synapse/storage/end_to_end_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 2040e022f..b9f1365f9 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -93,7 +93,7 @@ class EndToEndKeyStore(SQLBaseStore): query_clause = "user_id = ?" query_params.append(user_id) - if device_id: + if device_id is not None: query_clause += " AND device_id = ?" query_params.append(device_id) From fdbd90e25dc3682c4d6aa1dd85a9e0e4fedabb1a Mon Sep 17 00:00:00 2001 From: Daniel Dent Date: Wed, 8 Feb 2017 21:21:02 -0800 Subject: [PATCH 05/82] Update CAPTCHA_SETUP.rst X-Forwarded-For docs It looks like CAPTCHA_SETUP.rst contains information relevant to an old version of Synapse, but Synapse now has a different approach to configuring use of the X-Forwarded-For header. --- docs/CAPTCHA_SETUP.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/CAPTCHA_SETUP.rst b/docs/CAPTCHA_SETUP.rst index db621aedf..19a204d9c 100644 --- a/docs/CAPTCHA_SETUP.rst +++ b/docs/CAPTCHA_SETUP.rst @@ -25,6 +25,5 @@ Configuring IP used for auth The ReCaptcha API requires that the IP address of the user who solved the captcha is sent. If the client is connecting through a proxy or load balancer, it may be required to use the X-Forwarded-For (XFF) header instead of the origin -IP address. This can be configured as an option on the home server like so:: - - captcha_ip_origin_is_x_forwarded: true +IP address. This can be configured using the x_forwarded directive in the +listeners section of the homeserver.yaml configuration file. From a02d609b1ffdc313fc1e06a68228ad2cb45919ec Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 11 Feb 2017 20:44:16 -0500 Subject: [PATCH 06/82] Fix synapse_port_db failure (fixes #1902) See https://matrix.to/#/!cURbafjkfsMDVwdRDQ:matrix.org/$148686272020hCgRD:potatofrom.space Signed-off-by: Kevin Liu --- scripts/synapse_port_db | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 2cb2eab68..4bd110c32 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -40,6 +40,7 @@ BOOLEAN_COLUMNS = { "presence_list": ["accepted"], "presence_stream": ["currently_active"], "public_room_list_stream": ["visibility"], + "device_list_outbound_pokes": ["sent"], } From 70a00eacf9ebdca70966fa8c343d55d6e9232973 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sat, 11 Feb 2017 20:49:31 -0500 Subject: [PATCH 07/82] Fix typo This is what I get for not proofreading --- scripts/synapse_port_db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 4bd110c32..ea367a128 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -40,7 +40,7 @@ BOOLEAN_COLUMNS = { "presence_list": ["accepted"], "presence_stream": ["currently_active"], "public_room_list_stream": ["visibility"], - "device_list_outbound_pokes": ["sent"], + "device_lists_outbound_pokes": ["sent"], } From df4407d6656a218905bc04a41a72ed8b16abf73c Mon Sep 17 00:00:00 2001 From: Tyler Smith Date: Sat, 11 Feb 2017 23:02:57 -0800 Subject: [PATCH 08/82] Fix typo in config comments. Signed-off-by: Tyler Smith --- synapse/config/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 3c58d2de1..e081840a8 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -95,7 +95,7 @@ class TlsConfig(Config): # make HTTPS requests to this server will check that the TLS # certificates returned by this server match one of the fingerprints. # - # Synapse automatically adds its the fingerprint of its own certificate + # Synapse automatically adds the fingerprint of its own certificate # to the list. So if federation traffic is handle directly by synapse # then no modification to the list is required. # From 6a3743b0d4c9845d3c50b2ab5fa9954647ce5962 Mon Sep 17 00:00:00 2001 From: Andrew Shadura Date: Sun, 12 Feb 2017 09:54:56 +0100 Subject: [PATCH 09/82] Use signedjson.sign instead of syutil.crypto.jsonsign Functions from syutil.crypto.jsonsign are now available in signedjson, so use that instead of depending on syutil. Signed-off-by: Andrew Shadura --- contrib/cmdclient/console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/cmdclient/console.py b/contrib/cmdclient/console.py index 8bb03ce66..4918fa1a9 100755 --- a/contrib/cmdclient/console.py +++ b/contrib/cmdclient/console.py @@ -32,7 +32,7 @@ import urlparse import nacl.signing import nacl.encoding -from syutil.crypto.jsonsign import verify_signed_json, SignatureVerifyException +from signedjson.sign import verify_signed_json, SignatureVerifyException CONFIG_JSON = "cmdclient_config.json" From 3a46280ca3923d4f6e1103b9ee22bac0ddc2edf7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Feb 2017 11:16:53 +0000 Subject: [PATCH 10/82] Add db functions needed for room initial sync to slave --- synapse/app/synchrotron.py | 4 ++++ synapse/replication/slave/storage/account_data.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index b3fb408cf..3f2959525 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -87,6 +87,10 @@ class SynchrotronSlavedStore( RoomMemberStore.__dict__["who_forgot_in_room"] ) + did_forget = ( + RoomMemberStore.__dict__["did_forget"] + ) + # XXX: This is a bit broken because we don't persist the accepted list in a # way that can be replicated. This means that we don't have a way to # invalidate the cache correctly. diff --git a/synapse/replication/slave/storage/account_data.py b/synapse/replication/slave/storage/account_data.py index 735c03c7e..77c64722c 100644 --- a/synapse/replication/slave/storage/account_data.py +++ b/synapse/replication/slave/storage/account_data.py @@ -46,6 +46,12 @@ class SlavedAccountDataStore(BaseSlavedStore): ) get_tags_for_user = TagsStore.__dict__["get_tags_for_user"] + get_tags_for_room = ( + DataStore.get_tags_for_room.__func__ + ) + get_account_data_for_room = ( + DataStore.get_account_data_for_room.__func__ + ) get_updated_tags = DataStore.get_updated_tags.__func__ get_updated_account_data_for_user = ( From ecd7e36047d090cdb027f500b0f95a375ba61811 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 13 Feb 2017 13:16:48 +0000 Subject: [PATCH 11/82] http txns: Do not cache error responses Previously we did. This meant that, amongst other errors, rate-limiting errors would be cached and prevent messages with that txn ID being sent. --- synapse/rest/client/transactions.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index efa77b8c5..6396a0803 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -81,7 +81,16 @@ class HttpTransactionCache(object): Deferred which resolves to a tuple of (response_code, response_dict). """ try: - return self.transactions[txn_key][0].observe() + observable = self.transactions[txn_key][0] + if not observable.has_called() or observable.has_succeeded(): + return observable.observe() + # if the request has already been called with a non-2xx status + # (a Twisted failure), remove it from the transaction map. + # This is done to ensure that we don't cache rate-limiting errors, etc. + res = observable.get_result() + if res.value.code >= 300: + del self.transactions[txn_key] + # fall through except (KeyError, IndexError): pass # execute the function instead. From feb15dc99f02e6cb0a84a53e397529c51743f114 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 13 Feb 2017 13:33:12 +0000 Subject: [PATCH 12/82] Don't cache errors at all --- synapse/rest/client/transactions.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 6396a0803..95376a2fb 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -81,16 +81,7 @@ class HttpTransactionCache(object): Deferred which resolves to a tuple of (response_code, response_dict). """ try: - observable = self.transactions[txn_key][0] - if not observable.has_called() or observable.has_succeeded(): - return observable.observe() - # if the request has already been called with a non-2xx status - # (a Twisted failure), remove it from the transaction map. - # This is done to ensure that we don't cache rate-limiting errors, etc. - res = observable.get_result() - if res.value.code >= 300: - del self.transactions[txn_key] - # fall through + return self.transactions[txn_key][0].observe() except (KeyError, IndexError): pass # execute the function instead. @@ -101,6 +92,14 @@ class HttpTransactionCache(object): # to the observers. observable = ObservableDeferred(deferred, consumeErrors=True) self.transactions[txn_key] = (observable, self.clock.time_msec()) + + # if the request fails with a Twisted failure, remove it + # from the transaction map. This is done to ensure that we don't + # cache transient errors like rate-limiting errors, etc. + def remove_from_map(err): + del self.transactions[txn_key] + return err + observable.addErrback(remove_from_map) return observable.observe() def _cleanup(self): From 808ddf0ae72ef45d887e00c07ba834d0873ceb8d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 13 Feb 2017 13:36:15 +0000 Subject: [PATCH 13/82] Pop the txn from the map in case it has already been deleted somehow --- synapse/rest/client/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 95376a2fb..7b742ca7a 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -97,7 +97,7 @@ class HttpTransactionCache(object): # from the transaction map. This is done to ensure that we don't # cache transient errors like rate-limiting errors, etc. def remove_from_map(err): - del self.transactions[txn_key] + self.transactions.pop(txn_key, None) return err observable.addErrback(remove_from_map) return observable.observe() From d0497425f81ed21b58046f5f8725fc70ffcc0544 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 13 Feb 2017 13:49:44 +0000 Subject: [PATCH 14/82] Ordering is important on errbacks so add the cleanup func before creating an ObservableDeferred --- synapse/rest/client/transactions.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 7b742ca7a..fceca2ede 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -87,19 +87,19 @@ class HttpTransactionCache(object): deferred = fn(*args, **kwargs) - # We don't add an errback to the raw deferred, so we ask ObservableDeferred - # to swallow the error. This is fine as the error will still be reported - # to the observers. - observable = ObservableDeferred(deferred, consumeErrors=True) - self.transactions[txn_key] = (observable, self.clock.time_msec()) - # if the request fails with a Twisted failure, remove it # from the transaction map. This is done to ensure that we don't # cache transient errors like rate-limiting errors, etc. def remove_from_map(err): self.transactions.pop(txn_key, None) return err - observable.addErrback(remove_from_map) + deferred.addErrback(remove_from_map) + + # We don't add any other errbacks to the raw deferred, so we ask + # ObservableDeferred to swallow the error. This is fine as the error will + # still be reported to the observers. + observable = ObservableDeferred(deferred, consumeErrors=True) + self.transactions[txn_key] = (observable, self.clock.time_msec()) return observable.observe() def _cleanup(self): From 9e617cd4c2da527caf93799e286b42352ad492d2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Feb 2017 13:50:03 +0000 Subject: [PATCH 15/82] Cache get_presence storage --- synapse/handlers/presence.py | 2 +- synapse/replication/slave/storage/presence.py | 4 +++- synapse/storage/presence.py | 14 +++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index fdfce2a88..da610e430 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -531,7 +531,7 @@ class PresenceHandler(object): # There are things not in our in memory cache. Lets pull them out of # the database. res = yield self.store.get_presence_for_users(missing) - states.update({state.user_id: state for state in res}) + states.update(res) missing = [user_id for user_id, state in states.items() if not state] if missing: diff --git a/synapse/replication/slave/storage/presence.py b/synapse/replication/slave/storage/presence.py index 703f4a49b..40f6c9a38 100644 --- a/synapse/replication/slave/storage/presence.py +++ b/synapse/replication/slave/storage/presence.py @@ -18,6 +18,7 @@ from ._slaved_id_tracker import SlavedIdTracker from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.storage import DataStore +from synapse.storage.presence import PresenceStore class SlavedPresenceStore(BaseSlavedStore): @@ -35,7 +36,8 @@ class SlavedPresenceStore(BaseSlavedStore): _get_active_presence = DataStore._get_active_presence.__func__ take_presence_startup_info = DataStore.take_presence_startup_info.__func__ - get_presence_for_users = DataStore.get_presence_for_users.__func__ + _get_presence_for_user = PresenceStore.__dict__["_get_presence_for_user"] + get_presence_for_users = PresenceStore.__dict__["get_presence_for_users"] def get_current_presence_token(self): return self._presence_id_gen.get_current_token() diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 7460f98a1..4d1590d2b 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -15,7 +15,7 @@ from ._base import SQLBaseStore from synapse.api.constants import PresenceState -from synapse.util.caches.descriptors import cached, cachedInlineCallbacks +from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList from collections import namedtuple from twisted.internet import defer @@ -85,6 +85,9 @@ class PresenceStore(SQLBaseStore): self.presence_stream_cache.entity_has_changed, state.user_id, stream_id, ) + self._invalidate_cache_and_stream( + txn, self._get_presence_for_user, (state.user_id,) + ) # Actually insert new rows self._simple_insert_many_txn( @@ -143,7 +146,12 @@ class PresenceStore(SQLBaseStore): "get_all_presence_updates", get_all_presence_updates_txn ) - @defer.inlineCallbacks + @cached() + def _get_presence_for_user(self, user_id): + raise NotImplementedError() + + @cachedList(cached_method_name="_get_presence_for_user", list_name="user_ids", + num_args=1, inlineCallbacks=True) def get_presence_for_users(self, user_ids): rows = yield self._simple_select_many_batch( table="presence_stream", @@ -165,7 +173,7 @@ class PresenceStore(SQLBaseStore): for row in rows: row["currently_active"] = bool(row["currently_active"]) - defer.returnValue([UserPresenceState(**row) for row in rows]) + defer.returnValue({row["user_id"]: UserPresenceState(**row) for row in rows}) def get_current_presence_token(self): return self._presence_id_gen.get_current_token() From 095b45c1653bc93788a45b891f29efc30f0b1b07 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 3 Feb 2017 18:12:53 +0000 Subject: [PATCH 16/82] Aggregate event push actions --- synapse/replication/slave/storage/events.py | 6 + synapse/storage/event_push_actions.py | 271 ++++++++++++++---- synapse/storage/receipts.py | 1 + .../schema/delta/40/event_push_summary.sql | 37 +++ tests/storage/test_event_push_actions.py | 86 ++++++ 5 files changed, 342 insertions(+), 59 deletions(-) create mode 100644 synapse/storage/schema/delta/40/event_push_summary.sql diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py index d72ff6055..622b2d854 100644 --- a/synapse/replication/slave/storage/events.py +++ b/synapse/replication/slave/storage/events.py @@ -85,6 +85,12 @@ class SlavedEventStore(BaseSlavedStore): get_unread_event_push_actions_by_room_for_user = ( EventPushActionsStore.__dict__["get_unread_event_push_actions_by_room_for_user"] ) + _get_unread_counts_by_receipt_txn = ( + DataStore._get_unread_counts_by_receipt_txn.__func__ + ) + _get_unread_counts_by_pos_txn = ( + DataStore._get_unread_counts_by_pos_txn.__func__ + ) _get_state_group_for_events = ( StateStore.__dict__["_get_state_group_for_events"] ) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 522d0114c..300571b78 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -15,6 +15,7 @@ from ._base import SQLBaseStore from twisted.internet import defer +from synapse.util.async import sleep from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.types import RoomStreamToken from .stream import lower_bound @@ -29,7 +30,6 @@ class EventPushActionsStore(SQLBaseStore): EPA_HIGHLIGHT_INDEX = "epa_highlight_index" def __init__(self, hs): - self.stream_ordering_month_ago = None super(EventPushActionsStore, self).__init__(hs) self.register_background_index_update( @@ -47,6 +47,9 @@ class EventPushActionsStore(SQLBaseStore): where_clause="highlight=1" ) + self._doing_notif_rotation = False + self._clock.looping_call(self._rotate_notifs, 60 * 1000) + def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): """ Args: @@ -77,67 +80,90 @@ class EventPushActionsStore(SQLBaseStore): def get_unread_event_push_actions_by_room_for_user( self, room_id, user_id, last_read_event_id ): - def _get_unread_event_push_actions_by_room(txn): - sql = ( - "SELECT stream_ordering, topological_ordering" - " FROM events" - " WHERE room_id = ? AND event_id = ?" - ) - txn.execute( - sql, (room_id, last_read_event_id) - ) - results = txn.fetchall() - if len(results) == 0: - return {"notify_count": 0, "highlight_count": 0} - - stream_ordering = results[0][0] - topological_ordering = results[0][1] - token = RoomStreamToken( - topological_ordering, stream_ordering - ) - - # First get number of notifications. - # We don't need to put a notif=1 clause as all rows always have - # notif=1 - sql = ( - "SELECT count(*)" - " FROM event_push_actions ea" - " WHERE" - " user_id = ?" - " AND room_id = ?" - " AND %s" - ) % (lower_bound(token, self.database_engine, inclusive=False),) - - txn.execute(sql, (user_id, room_id)) - row = txn.fetchone() - notify_count = row[0] if row else 0 - - # Now get the number of highlights - sql = ( - "SELECT count(*)" - " FROM event_push_actions ea" - " WHERE" - " highlight = 1" - " AND user_id = ?" - " AND room_id = ?" - " AND %s" - ) % (lower_bound(token, self.database_engine, inclusive=False),) - - txn.execute(sql, (user_id, room_id)) - row = txn.fetchone() - highlight_count = row[0] if row else 0 - - return { - "notify_count": notify_count, - "highlight_count": highlight_count, - } - ret = yield self.runInteraction( "get_unread_event_push_actions_by_room", - _get_unread_event_push_actions_by_room + self._get_unread_counts_by_receipt_txn, + room_id, user_id, last_read_event_id ) defer.returnValue(ret) + def _get_unread_counts_by_receipt_txn(self, txn, room_id, user_id, + last_read_event_id): + sql = ( + "SELECT stream_ordering, topological_ordering" + " FROM events" + " WHERE room_id = ? AND event_id = ?" + ) + txn.execute( + sql, (room_id, last_read_event_id) + ) + results = txn.fetchall() + if len(results) == 0: + return {"notify_count": 0, "highlight_count": 0} + + stream_ordering = results[0][0] + topological_ordering = results[0][1] + + return self._get_unread_counts_by_pos_txn( + txn, room_id, user_id, topological_ordering, stream_ordering + ) + + def _get_unread_counts_by_pos_txn(self, txn, room_id, user_id, topological_ordering, + stream_ordering): + token = RoomStreamToken( + topological_ordering, stream_ordering + ) + + # First get number of notifications. + # We don't need to put a notif=1 clause as all rows always have + # notif=1 + sql = ( + "SELECT count(*)" + " FROM event_push_actions ea" + " WHERE" + " user_id = ?" + " AND room_id = ?" + " AND %s" + ) % (lower_bound(token, self.database_engine, inclusive=False),) + + txn.execute(sql, (user_id, room_id)) + row = txn.fetchone() + notify_count = row[0] if row else 0 + + summary_notif_count = self._simple_select_one_onecol_txn( + txn, + table="event_push_summary", + keyvalues={ + "user_id": user_id, + "room_id": room_id, + }, + retcol="notif_count", + allow_none=True, + ) + + if summary_notif_count: + notify_count += summary_notif_count + + # Now get the number of highlights + sql = ( + "SELECT count(*)" + " FROM event_push_actions ea" + " WHERE" + " highlight = 1" + " AND user_id = ?" + " AND room_id = ?" + " AND %s" + ) % (lower_bound(token, self.database_engine, inclusive=False),) + + txn.execute(sql, (user_id, room_id)) + row = txn.fetchone() + highlight_count = row[0] if row else 0 + + return { + "notify_count": notify_count, + "highlight_count": highlight_count, + } + @defer.inlineCallbacks def get_push_action_users_in_range(self, min_stream_ordering, max_stream_ordering): def f(txn): @@ -448,7 +474,7 @@ class EventPushActionsStore(SQLBaseStore): ) def _remove_old_push_actions_before_txn(self, txn, room_id, user_id, - topological_ordering): + topological_ordering, stream_ordering): """ Purges old push actions for a user and room before a given topological_ordering. @@ -479,11 +505,16 @@ class EventPushActionsStore(SQLBaseStore): txn.execute( "DELETE FROM event_push_actions " " WHERE user_id = ? AND room_id = ? AND " - " topological_ordering < ?" + " topological_ordering <= ?" " AND ((stream_ordering < ? AND highlight = 1) or highlight = 0)", (user_id, room_id, topological_ordering, self.stream_ordering_month_ago) ) + txn.execute(""" + DELETE FROM event_push_summary + WHERE room_id = ? AND user_id = ? AND stream_ordering <= ? + """, (room_id, user_id, stream_ordering)) + @defer.inlineCallbacks def _find_stream_orderings_for_times(self): yield self.runInteraction( @@ -500,6 +531,14 @@ class EventPushActionsStore(SQLBaseStore): "Found stream ordering 1 month ago: it's %d", self.stream_ordering_month_ago ) + logger.info("Searching for stream ordering 1 day ago") + self.stream_ordering_day_ago = self._find_first_stream_ordering_after_ts_txn( + txn, self._clock.time_msec() - 24 * 60 * 60 * 1000 + ) + logger.info( + "Found stream ordering 1 day ago: it's %d", + self.stream_ordering_day_ago + ) def _find_first_stream_ordering_after_ts_txn(self, txn, ts): """ @@ -539,6 +578,120 @@ class EventPushActionsStore(SQLBaseStore): return range_end + @defer.inlineCallbacks + def _rotate_notifs(self): + if self._doing_notif_rotation or self.stream_ordering_day_ago is None: + return + self._doing_notif_rotation = True + + try: + while True: + logger.info("Rotating notifications") + + caught_up = yield self.runInteraction( + "_rotate_notifs", + self._rotate_notifs_txn + ) + if caught_up: + break + yield sleep(1) + finally: + self._doing_notif_rotation = False + + def _rotate_notifs_txn(self, txn): + """Archives older notifications into event_push_summary. Returns whether + the archiving process has caught up or not. + """ + + # We want to make sure that we only ever do this one at a time + self.database_engine.lock_table(txn, "event_push_summary") + + # We don't to try and rotate millions of rows at once, so we cap the + # maximum stream ordering we'll rotate before. + txn.execute(""" + SELECT stream_ordering FROM event_push_actions + ORDER BY stream_ordering ASC LIMIT 1 OFFSET 50000 + """) + stream_row = txn.fetchone() + if stream_row: + offset_stream_ordering, = stream_row + rotate_to_stream_ordering = min( + self.stream_ordering_day_ago, offset_stream_ordering + ) + caught_up = offset_stream_ordering >= self.stream_ordering_day_ago + else: + rotate_to_stream_ordering = self.stream_ordering_day_ago + caught_up = True + + self._rotate_notifs_before_txn(txn, rotate_to_stream_ordering) + + # We have caught up iff we were limited by `stream_ordering_day_ago` + return caught_up + + def _rotate_notifs_before_txn(self, txn, rotate_to_stream_ordering): + old_rotate_stream_ordering = self._simple_select_one_onecol_txn( + txn, + table="event_push_summary_stream_ordering", + keyvalues={}, + retcol="stream_ordering", + ) + + # Calculate the new counts that should be upserted into event_push_summary + sql = """ + SELECT user_id, room_id, + coalesce(old.notif_count, 0) + upd.notif_count, + upd.stream_ordering, + old.user_id + FROM ( + SELECT user_id, room_id, count(*) as notif_count, + max(stream_ordering) as stream_ordering + FROM event_push_actions + WHERE ? <= stream_ordering AND stream_ordering < ? + AND highlight = 0 + GROUP BY user_id, room_id + ) AS upd + LEFT JOIN event_push_summary AS old USING (user_id, room_id) + """ + + txn.execute(sql, (old_rotate_stream_ordering, rotate_to_stream_ordering,)) + rows = txn.fetchall() + + # If the `old.user_id` above is NULL then we know there isn't already an + # entry in the table, so we simply insert it. Otherwise we update the + # existing table. + self._simple_insert_many_txn( + txn, + table="event_push_summary", + values=[ + { + "user_id": row[0], + "room_id": row[1], + "notif_count": row[2], + "stream_ordering": row[3], + } + for row in rows if row[4] is None + ] + ) + + txn.executemany( + """ + UPDATE event_push_summary SET notif_count = ?, stream_ordering = ? + WHERE user_id = ? AND room_id = ? + """, + ((row[2], row[3], row[0], row[1],) for row in rows if row[4] is not None) + ) + + txn.execute( + "DELETE FROM event_push_actions" + " WHERE ? <= stream_ordering AND stream_ordering < ? AND highlight = 0", + (old_rotate_stream_ordering, rotate_to_stream_ordering,) + ) + + txn.execute( + "UPDATE event_push_summary_stream_ordering SET stream_ordering = ?", + (rotate_to_stream_ordering,) + ) + def _action_has_highlight(actions): for action in actions: diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index f72d15f5e..5cf41501e 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -351,6 +351,7 @@ class ReceiptsStore(SQLBaseStore): room_id=room_id, user_id=user_id, topological_ordering=topological_ordering, + stream_ordering=stream_ordering, ) return True diff --git a/synapse/storage/schema/delta/40/event_push_summary.sql b/synapse/storage/schema/delta/40/event_push_summary.sql new file mode 100644 index 000000000..3918f0b79 --- /dev/null +++ b/synapse/storage/schema/delta/40/event_push_summary.sql @@ -0,0 +1,37 @@ +/* Copyright 2017 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Aggregate of old notification counts that have been deleted out of the +-- main event_push_actions table. This count does not include those that were +-- highlights, as they remain in the event_push_actions table. +CREATE TABLE event_push_summary ( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + notif_count BIGINT NOT NULL, + stream_ordering BIGINT NOT NULL +); + +CREATE INDEX event_push_summary_user_rm ON event_push_summary(user_id, room_id); + + +-- The stream ordering up to which we have aggregated the event_push_actions +-- table into event_push_summary +CREATE TABLE event_push_summary_stream_ordering ( + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + stream_ordering BIGINT NOT NULL, + CHECK (Lock='X') +); + +INSERT INTO event_push_summary_stream_ordering (stream_ordering) VALUES (0); diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index e9044afa2..313548835 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -17,9 +17,15 @@ from twisted.internet import defer import tests.unittest import tests.utils +from mock import Mock USER_ID = "@user:example.com" +PlAIN_NOTIF = ["notify", {"set_tweak": "highlight", "value": False}] +HIGHLIGHT = [ + "notify", {"set_tweak": "sound", "value": "default"}, {"set_tweak": "highlight"} +] + class EventPushActionsStoreTestCase(tests.unittest.TestCase): @@ -39,3 +45,83 @@ class EventPushActionsStoreTestCase(tests.unittest.TestCase): yield self.store.get_unread_push_actions_for_user_in_range_for_email( USER_ID, 0, 1000, 20 ) + + @defer.inlineCallbacks + def test_count_aggregation(self): + room_id = "!foo:example.com" + user_id = "@user1235:example.com" + + @defer.inlineCallbacks + def _assert_counts(noitf_count, highlight_count): + counts = yield self.store.runInteraction( + "", self.store._get_unread_counts_by_pos_txn, + room_id, user_id, 0, 0 + ) + self.assertEquals( + counts, + {"notify_count": noitf_count, "highlight_count": highlight_count} + ) + + def _inject_actions(stream, action): + event = Mock() + event.room_id = room_id + event.event_id = "$test:example.com" + event.internal_metadata.stream_ordering = stream + event.depth = stream + + tuples = [(user_id, action)] + + return self.store.runInteraction( + "", self.store._set_push_actions_for_event_and_users_txn, + event, tuples + ) + + def _rotate(stream): + return self.store.runInteraction( + "", self.store._rotate_notifs_before_txn, stream + ) + + def _mark_read(stream, depth): + return self.store.runInteraction( + "", self.store._remove_old_push_actions_before_txn, + room_id, user_id, depth, stream + ) + + yield _assert_counts(0, 0) + yield _inject_actions(1, PlAIN_NOTIF) + yield _assert_counts(1, 0) + yield _rotate(2) + yield _assert_counts(1, 0) + + yield _inject_actions(3, PlAIN_NOTIF) + yield _assert_counts(2, 0) + yield _rotate(4) + yield _assert_counts(2, 0) + + yield _inject_actions(5, PlAIN_NOTIF) + yield _mark_read(3, 3) + yield _assert_counts(1, 0) + + yield _mark_read(5, 5) + yield _assert_counts(0, 0) + + yield _inject_actions(6, PlAIN_NOTIF) + yield _rotate(7) + + yield self.store._simple_delete( + table="event_push_actions", + keyvalues={"1": 1}, + desc="", + ) + + yield _assert_counts(1, 0) + + yield _mark_read(7, 7) + yield _assert_counts(0, 0) + + yield _inject_actions(8, HIGHLIGHT) + yield _assert_counts(1, 1) + yield _rotate(9) + yield _assert_counts(1, 1) + yield _rotate(10) + yield _assert_counts(1, 1) From ce3c8df6dff456cdc548f1bc40a47f4b2a09a0a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Feb 2017 13:41:24 +0000 Subject: [PATCH 17/82] Less aggressive timers --- synapse/storage/event_push_actions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 300571b78..1b7888a91 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -48,7 +48,7 @@ class EventPushActionsStore(SQLBaseStore): ) self._doing_notif_rotation = False - self._clock.looping_call(self._rotate_notifs, 60 * 1000) + self._clock.looping_call(self._rotate_notifs, 30 * 60 * 1000) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): """ @@ -594,7 +594,7 @@ class EventPushActionsStore(SQLBaseStore): ) if caught_up: break - yield sleep(1) + yield sleep(5) finally: self._doing_notif_rotation = False From fc2f29c1d071ff9b140f10ca46f463f6638d8357 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 14 Feb 2017 13:59:50 +0000 Subject: [PATCH 18/82] Fix bugs in the /keys/changes api * `get_forward_extremeties_for_room` takes a numeric `stream_ordering`. We were passing a `RoomStreamToken`, which meant that it returned the *current* extremities, rather than those corresponding to the `from_token`. However: * `get_state_ids_for_events` required a second ('types') parameter; this meant that a `TypeError` was thrown and we ended up acting as though there was *no* prev state. * `get_state_ids_for_events` actually returns a map from event_id to state dictionary - just looking up the state keys in it again meant that we acted as though there was no prev state. We now check if each member's state has changed since *any* of the extremities. Also add/fix some comments. --- synapse/handlers/device.py | 38 ++++++++++++++++++++++------- synapse/storage/event_federation.py | 17 ++++++++++++- synapse/storage/state.py | 14 ++++++++++- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 8cb47ac41..ca7137f31 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from synapse.api import errors from synapse.api.constants import EventTypes from synapse.util import stringutils @@ -246,30 +245,51 @@ class DeviceHandler(BaseHandler): # Then work out if any users have since joined rooms_changed = self.store.get_rooms_that_changed(room_ids, from_token.room_key) + stream_ordering = RoomStreamToken.parse_stream_token( + from_token.room_key).stream + possibly_changed = set(changed) for room_id in rooms_changed: - # Fetch the current state at the time. - stream_ordering = RoomStreamToken.parse_stream_token(from_token.room_key) - + # Fetch the current state at the time. try: event_ids = yield self.store.get_forward_extremeties_for_room( room_id, stream_ordering=stream_ordering ) - prev_state_ids = yield self.store.get_state_ids_for_events(event_ids) - except: - prev_state_ids = {} + except errors.StoreError: + # we have purged the stream_ordering index since the stream + # ordering: treat it the same as a new room + event_ids = [] current_state_ids = yield self.state.get_current_state_ids(room_id) + # special-case for an empty prev state: include all members + # in the changed list + if not event_ids: + for key, event_id in current_state_ids.iteritems(): + etype, state_key = key + if etype != EventTypes.Member: + continue + possibly_changed.add(state_key) + continue + + # mapping from event_id -> state_dict + prev_state_ids = yield self.store.get_state_ids_for_events(event_ids) + # If there has been any change in membership, include them in the # possibly changed list. We'll check if they are joined below, # and we're not toooo worried about spuriously adding users. for key, event_id in current_state_ids.iteritems(): etype, state_key = key - if etype == EventTypes.Member: - prev_event_id = prev_state_ids.get(key, None) + if etype != EventTypes.Member: + continue + + # check if this member has changed since any of the extremities + # at the stream_ordering, and add them to the list if so. + for state_dict in prev_state_ids.values(): + prev_event_id = state_dict.get(key, None) if not prev_event_id or prev_event_id != event_id: possibly_changed.add(state_key) + break users_who_share_room = yield self.store.get_users_who_share_room_with_user( user_id diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index ee88c6195..256e50dc2 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -281,15 +281,30 @@ class EventFederationStore(SQLBaseStore): ) def get_forward_extremeties_for_room(self, room_id, stream_ordering): + """For a given room_id and stream_ordering, return the forward + extremeties of the room at that point in "time". + + Throws a StoreError if we have since purged the index for + stream_orderings from that point. + + Args: + room_id (str): + stream_ordering (int): + + Returns: + deferred, which resolves to a list of event_ids + """ # We want to make the cache more effective, so we clamp to the last # change before the given ordering. last_change = self._events_stream_cache.get_max_pos_of_last_change(room_id) # We don't always have a full stream_to_exterm_id table, e.g. after # the upgrade that introduced it, so we make sure we never ask for a - # try and pin to a stream_ordering from before a restart + # stream_ordering from before a restart last_change = max(self._stream_order_on_start, last_change) + # provided the last_change is recent enough, we now clamp the requested + # stream_ordering to it. if last_change > self.stream_ordering_month_ago: stream_ordering = min(last_change, stream_ordering) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 1b3800eb6..84482d828 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -413,7 +413,19 @@ class StateStore(SQLBaseStore): defer.returnValue({event: event_to_state[event] for event in event_ids}) @defer.inlineCallbacks - def get_state_ids_for_events(self, event_ids, types): + def get_state_ids_for_events(self, event_ids, types=None): + """ + Get the state dicts corresponding to a list of events + + Args: + event_ids(list(str)): events whose state should be returned + types(list[(str, str)]|None): List of (type, state_key) tuples + which are used to filter the state fetched. May be None, which + matches any key + + Returns: + A deferred dict from event_id -> (type, state_key) -> state_event + """ event_to_groups = yield self._get_state_group_for_events( event_ids, ) From ce3e583d94c9fb3ee98365e07f4695d1b9451434 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 14 Feb 2017 15:05:55 +0000 Subject: [PATCH 19/82] WIP support for msisdn 3pid proxy methods --- synapse/api/constants.py | 2 + synapse/handlers/auth.py | 30 +++++-- synapse/handlers/identity.py | 37 +++++++- synapse/python_dependencies.py | 2 + synapse/rest/client/v2_alpha/account.py | 110 +++++++++++++++++++++-- synapse/rest/client/v2_alpha/register.py | 66 ++++++++++++-- 6 files changed, 228 insertions(+), 19 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index ca23c9c46..489efb7f8 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -44,6 +45,7 @@ class JoinRules(object): class LoginType(object): PASSWORD = u"m.login.password" EMAIL_IDENTITY = u"m.login.email.identity" + MSISDN = u"m.login.msisdn" RECAPTCHA = u"m.login.recaptcha" DUMMY = u"m.login.dummy" diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index fffba3438..448bc0b31 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014 - 2016 OpenMarket Ltd +# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,6 +48,7 @@ class AuthHandler(BaseHandler): LoginType.PASSWORD: self._check_password_auth, LoginType.RECAPTCHA: self._check_recaptcha, LoginType.EMAIL_IDENTITY: self._check_email_identity, + LoginType.MSISDN: self._check_msisdn, LoginType.DUMMY: self._check_dummy_auth, } self.bcrypt_rounds = hs.config.bcrypt_rounds @@ -309,12 +311,26 @@ class AuthHandler(BaseHandler): @defer.inlineCallbacks def _check_email_identity(self, authdict, _): + defer.returnValue(self._check_threepid('email', authdict)) + + @defer.inlineCallbacks + def _check_msisdn(self, authdict, _): + defer.returnValue(self._check_threepid('msisdn', authdict)) + + @defer.inlineCallbacks + def _check_dummy_auth(self, authdict, _): + yield run_on_reactor() + defer.returnValue(True) + + @defer.inlineCallbacks + def _check_threepid(self, medium, authdict, ): yield run_on_reactor() if 'threepid_creds' not in authdict: raise LoginError(400, "Missing threepid_creds", Codes.MISSING_PARAM) threepid_creds = authdict['threepid_creds'] + identity_handler = self.hs.get_handlers().identity_handler logger.info("Getting validated threepid. threepidcreds: %r" % (threepid_creds,)) @@ -323,15 +339,19 @@ class AuthHandler(BaseHandler): if not threepid: raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + if threepid['medium'] != medium: + raise LoginError( + 401, + "Expecting threepid of type '%s', got '%s'" % ( + medium, threepid['medium'], + ), + errcode=Codes.UNAUTHORIZED + ) + threepid['threepid_creds'] = authdict['threepid_creds'] defer.returnValue(threepid) - @defer.inlineCallbacks - def _check_dummy_auth(self, authdict, _): - yield run_on_reactor() - defer.returnValue(True) - def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 559e5d5a7..6a53c5eb4 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -150,7 +151,7 @@ class IdentityHandler(BaseHandler): params.update(kwargs) try: - data = yield self.http_client.post_urlencoded_get_json( + data = yield self.http_client.post_json_get_json( "https://%s%s" % ( id_server, "/_matrix/identity/api/v1/validate/email/requestToken" @@ -161,3 +162,37 @@ class IdentityHandler(BaseHandler): except CodeMessageException as e: logger.info("Proxied requestToken failed: %r", e) raise e + + @defer.inlineCallbacks + def requestMsisdnToken( + self, id_server, country, phone_number, + client_secret, send_attempt, **kwargs + ): + yield run_on_reactor() + + if not self._should_trust_id_server(id_server): + raise SynapseError( + 400, "Untrusted ID server '%s'" % id_server, + Codes.SERVER_NOT_TRUSTED + ) + + params = { + 'country': country, + 'phone_number': phone_number, + 'client_secret': client_secret, + 'send_attempt': send_attempt, + } + params.update(kwargs) + + try: + data = yield self.http_client.post_json_get_json( + "https://%s%s" % ( + id_server, + "/_matrix/identity/api/v1/validate/msisdn/requestToken" + ), + params + ) + defer.returnValue(data) + except CodeMessageException as e: + logger.info("Proxied requestToken failed: %r", e) + raise e diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 7817b0cd9..c4777b2a2 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -1,4 +1,5 @@ # Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,6 +38,7 @@ REQUIREMENTS = { "pysaml2>=3.0.0,<4.0.0": ["saml2>=3.0.0,<4.0.0"], "pymacaroons-pynacl": ["pymacaroons"], "msgpack-python>=0.3.0": ["msgpack"], + "phonenumbers>=8.2.0": ["phonenumbers"], } CONDITIONAL_REQUIREMENTS = { "web_client": { diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 398e7f5eb..cf80f5ca2 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,15 +25,17 @@ from ._base import client_v2_patterns import logging +import phonenumbers + logger = logging.getLogger(__name__) -class PasswordRequestTokenRestServlet(RestServlet): +class EmailPasswordRequestTokenRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/password/email/requestToken$") def __init__(self, hs): - super(PasswordRequestTokenRestServlet, self).__init__() + super(EmailPasswordRequestTokenRestServlet, self).__init__() self.hs = hs self.identity_handler = hs.get_handlers().identity_handler @@ -60,6 +63,50 @@ class PasswordRequestTokenRestServlet(RestServlet): defer.returnValue((200, ret)) +class MsisdnPasswordRequestTokenRestServlet(RestServlet): + PATTERNS = client_v2_patterns("/account/password/msisdn/requestToken$") + + def __init__(self, hs): + super(MsisdnPasswordRequestTokenRestServlet, self).__init__() + self.hs = hs + self.identity_handler = hs.get_handlers().identity_handler + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_json_object_from_request(request) + + required = [ + 'id_server', 'client_secret', + 'country', 'phone_number', 'send_attempt', + ] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + if absent: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + + phoneNumber = None + try: + phoneNumber = phonenumbers.parse(body['phone_number'], body['country']) + except phonenumbers.NumberParseException: + raise SynapseError(400, "Unable to parse phone number") + msisdn = phonenumbers.format_number( + phoneNumber, phonenumbers.PhoneNumberFormat.E164 + )[1:] + + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( + 'msisdn', msisdn + ) + + if existingUid is None: + raise SynapseError(400, "MSISDN not found", Codes.THREEPID_NOT_FOUND) + + ret = yield self.identity_handler.requestEmailToken(**body) + defer.returnValue((200, ret)) + + class PasswordRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/password$") @@ -77,7 +124,8 @@ class PasswordRestServlet(RestServlet): authed, result, params, _ = yield self.auth_handler.check_auth([ [LoginType.PASSWORD], - [LoginType.EMAIL_IDENTITY] + [LoginType.EMAIL_IDENTITY], + [LoginType.MSISDN], ], body, self.hs.get_ip_from_request(request)) if not authed: @@ -169,12 +217,12 @@ class DeactivateAccountRestServlet(RestServlet): defer.returnValue((200, {})) -class ThreepidRequestTokenRestServlet(RestServlet): +class EmailThreepidRequestTokenRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/3pid/email/requestToken$") def __init__(self, hs): self.hs = hs - super(ThreepidRequestTokenRestServlet, self).__init__() + super(EmailThreepidRequestTokenRestServlet, self).__init__() self.identity_handler = hs.get_handlers().identity_handler @defer.inlineCallbacks @@ -201,6 +249,50 @@ class ThreepidRequestTokenRestServlet(RestServlet): defer.returnValue((200, ret)) +class MsisdnThreepidRequestTokenRestServlet(RestServlet): + PATTERNS = client_v2_patterns("/account/3pid/msisdn/requestToken$") + + def __init__(self, hs): + self.hs = hs + super(MsisdnThreepidRequestTokenRestServlet, self).__init__() + self.identity_handler = hs.get_handlers().identity_handler + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_json_object_from_request(request) + + required = [ + 'id_server', 'client_secret', + 'country', 'phone_number', 'send_attempt', + ] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + if absent: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + + phoneNumber = None + try: + phoneNumber = phonenumbers.parse(body['phone_number'], body['country']) + except phonenumbers.NumberParseException: + raise SynapseError(400, "Unable to parse phone number") + msisdn = phonenumbers.format_number( + phoneNumber, phonenumbers.PhoneNumberFormat.E164 + )[1:] + + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( + 'msisdn', msisdn + ) + + if existingUid is not None: + raise SynapseError(400, "MSISDN is already in use", Codes.THREEPID_IN_USE) + + ret = yield self.identity_handler.requestEmailToken(**body) + defer.returnValue((200, ret)) + + class ThreepidRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/3pid$") @@ -258,7 +350,7 @@ class ThreepidRestServlet(RestServlet): if 'bind' in body and body['bind']: logger.debug( - "Binding emails %s to %s", + "Binding threepid %s to %s", threepid, user_id ) yield self.identity_handler.bind_threepid( @@ -302,9 +394,11 @@ class ThreepidDeleteRestServlet(RestServlet): def register_servlets(hs, http_server): - PasswordRequestTokenRestServlet(hs).register(http_server) + EmailPasswordRequestTokenRestServlet(hs).register(http_server) + MsisdnPasswordRequestTokenRestServlet(hs).register(http_server) PasswordRestServlet(hs).register(http_server) DeactivateAccountRestServlet(hs).register(http_server) - ThreepidRequestTokenRestServlet(hs).register(http_server) + EmailThreepidRequestTokenRestServlet(hs).register(http_server) + MsisdnThreepidRequestTokenRestServlet(hs).register(http_server) ThreepidRestServlet(hs).register(http_server) ThreepidDeleteRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index ccca5a12d..39f61d70b 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2015 - 2016 OpenMarket Ltd +# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,6 +26,7 @@ from ._base import client_v2_patterns import logging import hmac +import phonenumbers from hashlib import sha1 from synapse.util.async import run_on_reactor @@ -43,7 +45,7 @@ else: logger = logging.getLogger(__name__) -class RegisterRequestTokenRestServlet(RestServlet): +class EmailRegisterRequestTokenRestServlet(RestServlet): PATTERNS = client_v2_patterns("/register/email/requestToken$") def __init__(self, hs): @@ -51,7 +53,7 @@ class RegisterRequestTokenRestServlet(RestServlet): Args: hs (synapse.server.HomeServer): server """ - super(RegisterRequestTokenRestServlet, self).__init__() + super(EmailRegisterRequestTokenRestServlet, self).__init__() self.hs = hs self.identity_handler = hs.get_handlers().identity_handler @@ -79,6 +81,55 @@ class RegisterRequestTokenRestServlet(RestServlet): defer.returnValue((200, ret)) +class MsisdnRegisterRequestTokenRestServlet(RestServlet): + PATTERNS = client_v2_patterns("/register/msisdn/requestToken$") + + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): server + """ + super(MsisdnRegisterRequestTokenRestServlet, self).__init__() + self.hs = hs + self.identity_handler = hs.get_handlers().identity_handler + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_json_object_from_request(request) + + required = [ + 'id_server', 'client_secret', + 'country', 'phone_number', + 'send_attempt', + ] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + if len(absent) > 0: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + + phoneNumber = None + try: + phoneNumber = phonenumbers.parse(body['phone_number'], body['country']) + except phonenumbers.NumberParseException: + raise SynapseError(400, "Unable to parse phone number") + msisdn = phonenumbers.format_number( + phoneNumber, phonenumbers.PhoneNumberFormat.E164 + )[1:] + + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( + 'msisdn', msisdn + ) + + if existingUid is not None: + raise SynapseError(400, "MSISDN is already in use", Codes.THREEPID_IN_USE) + + ret = yield self.identity_handler.requestMsisdnToken(**body) + defer.returnValue((200, ret)) + + class RegisterRestServlet(RestServlet): PATTERNS = client_v2_patterns("/register$") @@ -203,12 +254,16 @@ class RegisterRestServlet(RestServlet): if self.hs.config.enable_registration_captcha: flows = [ [LoginType.RECAPTCHA], - [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA] + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], + [LoginType.MSISDN, LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.MSISDN, LoginType.RECAPTCHA], ] else: flows = [ [LoginType.DUMMY], - [LoginType.EMAIL_IDENTITY] + [LoginType.EMAIL_IDENTITY], + [LoginType.MSISDN], + [LoginType.EMAIL_IDENTITY, LoginType.MSISDN], ] authed, auth_result, params, session_id = yield self.auth_handler.check_auth( @@ -449,5 +504,6 @@ class RegisterRestServlet(RestServlet): def register_servlets(hs, http_server): - RegisterRequestTokenRestServlet(hs).register(http_server) + EmailRegisterRequestTokenRestServlet(hs).register(http_server) + MsisdnRegisterRequestTokenRestServlet(hs).register(http_server) RegisterRestServlet(hs).register(http_server) From 355d62c499b63f337c484cca6704161a3a05da1d Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 14 Feb 2017 15:10:55 +0000 Subject: [PATCH 20/82] Make kick & ban reasons work We somehow specced APIs with reason strings, preserve the content in the events and even have the clients display them, but failed to actually pass the parameter through to the event content. --- synapse/rest/client/v1/room.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 2ebf5e59a..728e3df0e 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -609,6 +609,10 @@ class RoomMembershipRestServlet(ClientV1RestServlet): raise SynapseError(400, "Missing user_id key.") target = UserID.from_string(content["user_id"]) + event_content = None + if 'reason' in content and membership_action in ['kick', 'ban']: + event_content = {'reason': content['reason']} + yield self.handlers.room_member_handler.update_membership( requester=requester, target=target, @@ -616,6 +620,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): action=membership_action, txn_id=txn_id, third_party_signed=content.get("third_party_signed", None), + content=event_content, ) defer.returnValue((200, {})) From 474c9aadbe542440bb7cd764a654518b5c36f851 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 15 Feb 2017 19:32:20 +0000 Subject: [PATCH 21/82] Allow forgetting rooms you're banned from --- synapse/handlers/room_member.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b2806555c..2052d6d05 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -719,7 +719,9 @@ class RoomMemberHandler(BaseHandler): ) membership = member.membership if member else None - if membership is not None and membership != Membership.LEAVE: + if membership is not None and membership not in [ + Membership.LEAVE, Membership.BAN + ]: raise SynapseError(400, "User %s in room %s" % ( user_id, room_id )) From e6acf0c399b1baadc56f57d8398643a4cb1de56a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Feb 2017 16:37:25 +0000 Subject: [PATCH 22/82] Store the default push actions in a more efficient manner --- synapse/storage/event_push_actions.py | 51 +++++++++++++++++++++------ 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 1b7888a91..db20b7de2 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -26,6 +26,32 @@ import ujson as json logger = logging.getLogger(__name__) +DEFAULT_NOTIF_ACITON = ["notify", {"set_tweak": "highlight", "value": False}] +DEFAULT_HIGHLIGHT_ACITON = [ + "notify", {"set_tweak": "sound", "value": "default"}, {"set_tweak": "highlight"} +] + + +def _serialize_action(actions, is_highlight): + if is_highlight: + if actions == DEFAULT_HIGHLIGHT_ACITON: + return "" + else: + if actions == DEFAULT_NOTIF_ACITON: + return "" + return json.dumps(actions) + + +def _deserialize_action(actions, is_highlight): + if actions: + return json.loads(actions) + + if is_highlight: + return DEFAULT_HIGHLIGHT_ACITON + else: + return DEFAULT_NOTIF_ACITON + + class EventPushActionsStore(SQLBaseStore): EPA_HIGHLIGHT_INDEX = "epa_highlight_index" @@ -58,15 +84,17 @@ class EventPushActionsStore(SQLBaseStore): """ values = [] for uid, actions in tuples: + is_highlight = 1 if _action_has_highlight(actions) else 0 + values.append({ 'room_id': event.room_id, 'event_id': event.event_id, 'user_id': uid, - 'actions': json.dumps(actions), + 'actions': _serialize_action(actions, is_highlight), 'stream_ordering': event.internal_metadata.stream_ordering, 'topological_ordering': event.depth, 'notif': 1, - 'highlight': 1 if _action_has_highlight(actions) else 0, + 'highlight': is_highlight, }) for uid, __ in tuples: @@ -202,7 +230,8 @@ class EventPushActionsStore(SQLBaseStore): # find rooms that have a read receipt in them and return the next # push actions sql = ( - "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions" + "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," + " ep.highlight " " FROM (" " SELECT room_id," " MAX(topological_ordering) as topological_ordering," @@ -243,7 +272,7 @@ class EventPushActionsStore(SQLBaseStore): def get_no_receipt(txn): sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," - " e.received_ts" + " ep.highlight " " FROM event_push_actions AS ep" " INNER JOIN events AS e USING (room_id, event_id)" " WHERE" @@ -272,7 +301,7 @@ class EventPushActionsStore(SQLBaseStore): "event_id": row[0], "room_id": row[1], "stream_ordering": row[2], - "actions": json.loads(row[3]), + "actions": _deserialize_action(row[3], row[4]), } for row in after_read_receipt + no_read_receipt ] @@ -311,7 +340,7 @@ class EventPushActionsStore(SQLBaseStore): def get_after_receipt(txn): sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," - " e.received_ts" + " ep.highlight, e.received_ts" " FROM (" " SELECT room_id," " MAX(topological_ordering) as topological_ordering," @@ -353,7 +382,7 @@ class EventPushActionsStore(SQLBaseStore): def get_no_receipt(txn): sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," - " e.received_ts" + " ep.highlight, e.received_ts" " FROM event_push_actions AS ep" " INNER JOIN events AS e USING (room_id, event_id)" " WHERE" @@ -383,8 +412,8 @@ class EventPushActionsStore(SQLBaseStore): "event_id": row[0], "room_id": row[1], "stream_ordering": row[2], - "actions": json.loads(row[3]), - "received_ts": row[4], + "actions": _deserialize_action(row[3], row[4]), + "received_ts": row[5], } for row in after_read_receipt + no_read_receipt ] @@ -418,7 +447,7 @@ class EventPushActionsStore(SQLBaseStore): sql = ( "SELECT epa.event_id, epa.room_id," " epa.stream_ordering, epa.topological_ordering," - " epa.actions, epa.profile_tag, e.received_ts" + " epa.actions, epa.highlight, epa.profile_tag, e.received_ts" " FROM event_push_actions epa, events e" " WHERE epa.event_id = e.event_id" " AND epa.user_id = ? %s" @@ -433,7 +462,7 @@ class EventPushActionsStore(SQLBaseStore): "get_push_actions_for_user", f ) for pa in push_actions: - pa["actions"] = json.loads(pa["actions"]) + pa["actions"] = _deserialize_action(pa["actions"], pa["highlight"]) defer.returnValue(push_actions) @defer.inlineCallbacks From 502ae6c663e9b96950d1b474fbec7cf200e4e8a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Feb 2017 16:54:37 +0000 Subject: [PATCH 23/82] Comment --- synapse/storage/event_push_actions.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index db20b7de2..ea6722df6 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -26,30 +26,37 @@ import ujson as json logger = logging.getLogger(__name__) -DEFAULT_NOTIF_ACITON = ["notify", {"set_tweak": "highlight", "value": False}] -DEFAULT_HIGHLIGHT_ACITON = [ +DEFAULT_NOTIF_ACTION = ["notify", {"set_tweak": "highlight", "value": False}] +DEFAULT_HIGHLIGHT_ACTION = [ "notify", {"set_tweak": "sound", "value": "default"}, {"set_tweak": "highlight"} ] def _serialize_action(actions, is_highlight): + """Custom serializer for actions. This allows us to "compress" common actions. + + We use the fact that most users have the same actions for notifs (and for + highlights). We replaces these default actions with the emtpy string. + """ if is_highlight: - if actions == DEFAULT_HIGHLIGHT_ACITON: - return "" + if actions == DEFAULT_HIGHLIGHT_ACTION: + return "" # We use empty string as the column is non-NULL else: - if actions == DEFAULT_NOTIF_ACITON: + if actions == DEFAULT_NOTIF_ACTION: return "" return json.dumps(actions) def _deserialize_action(actions, is_highlight): + """Custom deserializer for actions. This allows us to "compress" common actions + """ if actions: return json.loads(actions) if is_highlight: - return DEFAULT_HIGHLIGHT_ACITON + return DEFAULT_HIGHLIGHT_ACTION else: - return DEFAULT_NOTIF_ACITON + return DEFAULT_NOTIF_ACTION class EventPushActionsStore(SQLBaseStore): From 138e030cfe6e00956d79eed0702ff3f284692357 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Feb 2017 15:03:36 +0000 Subject: [PATCH 24/82] Comment --- synapse/storage/event_push_actions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index ea6722df6..808c9d22f 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -36,7 +36,10 @@ def _serialize_action(actions, is_highlight): """Custom serializer for actions. This allows us to "compress" common actions. We use the fact that most users have the same actions for notifs (and for - highlights). We replaces these default actions with the emtpy string. + highlights). + We store these default actions as the empty string rather than the full JSON. + Since the empty string isn't valid JSON there is no risk of this clashing with + any real JSON actions """ if is_highlight: if actions == DEFAULT_HIGHLIGHT_ACTION: From b4017539d49b16e2d91ca8e7d312c9da4dbf3ae7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Feb 2017 10:42:57 +0000 Subject: [PATCH 25/82] Make the pushers lang field column longer To accommodate things like zh-Hans-CN Fixes https://github.com/vector-im/riot-ios/issues/1031 --- synapse/storage/schema/delta/40/pushers.sql | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 synapse/storage/schema/delta/40/pushers.sql diff --git a/synapse/storage/schema/delta/40/pushers.sql b/synapse/storage/schema/delta/40/pushers.sql new file mode 100644 index 000000000..7e34927ef --- /dev/null +++ b/synapse/storage/schema/delta/40/pushers.sql @@ -0,0 +1,39 @@ +/* Copyright 2017 Vector Creations Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS pushers2 ( + id BIGINT PRIMARY KEY, + user_name TEXT NOT NULL, + access_token BIGINT DEFAULT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey TEXT NOT NULL, + ts BIGINT NOT NULL, + lang VARCHAR(16), + data TEXT, + last_stream_ordering INTEGER, + last_success BIGINT, + failing_since BIGINT, + UNIQUE (app_id, pushkey, user_name) +); + +INSERT INTO pushers2 SELECT * FROM PUSHERS; + +DROP TABLE PUSHERS; + +ALTER TABLE pushers2 RENAME TO pushers; From 4aa29508af2ac9ac07f91a33d5d682b4b5815a9c Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Feb 2017 10:51:49 +0000 Subject: [PATCH 26/82] Use TEXT rather than VARCHAR While we're changing anyway --- synapse/storage/schema/delta/40/pushers.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/storage/schema/delta/40/pushers.sql b/synapse/storage/schema/delta/40/pushers.sql index 7e34927ef..054a223f1 100644 --- a/synapse/storage/schema/delta/40/pushers.sql +++ b/synapse/storage/schema/delta/40/pushers.sql @@ -17,14 +17,14 @@ CREATE TABLE IF NOT EXISTS pushers2 ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, access_token BIGINT DEFAULT NULL, - profile_tag VARCHAR(32) NOT NULL, - kind VARCHAR(8) NOT NULL, - app_id VARCHAR(64) NOT NULL, - app_display_name VARCHAR(64) NOT NULL, - device_display_name VARCHAR(128) NOT NULL, + profile_tag TEXT NOT NULL, + kind TEXT NOT NULL, + app_id TEXT NOT NULL, + app_display_name TEXT NOT NULL, + device_display_name TEXT NOT NULL, pushkey TEXT NOT NULL, ts BIGINT NOT NULL, - lang VARCHAR(16), + lang TEXT, data TEXT, last_stream_ordering INTEGER, last_success BIGINT, From 5aae844e608554bdf5fd4487c729dd978b6eeffc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 17 Feb 2017 12:48:53 +0000 Subject: [PATCH 27/82] Add an example log_config file --- contrib/example_log_config.yaml | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 contrib/example_log_config.yaml diff --git a/contrib/example_log_config.yaml b/contrib/example_log_config.yaml new file mode 100644 index 000000000..c582af42d --- /dev/null +++ b/contrib/example_log_config.yaml @@ -0,0 +1,48 @@ +# Example log_config file for synapse. To enable, point `log_config` to it in +# `homeserver.yaml`, and restart synapse. +# +# This configuration will produce similar results to the defaults within +# synapse, but can be edited to give more flexibility. + +version: 1 + +formatters: + fmt: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + # example output to console + console: + class: logging.StreamHandler + filters: [context] + + # example output to file - to enable, edit 'root' config below. + file: + class: logging.handlers.RotatingFileHandler + formatter: fmt + filename: /var/log/synapse/homeserver.log + maxBytes: 100000000 + backupCount: 3 + filters: [context] + + +root: + level: INFO + handlers: [console] # to use file handler instead, switch to [file] + +loggers: + synapse: + level: INFO + + synapse.storage: + level: INFO + + # example of enabling debugging for a component: + # + # synapse.federation.transport.server: + # level: DEBUG \ No newline at end of file From 66eb0bd548afd87f08f7e6e994774fe29589babe Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 17 Feb 2017 12:55:36 +0000 Subject: [PATCH 28/82] Update example_log_config.yaml add trailing NL --- contrib/example_log_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/example_log_config.yaml b/contrib/example_log_config.yaml index c582af42d..7f7c8ba58 100644 --- a/contrib/example_log_config.yaml +++ b/contrib/example_log_config.yaml @@ -45,4 +45,4 @@ loggers: # example of enabling debugging for a component: # # synapse.federation.transport.server: - # level: DEBUG \ No newline at end of file + # level: DEBUG From 699be7d1be5238172677c98078c2a973ab28fb9d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 18 Feb 2017 14:41:31 +0000 Subject: [PATCH 29/82] Fix up notif rotation --- synapse/storage/event_push_actions.py | 36 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 808c9d22f..52d2bd3ff 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -168,19 +168,13 @@ class EventPushActionsStore(SQLBaseStore): row = txn.fetchone() notify_count = row[0] if row else 0 - summary_notif_count = self._simple_select_one_onecol_txn( - txn, - table="event_push_summary", - keyvalues={ - "user_id": user_id, - "room_id": room_id, - }, - retcol="notif_count", - allow_none=True, - ) - - if summary_notif_count: - notify_count += summary_notif_count + txn.execute(""" + SELECT notif_count FROM event_push_summary + WHERE room_id = ? AND user_id = ? AND stream_ordering > ? + """, (room_id, user_id, stream_ordering,)) + rows = txn.fetchall() + if rows: + notify_count += rows[0][0] # Now get the number of highlights sql = ( @@ -645,12 +639,20 @@ class EventPushActionsStore(SQLBaseStore): # We want to make sure that we only ever do this one at a time self.database_engine.lock_table(txn, "event_push_summary") + old_rotate_stream_ordering = self._simple_select_one_onecol_txn( + txn, + table="event_push_summary_stream_ordering", + keyvalues={}, + retcol="stream_ordering", + ) + # We don't to try and rotate millions of rows at once, so we cap the # maximum stream ordering we'll rotate before. txn.execute(""" SELECT stream_ordering FROM event_push_actions + WHERE stream_ordering > ? ORDER BY stream_ordering ASC LIMIT 1 OFFSET 50000 - """) + """, (old_rotate_stream_ordering,)) stream_row = txn.fetchone() if stream_row: offset_stream_ordering, = stream_row @@ -662,6 +664,8 @@ class EventPushActionsStore(SQLBaseStore): rotate_to_stream_ordering = self.stream_ordering_day_ago caught_up = True + logger.info("Rotating notifications up to: %s", rotate_to_stream_ordering) + self._rotate_notifs_before_txn(txn, rotate_to_stream_ordering) # We have caught up iff we were limited by `stream_ordering_day_ago` @@ -695,6 +699,8 @@ class EventPushActionsStore(SQLBaseStore): txn.execute(sql, (old_rotate_stream_ordering, rotate_to_stream_ordering,)) rows = txn.fetchall() + logger.info("Rotating notifications, handling %d rows", len(rows)) + # If the `old.user_id` above is NULL then we know there isn't already an # entry in the table, so we simply insert it. Otherwise we update the # existing table. @@ -726,6 +732,8 @@ class EventPushActionsStore(SQLBaseStore): (old_rotate_stream_ordering, rotate_to_stream_ordering,) ) + logger.info("Rotating notifications, deleted %s push actions", txn.rowcount) + txn.execute( "UPDATE event_push_summary_stream_ordering SET stream_ordering = ?", (rotate_to_stream_ordering,) From 7f026792e12b7ca0d24ca4c8ed9ed239bf8a99ac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 Feb 2017 14:54:50 +0000 Subject: [PATCH 30/82] Fix /context/ visibiltiy rules --- synapse/handlers/room.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 7e7671c9a..73bc73a45 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -375,12 +375,15 @@ class RoomContextHandler(BaseHandler): now_token = yield self.hs.get_event_sources().get_current_token() + users = yield self.store.get_users_in_room(room_id) + is_peeking = user.to_string() not in users + def filter_evts(events): return filter_events_for_client( self.store, user.to_string(), events, - is_peeking=is_guest + is_peeking=is_peeking ) event = yield self.store.get_event(event_id, get_prev_content=True, From 17673404fbb1ca7b84efe7ca12955754bf366b09 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 Feb 2017 15:02:01 +0000 Subject: [PATCH 31/82] Remove unused param --- synapse/handlers/room.py | 2 +- synapse/rest/client/v1/room.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 73bc73a45..99cb7db0d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -356,7 +356,7 @@ class RoomCreationHandler(BaseHandler): class RoomContextHandler(BaseHandler): @defer.inlineCallbacks - def get_event_context(self, user, room_id, event_id, limit, is_guest): + def get_event_context(self, user, room_id, event_id, limit): """Retrieves events, pagination tokens and state around a given event in a room. diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 728e3df0e..90242a6ba 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -505,7 +505,6 @@ class RoomEventContext(ClientV1RestServlet): room_id, event_id, limit, - requester.is_guest, ) if not results: From 0c4cf9372b3200325109dc2e4975197aa9b7b7ca Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 20 Feb 2017 16:43:29 +0000 Subject: [PATCH 32/82] Fix a race in transaction queue It was theoretically possible for a PDU to get queued and not sent for ages. On closer inspection I think there were bigger problems elsewhere, but we might as well fix this since it's easy. --- synapse/federation/transaction_queue.py | 30 +++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index bb3d9258a..90235ff09 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -303,18 +303,10 @@ class TransactionQueue(object): try: self.pending_transactions[destination] = 1 + # XXX: what's this for? yield run_on_reactor() while True: - pending_pdus = self.pending_pdus_by_dest.pop(destination, []) - pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_presence = self.pending_presence_by_dest.pop(destination, {}) - pending_failures = self.pending_failures_by_dest.pop(destination, []) - - pending_edus.extend( - self.pending_edus_keyed_by_dest.pop(destination, {}).values() - ) - limiter = yield get_retry_limiter( destination, self.clock, @@ -326,6 +318,24 @@ class TransactionQueue(object): yield self._get_new_device_messages(destination) ) + # BEGIN CRITICAL SECTION + # + # In order to avoid a race condition, we need to make sure that + # the following code (from popping the queues up to the point + # where we decide if we actually have any pending messages) is + # atomic - otherwise new PDUs or EDUs might arrive in the + # meantime, but not get sent because we hold the + # pending_transactions flag. + + pending_pdus = self.pending_pdus_by_dest.pop(destination, []) + pending_edus = self.pending_edus_by_dest.pop(destination, []) + pending_presence = self.pending_presence_by_dest.pop(destination, {}) + pending_failures = self.pending_failures_by_dest.pop(destination, []) + + pending_edus.extend( + self.pending_edus_keyed_by_dest.pop(destination, {}).values() + ) + pending_edus.extend(device_message_edus) if pending_presence: pending_edus.append( @@ -355,6 +365,8 @@ class TransactionQueue(object): ) return + # END CRITICAL SECTION + success = yield self._send_new_transaction( destination, pending_pdus, pending_edus, pending_failures, limiter=limiter, From b7442c3e2b333616093af76ab7848fa82b6490a9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 21 Feb 2017 13:59:25 +0000 Subject: [PATCH 33/82] Store looping call --- synapse/storage/event_push_actions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 52d2bd3ff..cde141e20 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -84,7 +84,9 @@ class EventPushActionsStore(SQLBaseStore): ) self._doing_notif_rotation = False - self._clock.looping_call(self._rotate_notifs, 30 * 60 * 1000) + self._rotate_notif_loop = self._clock.looping_call( + self._rotate_notifs, 30 * 60 * 1000 + ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): """ From 7455ba436a2903c8e6c9086c9ebe89f4c88af946 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Feb 2017 12:08:14 +0000 Subject: [PATCH 34/82] Ensure we pass positive ints to delay function --- synapse/push/emailpusher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 2eb325c7c..c7afd1111 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -218,7 +218,8 @@ class EmailPusher(object): ) def seconds_until(self, ts_msec): - return (ts_msec - self.clock.time_msec()) / 1000 + secs = (ts_msec - self.clock.time_msec()) / 1000 + return max(secs, 0) def get_room_throttle_ms(self, room_id): if room_id in self.throttle_params: From b2d20e94fa0a2efa84a084ddededb52d89af64c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Feb 2017 14:23:54 +0000 Subject: [PATCH 35/82] Remove lock from rotate notifs --- synapse/storage/event_push_actions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index cde141e20..14543b426 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -638,9 +638,6 @@ class EventPushActionsStore(SQLBaseStore): the archiving process has caught up or not. """ - # We want to make sure that we only ever do this one at a time - self.database_engine.lock_table(txn, "event_push_summary") - old_rotate_stream_ordering = self._simple_select_one_onecol_txn( txn, table="event_push_summary_stream_ordering", From 1a4f8022e6312856b0888639d224e84d505202eb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Feb 2017 11:15:31 +0000 Subject: [PATCH 36/82] Strip newlines from SQL queries --- synapse/storage/_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b0dc39119..557701d0c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -80,7 +80,13 @@ class LoggingTransaction(object): def executemany(self, sql, *args): self._do_execute(self.txn.executemany, sql, *args) + def _make_sql_one_line(self, sql): + "Strip newlines out of SQL so that the loggers in the DB are on one line" + return " ".join(l.strip() for l in sql.splitlines() if l.strip()) + def _do_execute(self, func, sql, *args): + sql = self._make_sql_one_line(sql) + # TODO(paul): Maybe use 'info' and 'debug' for values? sql_logger.debug("[SQL] {%s} %s", self.name, sql) From aea546148879f7e376c626346eb789308d089249 Mon Sep 17 00:00:00 2001 From: Jurek Date: Fri, 24 Feb 2017 22:42:38 +0100 Subject: [PATCH 37/82] Fix dynamic thumbnails aspect --- synapse/rest/media/v1/media_repository.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 3cbeca503..481ffee20 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -240,6 +240,9 @@ class MediaRepository(object): if t_method == "crop": t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) elif t_method == "scale": + t_width, t_height = thumbnailer.aspect(t_width, t_height) + t_width = min(m_width, t_width) + t_height = min(m_height, t_height) t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) else: t_len = None From c80439a320f00383d002aca966ff77979d614f4b Mon Sep 17 00:00:00 2001 From: Sean Enck Date: Mon, 27 Feb 2017 10:06:15 -0500 Subject: [PATCH 38/82] the aur package is no longer there, community package in arch does exist --- README.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 77e0b470a..a73c71c77 100644 --- a/README.rst +++ b/README.rst @@ -332,9 +332,8 @@ https://obs.infoserver.lv/project/monitor/matrix-synapse ArchLinux --------- -The quickest way to get up and running with ArchLinux is probably with Ivan -Shapovalov's AUR package from -https://aur.archlinux.org/packages/matrix-synapse/, which should pull in all +The quickest way to get up and running with ArchLinux is probably with the community package +https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in all the necessary dependencies. Alternatively, to install using pip a few changes may be needed as ArchLinux From f58dbb02a6c7a65abb4908f99c68369127e950a9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Feb 2017 16:22:12 +0000 Subject: [PATCH 39/82] Cache get_user_devices_from_cache --- synapse/storage/devices.py | 118 +++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 45 deletions(-) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index 8e1780036..d22db0a0b 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -19,6 +19,8 @@ from twisted.internet import defer from synapse.api.errors import StoreError from ._base import SQLBaseStore +from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks + logger = logging.getLogger(__name__) @@ -144,6 +146,7 @@ class DeviceStore(SQLBaseStore): defer.returnValue({d["device_id"]: d for d in devices}) + @cached(max_entries=10000) def get_device_list_last_stream_id_for_remote(self, user_id): """Get the last stream_id we got for a user. May be None if we haven't got any information for them. @@ -156,16 +159,36 @@ class DeviceStore(SQLBaseStore): allow_none=True, ) + @cachedList(cached_method_name="get_device_list_last_stream_id_for_remote", + list_name="user_ids", inlineCallbacks=True) + def get_device_list_last_stream_id_for_remotes(self, user_ids): + rows = yield self._simple_select_many_batch( + table="device_lists_remote_extremeties", + column="user_id", + iterable=user_ids, + retcols=("user_id", "stream_id",), + desc="get_user_devices_from_cache", + ) + + results = {user_id: None for user_id in user_ids} + results.update({ + row["user_id"]: row["stream_id"] for row in rows + }) + + defer.returnValue(results) + + @defer.inlineCallbacks def mark_remote_user_device_list_as_unsubscribed(self, user_id): """Mark that we no longer track device lists for remote user. """ - return self._simple_delete( + yield self._simple_delete( table="device_lists_remote_extremeties", keyvalues={ "user_id": user_id, }, desc="mark_remote_user_device_list_as_unsubscribed", ) + self.get_device_list_last_stream_id_for_remote.invalidate((user_id,)) def update_remote_device_list_cache_entry(self, user_id, device_id, content, stream_id): @@ -191,6 +214,12 @@ class DeviceStore(SQLBaseStore): } ) + txn.call_after(self._get_cached_user_device.invalidate, (user_id, device_id,)) + txn.call_after(self._get_cached_devices_for_user.invalidate, (user_id,)) + txn.call_after( + self.get_device_list_last_stream_id_for_remote.invalidate, (user_id,) + ) + self._simple_upsert_txn( txn, table="device_lists_remote_extremeties", @@ -234,6 +263,12 @@ class DeviceStore(SQLBaseStore): ] ) + txn.call_after(self._get_cached_devices_for_user.invalidate, (user_id,)) + txn.call_after(self._get_cached_user_device.invalidate_many, (user_id,)) + txn.call_after( + self.get_device_list_last_stream_id_for_remote.invalidate, (user_id,) + ) + self._simple_upsert_txn( txn, table="device_lists_remote_extremeties", @@ -320,6 +355,7 @@ class DeviceStore(SQLBaseStore): return (now_stream_id, results) + @defer.inlineCallbacks def get_user_devices_from_cache(self, query_list): """Get the devices (and keys if any) for remote users from the cache. @@ -332,27 +368,11 @@ class DeviceStore(SQLBaseStore): a set of user_ids and results_map is a mapping of user_id -> device_id -> device_info """ - return self.runInteraction( - "get_user_devices_from_cache", self._get_user_devices_from_cache_txn, - query_list, + user_ids = set(user_id for user_id, _ in query_list) + user_map = yield self.get_device_list_last_stream_id_for_remotes(list(user_ids)) + user_ids_in_cache = set( + user_id for user_id, stream_id in user_map.items() if stream_id ) - - def _get_user_devices_from_cache_txn(self, txn, query_list): - user_ids = {user_id for user_id, _ in query_list} - - user_ids_in_cache = set() - for user_id in user_ids: - stream_ids = self._simple_select_onecol_txn( - txn, - table="device_lists_remote_extremeties", - keyvalues={ - "user_id": user_id, - }, - retcol="stream_id", - ) - if stream_ids: - user_ids_in_cache.add(user_id) - user_ids_not_in_cache = user_ids - user_ids_in_cache results = {} @@ -361,32 +381,40 @@ class DeviceStore(SQLBaseStore): continue if device_id: - content = self._simple_select_one_onecol_txn( - txn, - table="device_lists_remote_cache", - keyvalues={ - "user_id": user_id, - "device_id": device_id, - }, - retcol="content", - ) - results.setdefault(user_id, {})[device_id] = json.loads(content) + device = yield self._get_cached_user_device(user_id, device_id) + results.setdefault(user_id, {})[device_id] = device else: - devices = self._simple_select_list_txn( - txn, - table="device_lists_remote_cache", - keyvalues={ - "user_id": user_id, - }, - retcols=("device_id", "content"), - ) - results[user_id] = { - device["device_id"]: json.loads(device["content"]) - for device in devices - } - user_ids_in_cache.discard(user_id) + results[user_id] = yield self._get_cached_devices_for_user(user_id) - return user_ids_not_in_cache, results + defer.returnValue((user_ids_not_in_cache, results)) + + @cachedInlineCallbacks(num_args=2, tree=True) + def _get_cached_user_device(self, user_id, device_id): + content = yield self._simple_select_one_onecol( + table="device_lists_remote_cache", + keyvalues={ + "user_id": user_id, + "device_id": device_id, + }, + retcol="content", + desc="_get_cached_user_device", + ) + defer.returnValue(json.loads(content)) + + @cachedInlineCallbacks() + def _get_cached_devices_for_user(self, user_id): + devices = yield self._simple_select_list( + table="device_lists_remote_cache", + keyvalues={ + "user_id": user_id, + }, + retcols=("device_id", "content"), + desc="_get_cached_devices_for_user", + ) + defer.returnValue({ + device["device_id"]: json.loads(device["content"]) + for device in devices + }) def get_devices_with_keys_by_user(self, user_id): """Get all devices (with any device keys) for a user From 49f4bc470993136ee4f5af7c34bee5ddc22768bd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Feb 2017 18:33:41 +0000 Subject: [PATCH 40/82] Don't fetch current state in common case Currently we fetch the list of current state events whenever we send something in a room. This is overkill for the common case of persisting a simple chain of non-state events, so lets handle that case specially. --- synapse/storage/events.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index c88f689d3..a4ce71cb2 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -311,6 +311,23 @@ class EventsStore(SQLBaseStore): new_forward_extremeties[room_id] = new_latest_event_ids + len_1 = ( + len(latest_event_ids) == 1 + and len(new_latest_event_ids) == 1 + ) + if len_1: + all_single_prev_not_state = any( + len(event.prev_events) == 1 + and not event.is_state() + for event, ctx in ev_ctx_rm + if not event.internal_metadata.is_outlier() + and not ctx.rejected + ) + # Don't bother calculating state if they're just + # a long chain of single ancestor non-state events. + if all_single_prev_not_state: + continue + state = yield self._calculate_state_delta( room_id, ev_ctx_rm, new_latest_event_ids ) From c0d6045776909a83181a4a925198c58e625ea4a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Feb 2017 18:45:24 +0000 Subject: [PATCH 41/82] It should be all --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a4ce71cb2..08807deba 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -316,7 +316,7 @@ class EventsStore(SQLBaseStore): and len(new_latest_event_ids) == 1 ) if len_1: - all_single_prev_not_state = any( + all_single_prev_not_state = all( len(event.prev_events) == 1 and not event.is_state() for event, ctx in ev_ctx_rm From a41dce8f8ae8bfb957a5830783e00437b4636c10 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Feb 2017 18:54:43 +0000 Subject: [PATCH 42/82] Remove needless check --- synapse/storage/events.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 08807deba..db01eb6d1 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -320,8 +320,6 @@ class EventsStore(SQLBaseStore): len(event.prev_events) == 1 and not event.is_state() for event, ctx in ev_ctx_rm - if not event.internal_metadata.is_outlier() - and not ctx.rejected ) # Don't bother calculating state if they're just # a long chain of single ancestor non-state events. From 64a2cef9bb2b0cf62b2a72f63c25c4781c8354e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Feb 2017 19:15:36 +0000 Subject: [PATCH 43/82] Pop rather than del from dict --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 996bfd0e2..06d453017 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1096,7 +1096,7 @@ class FederationHandler(BaseHandler): if prev_id != event.event_id: results[(event.type, event.state_key)] = prev_id else: - del results[(event.type, event.state_key)] + results.pop((event.type, event.state_key)) defer.returnValue(results.values()) else: From 9037787f0b655067eb28ae689745e60e7472e7b9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 28 Feb 2017 00:29:54 +0000 Subject: [PATCH 44/82] merge in right archlinux package, thanks to @saram-kon from https://github.com/matrix-org/synapse/pull/1956 --- contrib/systemd/synapse.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/systemd/synapse.service b/contrib/systemd/synapse.service index 967a4debf..92d94b9d5 100644 --- a/contrib/systemd/synapse.service +++ b/contrib/systemd/synapse.service @@ -1,5 +1,5 @@ # This assumes that Synapse has been installed as a system package -# (e.g. https://aur.archlinux.org/packages/matrix-synapse/ for ArchLinux) +# (e.g. https://www.archlinux.org/packages/community/any/matrix-synapse/ for ArchLinux) # rather than in a user home directory or similar under virtualenv. [Unit] From 848cf95ea088ea6bff209d8fdb7bdb809d7aa8fa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Feb 2017 10:01:19 +0000 Subject: [PATCH 45/82] Pop with default value to stop throwing --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 06d453017..ed0fa51e7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1096,7 +1096,7 @@ class FederationHandler(BaseHandler): if prev_id != event.event_id: results[(event.type, event.state_key)] = prev_id else: - results.pop((event.type, event.state_key)) + results.pop((event.type, event.state_key), None) defer.returnValue(results.values()) else: From 8a12b6f1eb16465bb2be816d3f92f07b844564d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Feb 2017 10:15:50 +0000 Subject: [PATCH 46/82] Fix up txn name --- synapse/storage/devices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index d22db0a0b..7b5903bf8 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -517,7 +517,7 @@ class DeviceStore(SQLBaseStore): WHERE stream_id > ? """ return self._execute( - "get_users_and_hosts_device_list", None, + "get_all_device_list_changes_for_remotes", None, sql, from_key, ) From e4919b93297031de041b55473984a2913aeb0c5d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Feb 2017 11:17:27 +0000 Subject: [PATCH 47/82] Add stream_id index to device_lists_outbound_pokes As this is used for replication streaming --- synapse/storage/prepare_database.py | 2 +- .../schema/delta/41/device_outbound_index.sql | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/41/device_outbound_index.sql diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index b357f22be..ed84db6b4 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 40 +SCHEMA_VERSION = 41 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/41/device_outbound_index.sql b/synapse/storage/schema/delta/41/device_outbound_index.sql new file mode 100644 index 000000000..62f0b9892 --- /dev/null +++ b/synapse/storage/schema/delta/41/device_outbound_index.sql @@ -0,0 +1,16 @@ +/* Copyright 2017 Vector Creations Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes(stream_id); From b84907bdbbc70ede585f65ee092fe0e2013f0d5d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Feb 2017 14:37:29 +0000 Subject: [PATCH 48/82] Intern table column names once --- synapse/storage/_base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 557701d0c..4410cd9e6 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -18,7 +18,6 @@ from synapse.api.errors import StoreError from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.caches.dictionary_cache import DictionaryCache from synapse.util.caches.descriptors import Cache -from synapse.util.caches import intern_dict from synapse.storage.engines import PostgresEngine import synapse.metrics @@ -356,9 +355,9 @@ class SQLBaseStore(object): Returns: A list of dicts where the key is the column header. """ - col_headers = list(column[0] for column in cursor.description) + col_headers = list(intern(column[0]) for column in cursor.description) results = list( - intern_dict(dict(zip(col_headers, row))) for row in cursor.fetchall() + dict(zip(col_headers, row)) for row in cursor.fetchall() ) return results From e933a2712d3987dac5a209a3a49a2a0664f62213 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Feb 2017 16:22:41 +0000 Subject: [PATCH 49/82] Don't log unknown cache warnings in workers --- synapse/replication/slave/storage/_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/replication/slave/storage/_base.py b/synapse/replication/slave/storage/_base.py index 18076e0f3..ab133db87 100644 --- a/synapse/replication/slave/storage/_base.py +++ b/synapse/replication/slave/storage/_base.py @@ -54,7 +54,9 @@ class BaseSlavedStore(SQLBaseStore): try: getattr(self, cache_func).invalidate(tuple(keys)) except AttributeError: - logger.info("Got unexpected cache_func: %r", cache_func) + # We probably haven't pulled in the cache in this worker, + # which is fine. + pass self._cache_id_gen.advance(int(stream["position"])) return defer.succeed(None) From f4e7545d8863e24a2327636181a49589d1e78ed8 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 28 Feb 2017 18:11:52 +0000 Subject: [PATCH 50/82] No longer need to request all the sub-components to be split when running sytest+dendron --- jenkins-dendron-postgres.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/jenkins-dendron-postgres.sh b/jenkins-dendron-postgres.sh index 55ff31fd1..37ae746f4 100755 --- a/jenkins-dendron-postgres.sh +++ b/jenkins-dendron-postgres.sh @@ -17,9 +17,3 @@ export SYNAPSE_CACHE_FACTOR=1 ./sytest/jenkins/install_and_run.sh \ --synapse-directory $WORKSPACE \ --dendron $WORKSPACE/dendron/bin/dendron \ - --pusher \ - --synchrotron \ - --federation-reader \ - --client-reader \ - --appservice \ - --federation-sender \ From 6b1ffa5f3de93618ae748391215e7139099b265f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 28 Feb 2017 18:14:13 +0000 Subject: [PATCH 51/82] Added also a control script to run via the crazy dendron+haproxy hybrid we're temporarily using --- jenkins-dendron-haproxy-postgres.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 jenkins-dendron-haproxy-postgres.sh diff --git a/jenkins-dendron-haproxy-postgres.sh b/jenkins-dendron-haproxy-postgres.sh new file mode 100755 index 000000000..d64b2d2c9 --- /dev/null +++ b/jenkins-dendron-haproxy-postgres.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -eux + +: ${WORKSPACE:="$(pwd)"} + +export WORKSPACE +export PYTHONDONTWRITEBYTECODE=yep +export SYNAPSE_CACHE_FACTOR=1 + +export HAPROXY_BIN=/home/haproxy/haproxy-1.6.11/haproxy + +./jenkins/prepare_synapse.sh +./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git +./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git +./dendron/jenkins/build_dendron.sh +./sytest/jenkins/prep_sytest_for_postgres.sh + +./sytest/jenkins/install_and_run.sh \ + --synapse-directory $WORKSPACE \ + --dendron $WORKSPACE/dendron/bin/dendron \ + --haproxy \ From 3365117151217f0ea96291aabce222f8b58bd850 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Mar 2017 10:21:30 +0000 Subject: [PATCH 52/82] Clobber old device list stream entries --- synapse/storage/devices.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index 7b5903bf8..f9ed18d2a 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -546,6 +546,16 @@ class DeviceStore(SQLBaseStore): host, stream_id, ) + # Delete older entries in the table, as we really only care about + # when the latest change happened. + txn.executemany( + """ + DELETE FROM device_lists_stream + WHERE user_id = ? AND device_id = ? AND stream_id < ? + """, + [(user_id, device_id, stream_id) for device_id in device_ids] + ) + self._simple_insert_many_txn( txn, table="device_lists_stream", From 36be39b8b39f4fc9b3a272faa306660b95145dad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Mar 2017 14:12:11 +0000 Subject: [PATCH 53/82] Fix device list update to not constantly resync --- synapse/handlers/device.py | 177 ++++++++++++++++++++++++++----------- 1 file changed, 123 insertions(+), 54 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index ca7137f31..23dab53df 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -34,10 +34,11 @@ class DeviceHandler(BaseHandler): self.state = hs.get_state_handler() self.federation_sender = hs.get_federation_sender() self.federation = hs.get_replication_layer() - self._remote_edue_linearizer = Linearizer(name="remote_device_list") + + self._edu_updater = DeviceListEduUpdater(hs, self) self.federation.register_edu_handler( - "m.device_list_update", self._incoming_device_list_update, + "m.device_list_update", self._edu_updater.incoming_device_list_update, ) self.federation.register_query_handler( "user_devices", self.on_federation_query_user_devices, @@ -299,58 +300,6 @@ class DeviceHandler(BaseHandler): # and those that actually still share a room with the user defer.returnValue(users_who_share_room & possibly_changed) - @measure_func("_incoming_device_list_update") - @defer.inlineCallbacks - def _incoming_device_list_update(self, origin, edu_content): - user_id = edu_content["user_id"] - device_id = edu_content["device_id"] - stream_id = edu_content["stream_id"] - prev_ids = edu_content.get("prev_id", []) - - if get_domain_from_id(user_id) != origin: - # TODO: Raise? - logger.warning("Got device list update edu for %r from %r", user_id, origin) - return - - rooms = yield self.store.get_rooms_for_user(user_id) - if not rooms: - # We don't share any rooms with this user. Ignore update, as we - # probably won't get any further updates. - return - - with (yield self._remote_edue_linearizer.queue(user_id)): - # If the prev id matches whats in our cache table, then we don't need - # to resync the users device list, otherwise we do. - resync = True - if len(prev_ids) == 1: - extremity = yield self.store.get_device_list_last_stream_id_for_remote( - user_id - ) - logger.info("Extrem: %r, prev_ids: %r", extremity, prev_ids) - if str(extremity) == str(prev_ids[0]): - resync = False - - if resync: - # Fetch all devices for the user. - result = yield self.federation.query_user_devices(origin, user_id) - stream_id = result["stream_id"] - devices = result["devices"] - yield self.store.update_remote_device_list_cache( - user_id, devices, stream_id, - ) - device_ids = [device["device_id"] for device in devices] - yield self.notify_device_update(user_id, device_ids) - else: - # Simply update the single device, since we know that is the only - # change (becuase of the single prev_id matching the current cache) - content = dict(edu_content) - for key in ("user_id", "device_id", "stream_id", "prev_ids"): - content.pop(key, None) - yield self.store.update_remote_device_list_cache_entry( - user_id, device_id, content, stream_id, - ) - yield self.notify_device_update(user_id, [device_id]) - @defer.inlineCallbacks def on_federation_query_user_devices(self, user_id): stream_id, devices = yield self.store.get_devices_with_keys_by_user(user_id) @@ -376,3 +325,123 @@ def _update_device_from_client_ips(device, client_ips): "last_seen_ts": ip.get("last_seen"), "last_seen_ip": ip.get("ip"), }) + + +class DeviceListEduUpdater(object): + "Handles incoming device list updates from federation and updates the DB" + + def __init__(self, hs, device_handler): + self.store = hs.get_datastore() + self.federation = hs.get_replication_layer() + self.clock = hs.get_clock() + self.device_handler = device_handler + + self._remote_edue_linearizer = Linearizer(name="remote_device_list") + + # user_id -> list of updates waiting to be handled. + self._pending_updates = {} + + # Recently seen stream ids. We don't bother keeping these in the DB, + # but they're useful to have them about to reduce the number of spurious + # resyncs. + self._seen_updates = {} + + @defer.inlineCallbacks + def incoming_device_list_update(self, origin, edu_content): + """Called on incoming device list update from federation. Responsible + for parsing the EDU and adding to pending updates list. + """ + + user_id = edu_content.pop("user_id") + device_id = edu_content.pop("device_id") + stream_id = str(edu_content.pop("stream_id")) # They may come as ints + prev_ids = edu_content.pop("prev_id", []) + prev_ids = [str(p) for p in prev_ids] # They may come as ints + + if get_domain_from_id(user_id) != origin: + # TODO: Raise? + logger.warning("Got device list update edu for %r from %r", user_id, origin) + return + + rooms = yield self.store.get_rooms_for_user(user_id) + if not rooms: + # We don't share any rooms with this user. Ignore update, as we + # probably won't get any further updates. + return + + self._pending_updates.setdefault(user_id, []).append( + (device_id, stream_id, prev_ids, edu_content) + ) + + yield self._handle_device_updates(user_id) + + @measure_func("_incoming_device_list_update") + @defer.inlineCallbacks + def _handle_device_updates(self, user_id): + "Actually handle pending updates." + + with (yield self._remote_edue_linearizer.queue(user_id)): + pending_updates = self._pending_updates.pop(user_id, []) + if not pending_updates: + # This can happen since we batch updates + return + + resync = yield self._need_to_do_resync(user_id, pending_updates) + + if resync: + # Fetch all devices for the user. + origin = get_domain_from_id(user_id) + result = yield self.federation.query_user_devices(origin, user_id) + stream_id = result["stream_id"] + devices = result["devices"] + yield self.store.update_remote_device_list_cache( + user_id, devices, stream_id, + ) + device_ids = [device["device_id"] for device in devices] + yield self.device_handler.notify_device_update(user_id, device_ids) + else: + # Simply update the single device, since we know that is the only + # change (becuase of the single prev_id matching the current cache) + for device_id, stream_id, prev_ids, content in pending_updates: + yield self.store.update_remote_device_list_cache_entry( + user_id, device_id, content, stream_id, + ) + + yield self.device_handler.notify_device_update( + user_id, [device_id for device_id, _, _, _ in pending_updates] + ) + + self._seen_updates.setdefault(user_id, set()).update( + [stream_id for _, stream_id, _, _ in pending_updates] + ) + + @defer.inlineCallbacks + def _need_to_do_resync(self, user_id, updates): + """Given a list of updates for a user figure out if we need to do a full + resync, or whether we have enough data that we can just apply the delta. + """ + seen_updates = self._seen_updates.get(user_id, set()) + + extremity = yield self.store.get_device_list_last_stream_id_for_remote( + user_id + ) + + stream_id_in_updates = set() # stream_ids in updates list + for _, stream_id, prev_ids, _ in updates: + if not prev_ids: + # We always do a resync if there are no previous IDs + defer.returnValue(True) + + for prev_id in prev_ids: + if prev_id == extremity: + continue + elif prev_id in seen_updates: + continue + elif prev_id in stream_id_in_updates: + continue + else: + defer.returnValue(True) + + stream_id_in_updates.add(stream_id) + + defer.returnValue(False) From d766343668e63a7572dcfe571d38ea3e143f3c1c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Mar 2017 15:56:30 +0000 Subject: [PATCH 54/82] Add index to device_lists_stream --- synapse/storage/deviceinbox.py | 8 ++++---- synapse/storage/devices.py | 7 +++++++ .../schema/delta/41/device_list_stream_idx.sql | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 synapse/storage/schema/delta/41/device_list_stream_idx.sql diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py index bde3b5cbb..1951de0ce 100644 --- a/synapse/storage/deviceinbox.py +++ b/synapse/storage/deviceinbox.py @@ -31,10 +31,10 @@ class DeviceInboxStore(BackgroundUpdateStore): super(DeviceInboxStore, self).__init__(hs) self.register_background_index_update( - "device_inbox_stream_index", - index_name="device_inbox_stream_id_user_id", - table="device_inbox", - columns=["stream_id", "user_id"], + "device_lists_stream_idx", + index_name="device_lists_stream_user_id", + table="device_lists_stream", + columns=["user_id", "device_id"], ) self.register_background_update_handler( diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index f9ed18d2a..ed659b700 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -33,6 +33,13 @@ class DeviceStore(SQLBaseStore): self._prune_old_outbound_device_pokes, 60 * 60 * 1000 ) + self.register_background_index_update( + "device_inbox_stream_index", + index_name="device_inbox_stream_id_user_id", + table="device_inbox", + columns=["stream_id", "user_id"], + ) + @defer.inlineCallbacks def store_device(self, user_id, device_id, initial_device_display_name): diff --git a/synapse/storage/schema/delta/41/device_list_stream_idx.sql b/synapse/storage/schema/delta/41/device_list_stream_idx.sql new file mode 100644 index 000000000..b7bee8b69 --- /dev/null +++ b/synapse/storage/schema/delta/41/device_list_stream_idx.sql @@ -0,0 +1,17 @@ +/* Copyright 2017 Vector Creations Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT into background_updates (update_name, progress_json) + VALUES ('device_lists_stream_idx', '{}'); From 856a18f7a8076860b59060912a431bb2b80bec82 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 1 Mar 2017 16:24:11 +0000 Subject: [PATCH 55/82] kick jenkins From ad882cd54d8349750f363e6127eca14a2e52b2b6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 1 Mar 2017 18:08:51 +0000 Subject: [PATCH 56/82] Just return the deferred straight off defer.returnValue doth not maketh a generator: it would need a yield to be a generator, and this doesn't need a yield. --- synapse/handlers/auth.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 448bc0b31..2ea1d17ca 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -309,13 +309,11 @@ class AuthHandler(BaseHandler): defer.returnValue(True) raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) - @defer.inlineCallbacks def _check_email_identity(self, authdict, _): - defer.returnValue(self._check_threepid('email', authdict)) + return self._check_threepid('email', authdict) - @defer.inlineCallbacks def _check_msisdn(self, authdict, _): - defer.returnValue(self._check_threepid('msisdn', authdict)) + return self._check_threepid('msisdn', authdict) @defer.inlineCallbacks def _check_dummy_auth(self, authdict, _): From da52d3af317f03cd95c2f3158bf95290d828f2aa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 3 Mar 2017 15:29:13 +0000 Subject: [PATCH 57/82] Fix up --- synapse/storage/deviceinbox.py | 8 ++++---- synapse/storage/devices.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py index 1951de0ce..bde3b5cbb 100644 --- a/synapse/storage/deviceinbox.py +++ b/synapse/storage/deviceinbox.py @@ -31,10 +31,10 @@ class DeviceInboxStore(BackgroundUpdateStore): super(DeviceInboxStore, self).__init__(hs) self.register_background_index_update( - "device_lists_stream_idx", - index_name="device_lists_stream_user_id", - table="device_lists_stream", - columns=["user_id", "device_id"], + "device_inbox_stream_index", + index_name="device_inbox_stream_id_user_id", + table="device_inbox", + columns=["stream_id", "user_id"], ) self.register_background_update_handler( diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index ed659b700..81c43d31f 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -34,10 +34,10 @@ class DeviceStore(SQLBaseStore): ) self.register_background_index_update( - "device_inbox_stream_index", - index_name="device_inbox_stream_id_user_id", - table="device_inbox", - columns=["stream_id", "user_id"], + "device_lists_stream_idx", + index_name="device_lists_stream_user_id", + table="device_lists_stream", + columns=["user_id", "device_id"], ) @defer.inlineCallbacks From 9834367eeaecd7f356cf7cda1e1b3eb79f8f2ea2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 3 Mar 2017 15:31:57 +0000 Subject: [PATCH 58/82] Spelling --- synapse/handlers/device.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 23dab53df..540b43879 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -336,7 +336,7 @@ class DeviceListEduUpdater(object): self.clock = hs.get_clock() self.device_handler = device_handler - self._remote_edue_linearizer = Linearizer(name="remote_device_list") + self._remote_edu_linearizer = Linearizer(name="remote_device_list") # user_id -> list of updates waiting to be handled. self._pending_updates = {} @@ -380,7 +380,7 @@ class DeviceListEduUpdater(object): def _handle_device_updates(self, user_id): "Actually handle pending updates." - with (yield self._remote_edue_linearizer.queue(user_id)): + with (yield self._remote_edu_linearizer.queue(user_id)): pending_updates = self._pending_updates.pop(user_id, []) if not pending_updates: # This can happen since we batch updates From f2581ee8b81e72c49b0c16ca41071c87292c0227 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 3 Mar 2017 16:02:53 +0000 Subject: [PATCH 59/82] Don't keep around old stream IDs forever --- synapse/handlers/device.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 540b43879..e859b3165 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -16,6 +16,7 @@ from synapse.api import errors from synapse.api.constants import EventTypes from synapse.util import stringutils from synapse.util.async import Linearizer +from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.metrics import measure_func from synapse.types import get_domain_from_id, RoomStreamToken from twisted.internet import defer @@ -344,7 +345,13 @@ class DeviceListEduUpdater(object): # Recently seen stream ids. We don't bother keeping these in the DB, # but they're useful to have them about to reduce the number of spurious # resyncs. - self._seen_updates = {} + self._seen_updates = ExpiringCache( + cache_name="device_update_edu", + clock=self.clock, + max_len=10000, + expiry_ms=30 * 60 * 1000, + iterable=True, + ) @defer.inlineCallbacks def incoming_device_list_update(self, origin, edu_content): @@ -412,7 +419,7 @@ class DeviceListEduUpdater(object): ) self._seen_updates.setdefault(user_id, set()).update( - [stream_id for _, stream_id, _, _ in pending_updates] + stream_id for _, stream_id, _, _ in pending_updates ) @defer.inlineCallbacks From b0effa2160b28081e6900bd9dbff265e6e990784 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 3 Mar 2017 18:34:39 +0000 Subject: [PATCH 60/82] Add msisdns as 3pids during registration and support binding them with the bind_msisdn param --- synapse/rest/client/v2_alpha/register.py | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 39f61d70b..768e75308 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -281,6 +281,7 @@ class RegisterRestServlet(RestServlet): ) # don't re-register the email address add_email = False + add_msisdn = False else: # NB: This may be from the auth handler and NOT from the POST if 'password' not in params: @@ -305,6 +306,7 @@ class RegisterRestServlet(RestServlet): ) add_email = True + add_msisdn = True return_dict = yield self._create_registration_details( registered_user_id, params @@ -317,6 +319,13 @@ class RegisterRestServlet(RestServlet): params.get("bind_email") ) + if add_msisdn and auth_result and LoginType.MSISDN in auth_result: + threepid = auth_result[LoginType.MSISDN] + yield self._register_msisdn_threepid( + registered_user_id, threepid, return_dict["access_token"], + params.get("bind_msisdn") + ) + defer.returnValue((200, return_dict)) def on_OPTIONS(self, _): @@ -426,6 +435,44 @@ class RegisterRestServlet(RestServlet): else: logger.info("bind_email not specified: not binding email") + @defer.inlineCallbacks + def _register_msisdn_threepid(self, user_id, threepid, token, bind_msisdn): + """Add aphone number as a 3pid identifier + + Also optionally binds msisdn to the given user_id on the identity server + + Args: + user_id (str): id of user + threepid (object): m.login.msisdn auth response + token (str): access_token for the user + bind_email (bool): true if the client requested the email to be + bound at the identity server + Returns: + defer.Deferred: + """ + reqd = ('medium', 'address', 'validated_at') + if any(x not in threepid for x in reqd): + logger.info("Can't add incomplete 3pid") + defer.returnValue() + + yield self.auth_handler.add_threepid( + user_id, + threepid['medium'], + threepid['address'], + threepid['validated_at'], + ) + + if bind_msisdn: + logger.info("bind_msisdn specified: binding") + logger.debug("Binding msisdn %s to %s" % ( + threepid, user_id + )) + yield self.identity_handler.bind_threepid( + threepid['threepid_creds'], user_id + ) + else: + logger.info("bind_msisdn not specified: not binding msisdn") + @defer.inlineCallbacks def _create_registration_details(self, user_id, params): """Complete registration of newly-registered user From ac5491f56308405890530ae09ac6ffcf93ad48b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 6 Mar 2017 11:10:10 +0000 Subject: [PATCH 61/82] Select distinct devices from DB Otherwise we might pull out tonnes of duplicate user_ids and this can make synapse sad. --- synapse/storage/devices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index 81c43d31f..bd56ba251 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -508,7 +508,7 @@ class DeviceStore(SQLBaseStore): defer.returnValue(set(changed)) sql = """ - SELECT user_id FROM device_lists_stream WHERE stream_id > ? + SELECT DISTINCT user_id FROM device_lists_stream WHERE stream_id > ? """ rows = yield self._execute("get_user_whose_devices_changed", None, sql, from_key) defer.returnValue(set(row[0] for row in rows)) From 00466e2feb12802262835458aa2bf78897c5ae63 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 7 Mar 2017 16:37:23 +0000 Subject: [PATCH 62/82] Support new login format https://docs.google.com/document/d/1-6ZSSW5YvCGhVFDyD2QExAUAdpCWjccvJT5xiyTTG2Y/edit# --- synapse/rest/client/v1/login.py | 97 +++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 72057f1b0..8de1a0225 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -25,6 +25,7 @@ from .base import ClientV1RestServlet, client_path_patterns import simplejson as json import urllib import urlparse +import phonenumbers import logging from saml2 import BINDING_HTTP_POST @@ -37,6 +38,58 @@ import xml.etree.ElementTree as ET logger = logging.getLogger(__name__) +def login_submission_legacy_convert(submission): + """ + If the input login submission is an old style object + (ie. with top-level user / medium / address) convert it + to a typed object. + Returns: Typed-object style login identifier + """ + if "user" in submission: + submission["identifier"] = { + "type": "m.id.user", + "user": submission["user"], + } + del submission["user"] + + if "medium" in submission and "address" in submission: + submission["identifier"] = { + "type": "m.id.thirdparty", + "medium": submission["medium"], + "address": submission["address"], + } + del submission["medium"] + del submission["address"] + + return submission + + +def login_id_thirdparty_from_phone(identifier): + """ + Convert a phone login identifier type to a generic threepid identifier + Args: + identifier: Login identifier dict of type 'm.id.phone' + + Returns: Login identifier dict of type 'm.id.threepid' + """ + if "country" not in identifier or "number" not in identifier: + raise SynapseError(400, "Invalid phone-type identifier") + phoneNumber = None + try: + phoneNumber = phonenumbers.parse(identifier["number"], identifier["country"]) + except phonenumbers.NumberParseException: + raise SynapseError(400, "Unable to parse phone number") + msisdn = phonenumbers.format_number( + phoneNumber, phonenumbers.PhoneNumberFormat.E164 + )[1:] + + return { + "type": "m.id.thirdparty", + "medium": "msisdn", + "address": msisdn, + } + + class LoginRestServlet(ClientV1RestServlet): PATTERNS = client_path_patterns("/login$") PASS_TYPE = "m.login.password" @@ -117,20 +170,52 @@ class LoginRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def do_password_login(self, login_submission): - if 'medium' in login_submission and 'address' in login_submission: - address = login_submission['address'] - if login_submission['medium'] == 'email': + if "password" not in login_submission: + raise SynapseError(400, "Missing parameter: password") + + login_submission = login_submission_legacy_convert(login_submission) + + if "identifier" not in login_submission: + raise SynapseError(400, "Missing param: identifier") + + identifier = login_submission["identifier"] + if "type" not in identifier: + raise SynapseError(400, "Login identifier has no type") + + # convert phone type identifiers to geberic threepids + if identifier["type"] == "m.id.phone": + identifier = login_id_thirdparty_from_phone(identifier) + + # convert threepid identifiers to user IDs + if identifier["type"] == "m.id.thirdparty": + if not 'medium' in identifier or not 'address' in identifier: + raise SynapseError(400, "Invalid thirdparty identifier") + + address = identifier['address'] + if identifier['medium'] == 'email': # For emails, transform the address to lowercase. # We store all email addreses as lowercase in the DB. # (See add_threepid in synapse/handlers/auth.py) address = address.lower() user_id = yield self.hs.get_datastore().get_user_id_by_threepid( - login_submission['medium'], address + identifier['medium'], address ) if not user_id: raise LoginError(403, "", errcode=Codes.FORBIDDEN) - else: - user_id = login_submission['user'] + + identifier = { + "type": "m.id.user", + "user": user_id, + } + + # by this point, the identifier should be an m.id.user: if it's anything + # else, we haven't understood it. + if identifier["type"] != "m.id.user": + raise SynapseError(400, "Unknown login identifier type") + if "user" not in identifier: + raise SynapseError(400, "User identifier is missing 'user' key") + + user_id = identifier["user"] if not user_id.startswith('@'): user_id = UserID.create( From 402a7bf63d0b8fc715ae6659c3b451e1bd44b0f2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 09:33:40 +0000 Subject: [PATCH 63/82] Fix pep8 --- synapse/handlers/auth.py | 2 +- synapse/rest/client/v1/login.py | 6 +++--- synapse/rest/client/v2_alpha/account.py | 4 ++-- synapse/rest/client/v2_alpha/register.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 2ea1d17ca..620591b16 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -344,7 +344,7 @@ class AuthHandler(BaseHandler): medium, threepid['medium'], ), errcode=Codes.UNAUTHORIZED - ) + ) threepid['threepid_creds'] = authdict['threepid_creds'] diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 8de1a0225..337ba5c02 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -80,7 +80,7 @@ def login_id_thirdparty_from_phone(identifier): except phonenumbers.NumberParseException: raise SynapseError(400, "Unable to parse phone number") msisdn = phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 + phoneNumber, phonenumbers.PhoneNumberFormat.E164 )[1:] return { @@ -188,7 +188,7 @@ class LoginRestServlet(ClientV1RestServlet): # convert threepid identifiers to user IDs if identifier["type"] == "m.id.thirdparty": - if not 'medium' in identifier or not 'address' in identifier: + if 'medium' not in identifier or 'address' not in identifier: raise SynapseError(400, "Invalid thirdparty identifier") address = identifier['address'] @@ -198,7 +198,7 @@ class LoginRestServlet(ClientV1RestServlet): # (See add_threepid in synapse/handlers/auth.py) address = address.lower() user_id = yield self.hs.get_datastore().get_user_id_by_threepid( - identifier['medium'], address + identifier['medium'], address ) if not user_id: raise LoginError(403, "", errcode=Codes.FORBIDDEN) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index cf80f5ca2..06bb3617e 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -93,7 +93,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): except phonenumbers.NumberParseException: raise SynapseError(400, "Unable to parse phone number") msisdn = phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 + phoneNumber, phonenumbers.PhoneNumberFormat.E164 )[1:] existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( @@ -279,7 +279,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): except phonenumbers.NumberParseException: raise SynapseError(400, "Unable to parse phone number") msisdn = phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 + phoneNumber, phonenumbers.PhoneNumberFormat.E164 )[1:] existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 768e75308..ad3c70814 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -116,7 +116,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): except phonenumbers.NumberParseException: raise SynapseError(400, "Unable to parse phone number") msisdn = phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 + phoneNumber, phonenumbers.PhoneNumberFormat.E164 )[1:] existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( From 88df6c0c9a7ce12e7875680131cac421da87ad60 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 11:03:39 +0000 Subject: [PATCH 64/82] Factor out msisdn canonicalisation Plus a couple of other minor fixes --- synapse/handlers/auth.py | 2 +- synapse/rest/client/v1/login.py | 18 +++++------------- synapse/rest/client/v2_alpha/account.py | 21 +++------------------ synapse/rest/client/v2_alpha/register.py | 11 ++--------- 4 files changed, 11 insertions(+), 41 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 620591b16..b273a4bee 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -321,7 +321,7 @@ class AuthHandler(BaseHandler): defer.returnValue(True) @defer.inlineCallbacks - def _check_threepid(self, medium, authdict, ): + def _check_threepid(self, medium, authdict): yield run_on_reactor() if 'threepid_creds' not in authdict: diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 337ba5c02..77a0f00c5 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -19,13 +19,13 @@ from synapse.api.errors import SynapseError, LoginError, Codes from synapse.types import UserID from synapse.http.server import finish_request from synapse.http.servlet import parse_json_object_from_request +from synapse.util.msisdn import phone_number_to_msisdn from .base import ClientV1RestServlet, client_path_patterns import simplejson as json import urllib import urlparse -import phonenumbers import logging from saml2 import BINDING_HTTP_POST @@ -61,8 +61,6 @@ def login_submission_legacy_convert(submission): del submission["medium"] del submission["address"] - return submission - def login_id_thirdparty_from_phone(identifier): """ @@ -74,14 +72,8 @@ def login_id_thirdparty_from_phone(identifier): """ if "country" not in identifier or "number" not in identifier: raise SynapseError(400, "Invalid phone-type identifier") - phoneNumber = None - try: - phoneNumber = phonenumbers.parse(identifier["number"], identifier["country"]) - except phonenumbers.NumberParseException: - raise SynapseError(400, "Unable to parse phone number") - msisdn = phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 - )[1:] + + msisdn = phone_number_to_msisdn(identifier["country"], identifier["number"]) return { "type": "m.id.thirdparty", @@ -173,7 +165,7 @@ class LoginRestServlet(ClientV1RestServlet): if "password" not in login_submission: raise SynapseError(400, "Missing parameter: password") - login_submission = login_submission_legacy_convert(login_submission) + login_submission_legacy_convert(login_submission) if "identifier" not in login_submission: raise SynapseError(400, "Missing param: identifier") @@ -182,7 +174,7 @@ class LoginRestServlet(ClientV1RestServlet): if "type" not in identifier: raise SynapseError(400, "Login identifier has no type") - # convert phone type identifiers to geberic threepids + # convert phone type identifiers to generic threepids if identifier["type"] == "m.id.phone": identifier = login_id_thirdparty_from_phone(identifier) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 06bb3617e..199af60fc 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -20,13 +20,12 @@ from synapse.api.constants import LoginType from synapse.api.errors import LoginError, SynapseError, Codes from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.util.async import run_on_reactor +from synapse.util.msisdn import phone_number_to_msisdn from ._base import client_v2_patterns import logging -import phonenumbers - logger = logging.getLogger(__name__) @@ -87,14 +86,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): if absent: raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - phoneNumber = None - try: - phoneNumber = phonenumbers.parse(body['phone_number'], body['country']) - except phonenumbers.NumberParseException: - raise SynapseError(400, "Unable to parse phone number") - msisdn = phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 - )[1:] + msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'msisdn', msisdn @@ -273,14 +265,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): if absent: raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - phoneNumber = None - try: - phoneNumber = phonenumbers.parse(body['phone_number'], body['country']) - except phonenumbers.NumberParseException: - raise SynapseError(400, "Unable to parse phone number") - msisdn = phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 - )[1:] + msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'msisdn', msisdn diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index ad3c70814..95f9944c4 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -21,12 +21,12 @@ from synapse.api.auth import get_access_token_from_request, has_access_token from synapse.api.constants import LoginType from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.util.msisdn import phone_number_to_msisdn from ._base import client_v2_patterns import logging import hmac -import phonenumbers from hashlib import sha1 from synapse.util.async import run_on_reactor @@ -110,14 +110,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): if len(absent) > 0: raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - phoneNumber = None - try: - phoneNumber = phonenumbers.parse(body['phone_number'], body['country']) - except phonenumbers.NumberParseException: - raise SynapseError(400, "Unable to parse phone number") - msisdn = phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 - )[1:] + msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'msisdn', msisdn From 2e27339add58f71485af873cc70284d0bcc2a6bb Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 11:37:34 +0000 Subject: [PATCH 65/82] Refector out assert_params_in_request and replace requestEmailToken where we meant requestMsisdnToken --- synapse/http/servlet.py | 10 +++++++++ synapse/rest/client/v2_alpha/account.py | 28 ++++++++---------------- synapse/rest/client/v2_alpha/register.py | 26 +++++++--------------- 3 files changed, 27 insertions(+), 37 deletions(-) diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 8c22d6f00..9a4c36ad5 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -192,6 +192,16 @@ def parse_json_object_from_request(request): return content +def assert_params_in_request(body, required): + absent = [] + for k in required: + if k not in body: + absent.append(k) + + if len(absent) > 0: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + + class RestServlet(object): """ A Synapse REST Servlet. diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 199af60fc..9a5e1d871 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -18,7 +18,9 @@ from twisted.internet import defer from synapse.api.constants import LoginType from synapse.api.errors import LoginError, SynapseError, Codes -from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.http.servlet import ( + RestServlet, parse_json_object_from_request, assert_params_in_request +) from synapse.util.async import run_on_reactor from synapse.util.msisdn import phone_number_to_msisdn @@ -42,14 +44,9 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): def on_POST(self, request): body = parse_json_object_from_request(request) - required = ['id_server', 'client_secret', 'email', 'send_attempt'] - absent = [] - for k in required: - if k not in body: - absent.append(k) - - if absent: - raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + assert_params_in_request(body, [ + 'id_server', 'client_secret', 'email', 'send_attempt' + ]) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] @@ -74,17 +71,10 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): def on_POST(self, request): body = parse_json_object_from_request(request) - required = [ + assert_params_in_request(body,[ 'id_server', 'client_secret', 'country', 'phone_number', 'send_attempt', - ] - absent = [] - for k in required: - if k not in body: - absent.append(k) - - if absent: - raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + ]) msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) @@ -95,7 +85,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): if existingUid is None: raise SynapseError(400, "MSISDN not found", Codes.THREEPID_NOT_FOUND) - ret = yield self.identity_handler.requestEmailToken(**body) + ret = yield self.identity_handler.requestMsisdnToken(**body) defer.returnValue((200, ret)) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 95f9944c4..e97b9a32e 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -20,7 +20,9 @@ import synapse from synapse.api.auth import get_access_token_from_request, has_access_token from synapse.api.constants import LoginType from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError -from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.http.servlet import ( + RestServlet, parse_json_object_from_request, assert_params_in_request +) from synapse.util.msisdn import phone_number_to_msisdn from ._base import client_v2_patterns @@ -61,14 +63,9 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): def on_POST(self, request): body = parse_json_object_from_request(request) - required = ['id_server', 'client_secret', 'email', 'send_attempt'] - absent = [] - for k in required: - if k not in body: - absent.append(k) - - if len(absent) > 0: - raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + assert_params_in_request(body, [ + 'id_server', 'client_secret', 'email', 'send_attempt' + ]) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] @@ -97,18 +94,11 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): def on_POST(self, request): body = parse_json_object_from_request(request) - required = [ + assert_params_in_request(body, [ 'id_server', 'client_secret', 'country', 'phone_number', 'send_attempt', - ] - absent = [] - for k in required: - if k not in body: - absent.append(k) - - if len(absent) > 0: - raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + ]) msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) From 82c5e7de2540a64c7a664f864983409df59d5db9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 11:42:44 +0000 Subject: [PATCH 66/82] Typos --- synapse/rest/client/v2_alpha/register.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index e97b9a32e..4e872e1a8 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -262,7 +262,7 @@ class RegisterRestServlet(RestServlet): "Already registered user ID %r for this session", registered_user_id ) - # don't re-register the email address + # don't re-register the threepids add_email = False add_msisdn = False else: @@ -420,7 +420,7 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def _register_msisdn_threepid(self, user_id, threepid, token, bind_msisdn): - """Add aphone number as a 3pid identifier + """Add a phone number as a 3pid identifier Also optionally binds msisdn to the given user_id on the identity server From 0e0aee25c469fe3ba95d7410da670bc3fd73b510 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 11:46:22 +0000 Subject: [PATCH 67/82] Fix log line --- synapse/handlers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index b273a4bee..e7a1bb724 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -331,7 +331,7 @@ class AuthHandler(BaseHandler): identity_handler = self.hs.get_handlers().identity_handler - logger.info("Getting validated threepid. threepidcreds: %r" % (threepid_creds,)) + logger.info("Getting validated threepid. threepidcreds: %r", (threepid_creds,)) threepid = yield identity_handler.threepid_from_creds(threepid_creds) if not threepid: From 65d43f3ca5965acc45b605e7867d877c0166ee96 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 11:48:43 +0000 Subject: [PATCH 68/82] Minor fixes from PR feedback --- synapse/rest/client/v1/login.py | 2 +- synapse/rest/client/v2_alpha/register.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 77a0f00c5..a705fc838 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -66,7 +66,7 @@ def login_id_thirdparty_from_phone(identifier): """ Convert a phone login identifier type to a generic threepid identifier Args: - identifier: Login identifier dict of type 'm.id.phone' + identifier(dict): Login identifier dict of type 'm.id.phone' Returns: Login identifier dict of type 'm.id.threepid' """ diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 4e872e1a8..98b277e07 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -371,7 +371,7 @@ class RegisterRestServlet(RestServlet): reqd = ('medium', 'address', 'validated_at') if any(x not in threepid for x in reqd): logger.info("Can't add incomplete 3pid") - defer.returnValue() + return yield self.auth_handler.add_threepid( user_id, @@ -447,9 +447,7 @@ class RegisterRestServlet(RestServlet): if bind_msisdn: logger.info("bind_msisdn specified: binding") - logger.debug("Binding msisdn %s to %s" % ( - threepid, user_id - )) + logger.debug("Binding msisdn %s to %s", threepid, user_id) yield self.identity_handler.bind_threepid( threepid['threepid_creds'], user_id ) From 85bb322333f383768ceab1817f294fdb217f9b05 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 11:51:25 +0000 Subject: [PATCH 69/82] Pull out datastore in initialiser --- synapse/rest/client/v2_alpha/account.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 9a5e1d871..71723ef3d 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -65,6 +65,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): def __init__(self, hs): super(MsisdnPasswordRequestTokenRestServlet, self).__init__() self.hs = hs + self.datastore = self.hs.get_datastore() self.identity_handler = hs.get_handlers().identity_handler @defer.inlineCallbacks @@ -78,7 +79,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( + existingUid = yield self.datastore.get_user_id_by_threepid( 'msisdn', msisdn ) @@ -97,6 +98,7 @@ class PasswordRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() + self.datastore = self.hs.get_datastore() @defer.inlineCallbacks def on_POST(self, request): @@ -132,7 +134,7 @@ class PasswordRestServlet(RestServlet): # (See add_threepid in synapse/handlers/auth.py) threepid['address'] = threepid['address'].lower() # if using email, we must know about the email they're authing with! - threepid_user_id = yield self.hs.get_datastore().get_user_id_by_threepid( + threepid_user_id = yield self.datastore.get_user_id_by_threepid( threepid['medium'], threepid['address'] ) if not threepid_user_id: @@ -206,6 +208,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): self.hs = hs super(EmailThreepidRequestTokenRestServlet, self).__init__() self.identity_handler = hs.get_handlers().identity_handler + self.datastore = self.hs.get_datastore() @defer.inlineCallbacks def on_POST(self, request): @@ -220,7 +223,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): if absent: raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( + existingUid = yield self.datastore.get_user_id_by_threepid( 'email', body['email'] ) @@ -238,6 +241,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): self.hs = hs super(MsisdnThreepidRequestTokenRestServlet, self).__init__() self.identity_handler = hs.get_handlers().identity_handler + self.datastore = self.hs.get_datastore() @defer.inlineCallbacks def on_POST(self, request): @@ -257,7 +261,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( + existingUid = yield self.datastore.get_user_id_by_threepid( 'msisdn', msisdn ) @@ -277,6 +281,7 @@ class ThreepidRestServlet(RestServlet): self.identity_handler = hs.get_handlers().identity_handler self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() + self.datastore = self.hs.get_datastore() @defer.inlineCallbacks def on_GET(self, request): @@ -284,7 +289,7 @@ class ThreepidRestServlet(RestServlet): requester = yield self.auth.get_user_by_req(request) - threepids = yield self.hs.get_datastore().user_get_threepids( + threepids = yield self.datastore.user_get_threepids( requester.user.to_string() ) From a9e2b9ec16e15267d812b0148a014dde6e3db64c Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 11:53:36 +0000 Subject: [PATCH 70/82] Add msisdn util file --- synapse/util/msisdn.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 synapse/util/msisdn.py diff --git a/synapse/util/msisdn.py b/synapse/util/msisdn.py new file mode 100644 index 000000000..77ef0aa19 --- /dev/null +++ b/synapse/util/msisdn.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import phonenumbers +from synapse.api.errors import SynapseError + +def phone_number_to_msisdn(country, number): + try: + phoneNumber = phonenumbers.parse(number, country) + except phonenumbers.NumberParseException: + raise SynapseError(400, "Unable to parse phone number") + return phonenumbers.format_number( + phoneNumber, phonenumbers.PhoneNumberFormat.E164 + )[1:] From 1c99934b280485f564b357de204785d55cf2ca62 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 11:58:20 +0000 Subject: [PATCH 71/82] pep8 --- synapse/rest/client/v2_alpha/account.py | 2 +- synapse/util/msisdn.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 71723ef3d..aac76edf1 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -72,7 +72,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): def on_POST(self, request): body = parse_json_object_from_request(request) - assert_params_in_request(body,[ + assert_params_in_request(body, [ 'id_server', 'client_secret', 'country', 'phone_number', 'send_attempt', ]) diff --git a/synapse/util/msisdn.py b/synapse/util/msisdn.py index 77ef0aa19..6905c5ce7 100644 --- a/synapse/util/msisdn.py +++ b/synapse/util/msisdn.py @@ -17,6 +17,7 @@ import phonenumbers from synapse.api.errors import SynapseError + def phone_number_to_msisdn(country, number): try: phoneNumber = phonenumbers.parse(number, country) From d4d3629aaf930d2febfc98f8ef1ef6bd315b8a4e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 17:01:26 +0000 Subject: [PATCH 72/82] Better error message --- synapse/rest/client/v2_alpha/register.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 98b277e07..2c184b731 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -107,7 +107,9 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): ) if existingUid is not None: - raise SynapseError(400, "MSISDN is already in use", Codes.THREEPID_IN_USE) + raise SynapseError( + 400, "Phone number is already in use", Codes.THREEPID_IN_USE + ) ret = yield self.identity_handler.requestMsisdnToken(**body) defer.returnValue((200, ret)) From 6ad71cc29d8e0f8f5cc7976a84fa0b953edf0f82 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 8 Mar 2017 18:00:44 +0000 Subject: [PATCH 73/82] Remove spurious SQL logging (#1972) looks like the upsert function was accidentally sending sql logging to the general logger. We already log the sql in `txn.execute`. --- synapse/storage/_base.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4410cd9e6..a7a8ec9b7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -488,10 +488,6 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k,) for k in keyvalues) ) sqlargs = values.values() + keyvalues.values() - logger.debug( - "[SQL] %s Args=%s", - sql, sqlargs, - ) txn.execute(sql, sqlargs) if txn.rowcount == 0: @@ -506,10 +502,6 @@ class SQLBaseStore(object): ", ".join(k for k in allvalues), ", ".join("?" for _ in allvalues) ) - logger.debug( - "[SQL] %s Args=%s", - sql, keyvalues.values(), - ) txn.execute(sql, allvalues.values()) return True From 727124a76250ece5f1014b4099d5efc7b43f2aaf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 19:00:23 +0000 Subject: [PATCH 74/82] Not any more, it doesn't --- synapse/rest/client/v1/login.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index a705fc838..c4bbb7027 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -43,7 +43,6 @@ def login_submission_legacy_convert(submission): If the input login submission is an old style object (ie. with top-level user / medium / address) convert it to a typed object. - Returns: Typed-object style login identifier """ if "user" in submission: submission["identifier"] = { From 3edc57296dbefa4a453ac11031614d827e32edf3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 19:00:51 +0000 Subject: [PATCH 75/82] Incorrectly copied copyright This file post-dates OM --- synapse/util/msisdn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/util/msisdn.py b/synapse/util/msisdn.py index 6905c5ce7..d320e411f 100644 --- a/synapse/util/msisdn.py +++ b/synapse/util/msisdn.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# Copyright 2015, 2016 OpenMarket Ltd # Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); From 9d0d40fc15e4bc79eee46242ae27980073cd528b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 19:05:29 +0000 Subject: [PATCH 76/82] Docs --- synapse/util/msisdn.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/synapse/util/msisdn.py b/synapse/util/msisdn.py index d320e411f..607161e7f 100644 --- a/synapse/util/msisdn.py +++ b/synapse/util/msisdn.py @@ -18,6 +18,19 @@ from synapse.api.errors import SynapseError def phone_number_to_msisdn(country, number): + """ + Takes an ISO-3166-1 2 letter country code and phone number and + returns an msisdn representing the canonical version of that + phone number. + Args: + country (str): ISO-3166-1 2 letter country code + number (str): Phone number in a national or international format + + Returns: + (str) The canonical form of the phone number, as an msisdn + Raises: + SynapseError if the number could not be parsed. + """ try: phoneNumber = phonenumbers.parse(number, country) except phonenumbers.NumberParseException: From ece7e00048e6990434eda33b06b598f88237e0c4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Mar 2017 19:07:18 +0000 Subject: [PATCH 77/82] Comment when our 3pids would be incomplete --- synapse/rest/client/v2_alpha/register.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 2c184b731..7448c1346 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -372,6 +372,7 @@ class RegisterRestServlet(RestServlet): """ reqd = ('medium', 'address', 'validated_at') if any(x not in threepid for x in reqd): + # This will only happen if the ID server returns a malformed response logger.info("Can't add incomplete 3pid") return @@ -437,6 +438,7 @@ class RegisterRestServlet(RestServlet): """ reqd = ('medium', 'address', 'validated_at') if any(x not in threepid for x in reqd): + # This will only happen if the ID server returns a malformed response logger.info("Can't add incomplete 3pid") defer.returnValue() From 45d173a59a06c06e0c8414c465851492ecf0081a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 9 Mar 2017 13:15:12 +0000 Subject: [PATCH 78/82] Fix docstring --- synapse/federation/federation_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index b5bcfd705..5dcd4eecc 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -206,8 +206,7 @@ class FederationClient(FederationBase): Args: destinations (list): Which home servers to query - pdu_origin (str): The home server that originally sent the pdu. - event_id (str) + event_id (str): event to fetch outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if it's from an arbitary point in the context as opposed to part of the current block of PDUs. Defaults to `False` From 3545e17f43a42273c60f8607ab9e9d56ff1946cd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Mar 2017 10:30:38 +0000 Subject: [PATCH 79/82] Add setdefault key to ExpiringCache --- synapse/util/caches/expiringcache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 2987c38a2..cbdde34a5 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -100,6 +100,13 @@ class ExpiringCache(object): except KeyError: return default + def setdefault(self, key, value): + try: + return self[key] + except KeyError: + self[key] = value + return value + def _prune_cache(self): if not self._expiry_ms: # zero expiry time means don't expire. This should never get called From 64d62e41b86b201e03c752b5f0dcb2ef5811b9f9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Mar 2017 14:50:40 +0000 Subject: [PATCH 80/82] Noop repated delete device inbox calls from /sync --- synapse/handlers/sync.py | 6 +++--- synapse/storage/deviceinbox.py | 36 +++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index d7dcd1ce5..5572cb883 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -609,14 +609,14 @@ class SyncHandler(object): deleted = yield self.store.delete_messages_for_device( user_id, device_id, since_stream_id ) - logger.info("Deleted %d to-device messages up to %d", - deleted, since_stream_id) + logger.debug("Deleted %d to-device messages up to %d", + deleted, since_stream_id) messages, stream_id = yield self.store.get_new_messages_for_device( user_id, device_id, since_stream_id, now_token.to_device_key ) - logger.info( + logger.debug( "Returning %d to-device messages between %d and %d (current token: %d)", len(messages), since_stream_id, stream_id, now_token.to_device_key ) diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py index bde3b5cbb..5c7db5e5f 100644 --- a/synapse/storage/deviceinbox.py +++ b/synapse/storage/deviceinbox.py @@ -20,6 +20,8 @@ from twisted.internet import defer from .background_updates import BackgroundUpdateStore +from synapse.util.caches.expiringcache import ExpiringCache + logger = logging.getLogger(__name__) @@ -42,6 +44,15 @@ class DeviceInboxStore(BackgroundUpdateStore): self._background_drop_index_device_inbox, ) + # Map of (user_id, device_id) to the last stream_id that has been + # deleted up to. This is so that we can no op deletions. + self._last_device_delete_cache = ExpiringCache( + cache_name="last_device_delete_cache", + clock=self._clock, + max_len=10000, + expiry_ms=30 * 60 * 1000, + ) + @defer.inlineCallbacks def add_messages_to_device_inbox(self, local_messages_by_user_then_device, remote_messages_by_destination): @@ -251,6 +262,7 @@ class DeviceInboxStore(BackgroundUpdateStore): "get_new_messages_for_device", get_new_messages_for_device_txn, ) + @defer.inlineCallbacks def delete_messages_for_device(self, user_id, device_id, up_to_stream_id): """ Args: @@ -260,6 +272,18 @@ class DeviceInboxStore(BackgroundUpdateStore): Returns: A deferred that resolves to the number of messages deleted. """ + # If we have cached the last stream id we've deleted up to, we can + # check if there is likely to be anything that needs deleting + last_deleted_stream_id = self._last_device_delete_cache.get( + (user_id, device_id), None + ) + if last_deleted_stream_id: + has_changed = self._device_inbox_stream_cache.has_entity_changed( + user_id, last_deleted_stream_id + ) + if not has_changed: + defer.returnValue(0) + def delete_messages_for_device_txn(txn): sql = ( "DELETE FROM device_inbox" @@ -269,10 +293,20 @@ class DeviceInboxStore(BackgroundUpdateStore): txn.execute(sql, (user_id, device_id, up_to_stream_id)) return txn.rowcount - return self.runInteraction( + count = yield self.runInteraction( "delete_messages_for_device", delete_messages_for_device_txn ) + # Update the cache, ensuring that we only ever increase the value + last_deleted_stream_id = self._last_device_delete_cache.get( + (user_id, device_id), 0 + ) + self._last_device_delete_cache[(user_id, device_id)] = max( + last_deleted_stream_id, up_to_stream_id + ) + + defer.returnValue(count) + def get_all_new_device_messages(self, last_pos, current_pos, limit): """ Args: From 8f267fa8a8d6cf50383a6a05c26b83dee06076fb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Mar 2017 11:22:25 +0000 Subject: [PATCH 81/82] Fix it for the workers --- synapse/replication/slave/storage/deviceinbox.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/replication/slave/storage/deviceinbox.py b/synapse/replication/slave/storage/deviceinbox.py index cc860f9f9..f9102e0d8 100644 --- a/synapse/replication/slave/storage/deviceinbox.py +++ b/synapse/replication/slave/storage/deviceinbox.py @@ -17,6 +17,7 @@ from ._base import BaseSlavedStore from ._slaved_id_tracker import SlavedIdTracker from synapse.storage import DataStore from synapse.util.caches.stream_change_cache import StreamChangeCache +from synapse.util.caches.expiringcache import ExpiringCache class SlavedDeviceInboxStore(BaseSlavedStore): @@ -34,6 +35,13 @@ class SlavedDeviceInboxStore(BaseSlavedStore): self._device_inbox_id_gen.get_current_token() ) + self._last_device_delete_cache = ExpiringCache( + cache_name="last_device_delete_cache", + clock=self._clock, + max_len=10000, + expiry_ms=30 * 60 * 1000, + ) + get_to_device_stream_token = DataStore.get_to_device_stream_token.__func__ get_new_messages_for_device = DataStore.get_new_messages_for_device.__func__ get_new_device_msgs_for_remote = DataStore.get_new_device_msgs_for_remote.__func__ From 7eae6eaa2f12474060f0bae03b4aa5bab9c9e1f7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Mar 2017 09:58:39 +0000 Subject: [PATCH 82/82] Revert "Support registration & login with phone number" --- synapse/api/constants.py | 2 - synapse/handlers/auth.py | 32 ++---- synapse/handlers/identity.py | 37 +------ synapse/http/servlet.py | 10 -- synapse/python_dependencies.py | 2 - synapse/rest/client/v1/login.py | 88 ++--------------- synapse/rest/client/v2_alpha/account.py | 114 ++++----------------- synapse/rest/client/v2_alpha/register.py | 120 +++-------------------- synapse/util/msisdn.py | 40 -------- 9 files changed, 50 insertions(+), 395 deletions(-) delete mode 100644 synapse/util/msisdn.py diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 489efb7f8..ca23c9c46 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -45,7 +44,6 @@ class JoinRules(object): class LoginType(object): PASSWORD = u"m.login.password" EMAIL_IDENTITY = u"m.login.email.identity" - MSISDN = u"m.login.msisdn" RECAPTCHA = u"m.login.recaptcha" DUMMY = u"m.login.dummy" diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index e7a1bb724..fffba3438 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Copyright 2014 - 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -48,7 +47,6 @@ class AuthHandler(BaseHandler): LoginType.PASSWORD: self._check_password_auth, LoginType.RECAPTCHA: self._check_recaptcha, LoginType.EMAIL_IDENTITY: self._check_email_identity, - LoginType.MSISDN: self._check_msisdn, LoginType.DUMMY: self._check_dummy_auth, } self.bcrypt_rounds = hs.config.bcrypt_rounds @@ -309,47 +307,31 @@ class AuthHandler(BaseHandler): defer.returnValue(True) raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + @defer.inlineCallbacks def _check_email_identity(self, authdict, _): - return self._check_threepid('email', authdict) - - def _check_msisdn(self, authdict, _): - return self._check_threepid('msisdn', authdict) - - @defer.inlineCallbacks - def _check_dummy_auth(self, authdict, _): - yield run_on_reactor() - defer.returnValue(True) - - @defer.inlineCallbacks - def _check_threepid(self, medium, authdict): yield run_on_reactor() if 'threepid_creds' not in authdict: raise LoginError(400, "Missing threepid_creds", Codes.MISSING_PARAM) threepid_creds = authdict['threepid_creds'] - identity_handler = self.hs.get_handlers().identity_handler - logger.info("Getting validated threepid. threepidcreds: %r", (threepid_creds,)) + logger.info("Getting validated threepid. threepidcreds: %r" % (threepid_creds,)) threepid = yield identity_handler.threepid_from_creds(threepid_creds) if not threepid: raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) - if threepid['medium'] != medium: - raise LoginError( - 401, - "Expecting threepid of type '%s', got '%s'" % ( - medium, threepid['medium'], - ), - errcode=Codes.UNAUTHORIZED - ) - threepid['threepid_creds'] = authdict['threepid_creds'] defer.returnValue(threepid) + @defer.inlineCallbacks + def _check_dummy_auth(self, authdict, _): + yield run_on_reactor() + defer.returnValue(True) + def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 6a53c5eb4..559e5d5a7 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -151,7 +150,7 @@ class IdentityHandler(BaseHandler): params.update(kwargs) try: - data = yield self.http_client.post_json_get_json( + data = yield self.http_client.post_urlencoded_get_json( "https://%s%s" % ( id_server, "/_matrix/identity/api/v1/validate/email/requestToken" @@ -162,37 +161,3 @@ class IdentityHandler(BaseHandler): except CodeMessageException as e: logger.info("Proxied requestToken failed: %r", e) raise e - - @defer.inlineCallbacks - def requestMsisdnToken( - self, id_server, country, phone_number, - client_secret, send_attempt, **kwargs - ): - yield run_on_reactor() - - if not self._should_trust_id_server(id_server): - raise SynapseError( - 400, "Untrusted ID server '%s'" % id_server, - Codes.SERVER_NOT_TRUSTED - ) - - params = { - 'country': country, - 'phone_number': phone_number, - 'client_secret': client_secret, - 'send_attempt': send_attempt, - } - params.update(kwargs) - - try: - data = yield self.http_client.post_json_get_json( - "https://%s%s" % ( - id_server, - "/_matrix/identity/api/v1/validate/msisdn/requestToken" - ), - params - ) - defer.returnValue(data) - except CodeMessageException as e: - logger.info("Proxied requestToken failed: %r", e) - raise e diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 9a4c36ad5..8c22d6f00 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -192,16 +192,6 @@ def parse_json_object_from_request(request): return content -def assert_params_in_request(body, required): - absent = [] - for k in required: - if k not in body: - absent.append(k) - - if len(absent) > 0: - raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - - class RestServlet(object): """ A Synapse REST Servlet. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index c4777b2a2..7817b0cd9 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -1,5 +1,4 @@ # Copyright 2015, 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,7 +37,6 @@ REQUIREMENTS = { "pysaml2>=3.0.0,<4.0.0": ["saml2>=3.0.0,<4.0.0"], "pymacaroons-pynacl": ["pymacaroons"], "msgpack-python>=0.3.0": ["msgpack"], - "phonenumbers>=8.2.0": ["phonenumbers"], } CONDITIONAL_REQUIREMENTS = { "web_client": { diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index c4bbb7027..72057f1b0 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -19,7 +19,6 @@ from synapse.api.errors import SynapseError, LoginError, Codes from synapse.types import UserID from synapse.http.server import finish_request from synapse.http.servlet import parse_json_object_from_request -from synapse.util.msisdn import phone_number_to_msisdn from .base import ClientV1RestServlet, client_path_patterns @@ -38,49 +37,6 @@ import xml.etree.ElementTree as ET logger = logging.getLogger(__name__) -def login_submission_legacy_convert(submission): - """ - If the input login submission is an old style object - (ie. with top-level user / medium / address) convert it - to a typed object. - """ - if "user" in submission: - submission["identifier"] = { - "type": "m.id.user", - "user": submission["user"], - } - del submission["user"] - - if "medium" in submission and "address" in submission: - submission["identifier"] = { - "type": "m.id.thirdparty", - "medium": submission["medium"], - "address": submission["address"], - } - del submission["medium"] - del submission["address"] - - -def login_id_thirdparty_from_phone(identifier): - """ - Convert a phone login identifier type to a generic threepid identifier - Args: - identifier(dict): Login identifier dict of type 'm.id.phone' - - Returns: Login identifier dict of type 'm.id.threepid' - """ - if "country" not in identifier or "number" not in identifier: - raise SynapseError(400, "Invalid phone-type identifier") - - msisdn = phone_number_to_msisdn(identifier["country"], identifier["number"]) - - return { - "type": "m.id.thirdparty", - "medium": "msisdn", - "address": msisdn, - } - - class LoginRestServlet(ClientV1RestServlet): PATTERNS = client_path_patterns("/login$") PASS_TYPE = "m.login.password" @@ -161,52 +117,20 @@ class LoginRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def do_password_login(self, login_submission): - if "password" not in login_submission: - raise SynapseError(400, "Missing parameter: password") - - login_submission_legacy_convert(login_submission) - - if "identifier" not in login_submission: - raise SynapseError(400, "Missing param: identifier") - - identifier = login_submission["identifier"] - if "type" not in identifier: - raise SynapseError(400, "Login identifier has no type") - - # convert phone type identifiers to generic threepids - if identifier["type"] == "m.id.phone": - identifier = login_id_thirdparty_from_phone(identifier) - - # convert threepid identifiers to user IDs - if identifier["type"] == "m.id.thirdparty": - if 'medium' not in identifier or 'address' not in identifier: - raise SynapseError(400, "Invalid thirdparty identifier") - - address = identifier['address'] - if identifier['medium'] == 'email': + if 'medium' in login_submission and 'address' in login_submission: + address = login_submission['address'] + if login_submission['medium'] == 'email': # For emails, transform the address to lowercase. # We store all email addreses as lowercase in the DB. # (See add_threepid in synapse/handlers/auth.py) address = address.lower() user_id = yield self.hs.get_datastore().get_user_id_by_threepid( - identifier['medium'], address + login_submission['medium'], address ) if not user_id: raise LoginError(403, "", errcode=Codes.FORBIDDEN) - - identifier = { - "type": "m.id.user", - "user": user_id, - } - - # by this point, the identifier should be an m.id.user: if it's anything - # else, we haven't understood it. - if identifier["type"] != "m.id.user": - raise SynapseError(400, "Unknown login identifier type") - if "user" not in identifier: - raise SynapseError(400, "User identifier is missing 'user' key") - - user_id = identifier["user"] + else: + user_id = login_submission['user'] if not user_id.startswith('@'): user_id = UserID.create( diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index aac76edf1..398e7f5eb 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,11 +17,8 @@ from twisted.internet import defer from synapse.api.constants import LoginType from synapse.api.errors import LoginError, SynapseError, Codes -from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request, assert_params_in_request -) +from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.util.async import run_on_reactor -from synapse.util.msisdn import phone_number_to_msisdn from ._base import client_v2_patterns @@ -32,11 +28,11 @@ import logging logger = logging.getLogger(__name__) -class EmailPasswordRequestTokenRestServlet(RestServlet): +class PasswordRequestTokenRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/password/email/requestToken$") def __init__(self, hs): - super(EmailPasswordRequestTokenRestServlet, self).__init__() + super(PasswordRequestTokenRestServlet, self).__init__() self.hs = hs self.identity_handler = hs.get_handlers().identity_handler @@ -44,9 +40,14 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): def on_POST(self, request): body = parse_json_object_from_request(request) - assert_params_in_request(body, [ - 'id_server', 'client_secret', 'email', 'send_attempt' - ]) + required = ['id_server', 'client_secret', 'email', 'send_attempt'] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + if absent: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] @@ -59,37 +60,6 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): defer.returnValue((200, ret)) -class MsisdnPasswordRequestTokenRestServlet(RestServlet): - PATTERNS = client_v2_patterns("/account/password/msisdn/requestToken$") - - def __init__(self, hs): - super(MsisdnPasswordRequestTokenRestServlet, self).__init__() - self.hs = hs - self.datastore = self.hs.get_datastore() - self.identity_handler = hs.get_handlers().identity_handler - - @defer.inlineCallbacks - def on_POST(self, request): - body = parse_json_object_from_request(request) - - assert_params_in_request(body, [ - 'id_server', 'client_secret', - 'country', 'phone_number', 'send_attempt', - ]) - - msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - - existingUid = yield self.datastore.get_user_id_by_threepid( - 'msisdn', msisdn - ) - - if existingUid is None: - raise SynapseError(400, "MSISDN not found", Codes.THREEPID_NOT_FOUND) - - ret = yield self.identity_handler.requestMsisdnToken(**body) - defer.returnValue((200, ret)) - - class PasswordRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/password$") @@ -98,7 +68,6 @@ class PasswordRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() - self.datastore = self.hs.get_datastore() @defer.inlineCallbacks def on_POST(self, request): @@ -108,8 +77,7 @@ class PasswordRestServlet(RestServlet): authed, result, params, _ = yield self.auth_handler.check_auth([ [LoginType.PASSWORD], - [LoginType.EMAIL_IDENTITY], - [LoginType.MSISDN], + [LoginType.EMAIL_IDENTITY] ], body, self.hs.get_ip_from_request(request)) if not authed: @@ -134,7 +102,7 @@ class PasswordRestServlet(RestServlet): # (See add_threepid in synapse/handlers/auth.py) threepid['address'] = threepid['address'].lower() # if using email, we must know about the email they're authing with! - threepid_user_id = yield self.datastore.get_user_id_by_threepid( + threepid_user_id = yield self.hs.get_datastore().get_user_id_by_threepid( threepid['medium'], threepid['address'] ) if not threepid_user_id: @@ -201,14 +169,13 @@ class DeactivateAccountRestServlet(RestServlet): defer.returnValue((200, {})) -class EmailThreepidRequestTokenRestServlet(RestServlet): +class ThreepidRequestTokenRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/3pid/email/requestToken$") def __init__(self, hs): self.hs = hs - super(EmailThreepidRequestTokenRestServlet, self).__init__() + super(ThreepidRequestTokenRestServlet, self).__init__() self.identity_handler = hs.get_handlers().identity_handler - self.datastore = self.hs.get_datastore() @defer.inlineCallbacks def on_POST(self, request): @@ -223,7 +190,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): if absent: raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - existingUid = yield self.datastore.get_user_id_by_threepid( + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] ) @@ -234,44 +201,6 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): defer.returnValue((200, ret)) -class MsisdnThreepidRequestTokenRestServlet(RestServlet): - PATTERNS = client_v2_patterns("/account/3pid/msisdn/requestToken$") - - def __init__(self, hs): - self.hs = hs - super(MsisdnThreepidRequestTokenRestServlet, self).__init__() - self.identity_handler = hs.get_handlers().identity_handler - self.datastore = self.hs.get_datastore() - - @defer.inlineCallbacks - def on_POST(self, request): - body = parse_json_object_from_request(request) - - required = [ - 'id_server', 'client_secret', - 'country', 'phone_number', 'send_attempt', - ] - absent = [] - for k in required: - if k not in body: - absent.append(k) - - if absent: - raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - - msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - - existingUid = yield self.datastore.get_user_id_by_threepid( - 'msisdn', msisdn - ) - - if existingUid is not None: - raise SynapseError(400, "MSISDN is already in use", Codes.THREEPID_IN_USE) - - ret = yield self.identity_handler.requestEmailToken(**body) - defer.returnValue((200, ret)) - - class ThreepidRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/3pid$") @@ -281,7 +210,6 @@ class ThreepidRestServlet(RestServlet): self.identity_handler = hs.get_handlers().identity_handler self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() - self.datastore = self.hs.get_datastore() @defer.inlineCallbacks def on_GET(self, request): @@ -289,7 +217,7 @@ class ThreepidRestServlet(RestServlet): requester = yield self.auth.get_user_by_req(request) - threepids = yield self.datastore.user_get_threepids( + threepids = yield self.hs.get_datastore().user_get_threepids( requester.user.to_string() ) @@ -330,7 +258,7 @@ class ThreepidRestServlet(RestServlet): if 'bind' in body and body['bind']: logger.debug( - "Binding threepid %s to %s", + "Binding emails %s to %s", threepid, user_id ) yield self.identity_handler.bind_threepid( @@ -374,11 +302,9 @@ class ThreepidDeleteRestServlet(RestServlet): def register_servlets(hs, http_server): - EmailPasswordRequestTokenRestServlet(hs).register(http_server) - MsisdnPasswordRequestTokenRestServlet(hs).register(http_server) + PasswordRequestTokenRestServlet(hs).register(http_server) PasswordRestServlet(hs).register(http_server) DeactivateAccountRestServlet(hs).register(http_server) - EmailThreepidRequestTokenRestServlet(hs).register(http_server) - MsisdnThreepidRequestTokenRestServlet(hs).register(http_server) + ThreepidRequestTokenRestServlet(hs).register(http_server) ThreepidRestServlet(hs).register(http_server) ThreepidDeleteRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 7448c1346..ccca5a12d 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Copyright 2015 - 2016 OpenMarket Ltd -# Copyright 2017 Vector Creations Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,10 +19,7 @@ import synapse from synapse.api.auth import get_access_token_from_request, has_access_token from synapse.api.constants import LoginType from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError -from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request, assert_params_in_request -) -from synapse.util.msisdn import phone_number_to_msisdn +from synapse.http.servlet import RestServlet, parse_json_object_from_request from ._base import client_v2_patterns @@ -47,7 +43,7 @@ else: logger = logging.getLogger(__name__) -class EmailRegisterRequestTokenRestServlet(RestServlet): +class RegisterRequestTokenRestServlet(RestServlet): PATTERNS = client_v2_patterns("/register/email/requestToken$") def __init__(self, hs): @@ -55,7 +51,7 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): Args: hs (synapse.server.HomeServer): server """ - super(EmailRegisterRequestTokenRestServlet, self).__init__() + super(RegisterRequestTokenRestServlet, self).__init__() self.hs = hs self.identity_handler = hs.get_handlers().identity_handler @@ -63,9 +59,14 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): def on_POST(self, request): body = parse_json_object_from_request(request) - assert_params_in_request(body, [ - 'id_server', 'client_secret', 'email', 'send_attempt' - ]) + required = ['id_server', 'client_secret', 'email', 'send_attempt'] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + if len(absent) > 0: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] @@ -78,43 +79,6 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): defer.returnValue((200, ret)) -class MsisdnRegisterRequestTokenRestServlet(RestServlet): - PATTERNS = client_v2_patterns("/register/msisdn/requestToken$") - - def __init__(self, hs): - """ - Args: - hs (synapse.server.HomeServer): server - """ - super(MsisdnRegisterRequestTokenRestServlet, self).__init__() - self.hs = hs - self.identity_handler = hs.get_handlers().identity_handler - - @defer.inlineCallbacks - def on_POST(self, request): - body = parse_json_object_from_request(request) - - assert_params_in_request(body, [ - 'id_server', 'client_secret', - 'country', 'phone_number', - 'send_attempt', - ]) - - msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - - existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( - 'msisdn', msisdn - ) - - if existingUid is not None: - raise SynapseError( - 400, "Phone number is already in use", Codes.THREEPID_IN_USE - ) - - ret = yield self.identity_handler.requestMsisdnToken(**body) - defer.returnValue((200, ret)) - - class RegisterRestServlet(RestServlet): PATTERNS = client_v2_patterns("/register$") @@ -239,16 +203,12 @@ class RegisterRestServlet(RestServlet): if self.hs.config.enable_registration_captcha: flows = [ [LoginType.RECAPTCHA], - [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], - [LoginType.MSISDN, LoginType.RECAPTCHA], - [LoginType.EMAIL_IDENTITY, LoginType.MSISDN, LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA] ] else: flows = [ [LoginType.DUMMY], - [LoginType.EMAIL_IDENTITY], - [LoginType.MSISDN], - [LoginType.EMAIL_IDENTITY, LoginType.MSISDN], + [LoginType.EMAIL_IDENTITY] ] authed, auth_result, params, session_id = yield self.auth_handler.check_auth( @@ -264,9 +224,8 @@ class RegisterRestServlet(RestServlet): "Already registered user ID %r for this session", registered_user_id ) - # don't re-register the threepids + # don't re-register the email address add_email = False - add_msisdn = False else: # NB: This may be from the auth handler and NOT from the POST if 'password' not in params: @@ -291,7 +250,6 @@ class RegisterRestServlet(RestServlet): ) add_email = True - add_msisdn = True return_dict = yield self._create_registration_details( registered_user_id, params @@ -304,13 +262,6 @@ class RegisterRestServlet(RestServlet): params.get("bind_email") ) - if add_msisdn and auth_result and LoginType.MSISDN in auth_result: - threepid = auth_result[LoginType.MSISDN] - yield self._register_msisdn_threepid( - registered_user_id, threepid, return_dict["access_token"], - params.get("bind_msisdn") - ) - defer.returnValue((200, return_dict)) def on_OPTIONS(self, _): @@ -372,9 +323,8 @@ class RegisterRestServlet(RestServlet): """ reqd = ('medium', 'address', 'validated_at') if any(x not in threepid for x in reqd): - # This will only happen if the ID server returns a malformed response logger.info("Can't add incomplete 3pid") - return + defer.returnValue() yield self.auth_handler.add_threepid( user_id, @@ -421,43 +371,6 @@ class RegisterRestServlet(RestServlet): else: logger.info("bind_email not specified: not binding email") - @defer.inlineCallbacks - def _register_msisdn_threepid(self, user_id, threepid, token, bind_msisdn): - """Add a phone number as a 3pid identifier - - Also optionally binds msisdn to the given user_id on the identity server - - Args: - user_id (str): id of user - threepid (object): m.login.msisdn auth response - token (str): access_token for the user - bind_email (bool): true if the client requested the email to be - bound at the identity server - Returns: - defer.Deferred: - """ - reqd = ('medium', 'address', 'validated_at') - if any(x not in threepid for x in reqd): - # This will only happen if the ID server returns a malformed response - logger.info("Can't add incomplete 3pid") - defer.returnValue() - - yield self.auth_handler.add_threepid( - user_id, - threepid['medium'], - threepid['address'], - threepid['validated_at'], - ) - - if bind_msisdn: - logger.info("bind_msisdn specified: binding") - logger.debug("Binding msisdn %s to %s", threepid, user_id) - yield self.identity_handler.bind_threepid( - threepid['threepid_creds'], user_id - ) - else: - logger.info("bind_msisdn not specified: not binding msisdn") - @defer.inlineCallbacks def _create_registration_details(self, user_id, params): """Complete registration of newly-registered user @@ -536,6 +449,5 @@ class RegisterRestServlet(RestServlet): def register_servlets(hs, http_server): - EmailRegisterRequestTokenRestServlet(hs).register(http_server) - MsisdnRegisterRequestTokenRestServlet(hs).register(http_server) + RegisterRequestTokenRestServlet(hs).register(http_server) RegisterRestServlet(hs).register(http_server) diff --git a/synapse/util/msisdn.py b/synapse/util/msisdn.py deleted file mode 100644 index 607161e7f..000000000 --- a/synapse/util/msisdn.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import phonenumbers -from synapse.api.errors import SynapseError - - -def phone_number_to_msisdn(country, number): - """ - Takes an ISO-3166-1 2 letter country code and phone number and - returns an msisdn representing the canonical version of that - phone number. - Args: - country (str): ISO-3166-1 2 letter country code - number (str): Phone number in a national or international format - - Returns: - (str) The canonical form of the phone number, as an msisdn - Raises: - SynapseError if the number could not be parsed. - """ - try: - phoneNumber = phonenumbers.parse(number, country) - except phonenumbers.NumberParseException: - raise SynapseError(400, "Unable to parse phone number") - return phonenumbers.format_number( - phoneNumber, phonenumbers.PhoneNumberFormat.E164 - )[1:]