From 97bd18af4ee368a5fe8bf8fb06d0299f6b2c1cfd Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 16:32:58 +0000 Subject: [PATCH 01/10] Add automagical AS Publicised Group(s) via registration file "users" namespace: ```YAML ... namespaces: users: - exclusive: true regex: '.*luke.*' group_id: '+all_the_lukes:hsdomain' ... ``` This is part of giving App Services their own groups for matching users. With this, ghost users will be given the appeareance that they are in a group and that they have publicised the fact, but _only_ from the perspective of the `get_publicised_groups_for_user` API. --- synapse/appservice/__init__.py | 22 ++++++++++++++++++++++ synapse/handlers/groups_local.py | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index b98900731..5c6c724fa 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -81,6 +81,8 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + GROUP_ID_REGEX = re.compile('\+.*:.+') + def __init__(self, token, url=None, namespaces=None, hs_token=None, sender=None, id=None, protocols=None, rate_limited=True): self.token = token @@ -125,6 +127,17 @@ class ApplicationService(object): raise ValueError( "Expected bool for 'exclusive' in ns '%s'" % ns ) + if regex_obj.get("group_id"): + if not isinstance(regex_obj.get("group_id"), str): + raise ValueError( + "Expected string for 'group_id' in ns '%s'" % ns + ) + if not ApplicationService.GROUP_ID_REGEX.match( + regex_obj.get("group_id")): + raise ValueError( + "Expected valid group ID for 'group_id' in ns '%s'" % ns + ) + regex = regex_obj.get("regex") if isinstance(regex, basestring): regex_obj["regex"] = re.compile(regex) # Pre-compile regex @@ -251,6 +264,15 @@ class ApplicationService(object): if regex_obj["exclusive"] ] + def get_groups_for_user(self, user_id): + """Get the groups that this user is associated with by this AS + """ + return [ + regex_obj["group_id"] + for regex_obj in self.namespaces[ApplicationService.NS_USERS] + if "group_id" in regex_obj and regex_obj["regex"].match(user_id) + ] + def is_rate_limited(self): return self.rate_limited diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index da00aeb0f..5cc4b86af 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -375,6 +375,12 @@ class GroupsLocalHandler(object): def get_publicised_groups_for_user(self, user_id): if self.hs.is_mine_id(user_id): result = yield self.store.get_publicised_groups_for_user(user_id) + + # Check AS associated groups for this user - this depends on the + # RegExps in the AS registration file (under `users`) + for app_service in self.store.get_app_services(): + result.extend(app_service.get_groups_for_user(user_id)) + defer.returnValue({"groups": result}) else: result = yield self.transport_client.get_publicised_groups_for_user( From b1edf260519eeef06d264f75604994f905c8916a Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 17:54:27 +0000 Subject: [PATCH 02/10] Check group_id belongs to this domain --- synapse/appservice/__init__.py | 14 +++++++++++--- synapse/config/appservice.py | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 5c6c724fa..5be5120c9 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -14,6 +14,7 @@ # limitations under the License. from synapse.api.constants import EventTypes from synapse.util.caches.descriptors import cachedInlineCallbacks +from synapse.types import GroupID, get_domain_from_id from twisted.internet import defer @@ -83,12 +84,13 @@ class ApplicationService(object): GROUP_ID_REGEX = re.compile('\+.*:.+') - def __init__(self, token, url=None, namespaces=None, hs_token=None, + def __init__(self, token, hostname, url=None, namespaces=None, hs_token=None, sender=None, id=None, protocols=None, rate_limited=True): self.token = token self.url = url self.hs_token = hs_token self.sender = sender + self.server_name = hostname self.namespaces = self._check_namespaces(namespaces) self.id = id @@ -132,12 +134,18 @@ class ApplicationService(object): raise ValueError( "Expected string for 'group_id' in ns '%s'" % ns ) - if not ApplicationService.GROUP_ID_REGEX.match( - regex_obj.get("group_id")): + try: + GroupID.from_string(regex_obj.get("group_id")) + except Exception: raise ValueError( "Expected valid group ID for 'group_id' in ns '%s'" % ns ) + if get_domain_from_id(regex_obj.get("group_id")) != self.server_name: + raise ValueError( + "Expected string for 'group_id' to be for this host in ns '%s'" % ns + ) + regex = regex_obj.get("regex") if isinstance(regex, basestring): regex_obj["regex"] = re.compile(regex) # Pre-compile regex diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 82c50b824..aba0aec6e 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -154,6 +154,7 @@ def _load_appservice(hostname, as_info, config_filename): ) return ApplicationService( token=as_info["as_token"], + hostname=hostname, url=as_info["url"], namespaces=as_info["namespaces"], hs_token=as_info["hs_token"], From 5b48eec4a12032df0c17f054bb50ebb510bd9ae5 Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 17:55:15 +0000 Subject: [PATCH 03/10] Make sure we check AS groups for lookup on bulk --- synapse/handlers/groups_local.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index 5cc4b86af..7e5d3f148 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -421,4 +421,9 @@ class GroupsLocalHandler(object): uid ) + # Check AS associated groups for this user - this depends on the + # RegExps in the AS registration file (under `users`) + for app_service in self.store.get_app_services(): + results[uid].extend(app_service.get_groups_for_user(uid)) + defer.returnValue({"users": results}) From 9d83d52027e42728040bdba94dfb2ea9fa418680 Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 17:57:34 +0000 Subject: [PATCH 04/10] Use a generator instead of a list --- synapse/appservice/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 5be5120c9..fbb4e4420 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -275,11 +275,11 @@ class ApplicationService(object): def get_groups_for_user(self, user_id): """Get the groups that this user is associated with by this AS """ - return [ + return ( regex_obj["group_id"] for regex_obj in self.namespaces[ApplicationService.NS_USERS] if "group_id" in regex_obj and regex_obj["regex"].match(user_id) - ] + ) def is_rate_limited(self): return self.rate_limited From 270f9cd23a4f20781ecb16a042e928a0e9f3a239 Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 18:03:31 +0000 Subject: [PATCH 05/10] Flake8 --- synapse/appservice/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index fbb4e4420..916ccf714 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -143,7 +143,7 @@ class ApplicationService(object): if get_domain_from_id(regex_obj.get("group_id")) != self.server_name: raise ValueError( - "Expected string for 'group_id' to be for this host in ns '%s'" % ns + "Expected 'group_id' to be this host in ns '%s'" % ns ) regex = regex_obj.get("regex") From 4e8374856d6611980c676e355eaf4de464ec312e Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 18:03:46 +0000 Subject: [PATCH 06/10] Document get_groups_for_user --- synapse/appservice/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 916ccf714..e49452532 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -274,6 +274,12 @@ class ApplicationService(object): def get_groups_for_user(self, user_id): """Get the groups that this user is associated with by this AS + + Args: + user_id (str): The ID of the user. + + Returns: + iterable[str]: an iterable that yields group_id strings. """ return ( regex_obj["group_id"] From d8391f0541929b141647505ae0a5a4fb593ed68b Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 18:05:57 +0000 Subject: [PATCH 07/10] Remove unused GROUP_ID_REGEX --- synapse/appservice/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index e49452532..f330d77b6 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -82,8 +82,6 @@ class ApplicationService(object): # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] - GROUP_ID_REGEX = re.compile('\+.*:.+') - def __init__(self, token, hostname, url=None, namespaces=None, hs_token=None, sender=None, id=None, protocols=None, rate_limited=True): self.token = token From e836bdf734b597526a40d1e759124df7b97dddae Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 18:14:39 +0000 Subject: [PATCH 08/10] Fix tests --- tests/appservice/test_appservice.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 7586ea905..f943dbfa0 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -36,6 +36,7 @@ class ApplicationServiceTestCase(unittest.TestCase): id="unique_identifier", url="some_url", token="some_token", + hostname="matrix.org", # only used by get_groups_for_user namespaces={ ApplicationService.NS_USERS: [], ApplicationService.NS_ROOMS: [], From 7fb0e98b0327f56f13bad7c21e897911e1e2a355 Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 18:18:30 +0000 Subject: [PATCH 09/10] Extract group_id from the dict for multiple use --- synapse/appservice/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index f330d77b6..d5a7a5ce2 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -127,19 +127,20 @@ class ApplicationService(object): raise ValueError( "Expected bool for 'exclusive' in ns '%s'" % ns ) - if regex_obj.get("group_id"): - if not isinstance(regex_obj.get("group_id"), str): + group_id = regex_obj.get("group_id") + if group_id: + if not isinstance(group_id, str): raise ValueError( "Expected string for 'group_id' in ns '%s'" % ns ) try: - GroupID.from_string(regex_obj.get("group_id")) + GroupID.from_string(group_id) except Exception: raise ValueError( "Expected valid group ID for 'group_id' in ns '%s'" % ns ) - if get_domain_from_id(regex_obj.get("group_id")) != self.server_name: + if get_domain_from_id(group_id) != self.server_name: raise ValueError( "Expected 'group_id' to be this host in ns '%s'" % ns ) From ab1dc84779fea9b9950e05e7610eadf195cac967 Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 16 Nov 2017 18:22:40 +0000 Subject: [PATCH 10/10] Add extra space before inline comment --- tests/appservice/test_appservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index f943dbfa0..5b2b95860 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -36,7 +36,7 @@ class ApplicationServiceTestCase(unittest.TestCase): id="unique_identifier", url="some_url", token="some_token", - hostname="matrix.org", # only used by get_groups_for_user + hostname="matrix.org", # only used by get_groups_for_user namespaces={ ApplicationService.NS_USERS: [], ApplicationService.NS_ROOMS: [],