From 63c19e1df949dd8fc15defc1942e2405c5426958 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 24 Aug 2016 11:55:57 +0100 Subject: [PATCH 1/9] Move 3PU/3PL lookup APIs into /thirdparty containing entity --- synapse/appservice/api.py | 4 ++-- synapse/rest/client/v2_alpha/thirdparty.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 066127b66..07ec33b9a 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -97,10 +97,10 @@ class ApplicationServiceApi(SimpleHttpClient): @defer.inlineCallbacks def query_3pe(self, service, kind, protocol, fields): if kind == ThirdPartyEntityKind.USER: - uri = "%s/3pu/%s" % (service.url, urllib.quote(protocol)) + uri = "%s/thirdparty/user/%s" % (service.url, urllib.quote(protocol)) required_field = "userid" elif kind == ThirdPartyEntityKind.LOCATION: - uri = "%s/3pl/%s" % (service.url, urllib.quote(protocol)) + uri = "%s/thirdparty/location/%s" % (service.url, urllib.quote(protocol)) required_field = "alias" else: raise ValueError( diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py index 9abca3a8a..8821e101b 100644 --- a/synapse/rest/client/v2_alpha/thirdparty.py +++ b/synapse/rest/client/v2_alpha/thirdparty.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class ThirdPartyUserServlet(RestServlet): - PATTERNS = client_v2_patterns("/3pu(/(?P[^/]+))?$", + PATTERNS = client_v2_patterns("/thirdparty/user(/(?P[^/]+))?$", releases=()) def __init__(self, hs): @@ -50,7 +50,7 @@ class ThirdPartyUserServlet(RestServlet): class ThirdPartyLocationServlet(RestServlet): - PATTERNS = client_v2_patterns("/3pl(/(?P[^/]+))?$", + PATTERNS = client_v2_patterns("/thirdparty/location(/(?P[^/]+))?$", releases=()) def __init__(self, hs): From 9899824b851bc1e1ccaa0da8cb2dc5bf783014e8 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 24 Aug 2016 12:33:01 +0100 Subject: [PATCH 2/9] Initial hack at the 3PN protocols metadata lookup API --- synapse/appservice/__init__.py | 2 ++ synapse/handlers/appservice.py | 8 +++++ synapse/rest/client/v2_alpha/thirdparty.py | 34 ++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index bde9b51b2..126a10efb 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -88,6 +88,8 @@ class ApplicationService(object): self.sender = sender self.namespaces = self._check_namespaces(namespaces) self.id = id + + # .protocols is a publicly visible field if protocols: self.protocols = set(protocols) else: diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index dd285452c..6273ff524 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -175,6 +175,14 @@ class ApplicationServicesHandler(object): defer.returnValue(ret) + @defer.inlineCallbacks + def get_3pe_protocols(self): + services = yield self.store.get_app_services() + protocols = set() + for s in services: + protocols.update(s.protocols) + defer.returnValue(protocols) + @defer.inlineCallbacks def _get_services_for_event(self, event): """Retrieve a list of application services interested in this event. diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py index 8821e101b..2decadb00 100644 --- a/synapse/rest/client/v2_alpha/thirdparty.py +++ b/synapse/rest/client/v2_alpha/thirdparty.py @@ -25,6 +25,39 @@ from ._base import client_v2_patterns logger = logging.getLogger(__name__) +class ThirdPartyProtocolsServlet(RestServlet): + PATTERNS = client_v2_patterns("/thirdparty/protocols", releases=()) + + META = { + # TODO(paul): Declare kinds of metadata in here + } + + def __init__(self, hs): + super(ThirdPartyProtocolsServlet, self).__init__() + + self.auth = hs.get_auth() + self.appservice_handler = hs.get_application_service_handler() + + @defer.inlineCallbacks + def on_GET(self, request): + yield self.auth.get_user_by_req(request) + + protocols = yield self.appservice_handler.get_3pe_protocols() + + result = {} + # TODO(paul): Eventually this kind of metadata wants to come from the + # ASes themselves + for protocol in protocols: + if protocol in self.META: + result[protocol] = self.META[protocol] + else: + # We don't know any metadata for it, but we'd best at least + # still declare that we know it exists + result[protocol] = {} + + defer.returnValue((200, result)) + + class ThirdPartyUserServlet(RestServlet): PATTERNS = client_v2_patterns("/thirdparty/user(/(?P[^/]+))?$", releases=()) @@ -74,5 +107,6 @@ class ThirdPartyLocationServlet(RestServlet): def register_servlets(hs, http_server): + ThirdPartyProtocolsServlet(hs).register(http_server) ThirdPartyUserServlet(hs).register(http_server) ThirdPartyLocationServlet(hs).register(http_server) From 965f33c9018c80a6afce1eb58b5a8ef442108fc3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 24 Aug 2016 12:34:03 +0100 Subject: [PATCH 3/9] Declare 'gitter' known protocol, with user lookup --- synapse/rest/client/v2_alpha/thirdparty.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py index 2decadb00..ae7901551 100644 --- a/synapse/rest/client/v2_alpha/thirdparty.py +++ b/synapse/rest/client/v2_alpha/thirdparty.py @@ -30,6 +30,9 @@ class ThirdPartyProtocolsServlet(RestServlet): META = { # TODO(paul): Declare kinds of metadata in here + "gitter": { + "user_fields": ["username"], + } } def __init__(self, hs): From 8e1ed09dff902c84beb9dff59305aa78e8772d81 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 24 Aug 2016 13:01:53 +0100 Subject: [PATCH 4/9] Move static knowledge of protocol metadata into AS handler; cache the result --- synapse/handlers/appservice.py | 26 ++++++++++++++++++++-- synapse/rest/client/v2_alpha/thirdparty.py | 21 +---------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 6273ff524..1c5c5ec30 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -37,6 +37,13 @@ def log_failure(failure): class ApplicationServicesHandler(object): + PROTOCOL_META = { + # TODO(paul): Declare kinds of metadata in here + "gitter": { + "user_fields": ["username"], + } + } + def __init__(self, hs): self.store = hs.get_datastore() self.is_mine_id = hs.is_mine_id @@ -48,6 +55,7 @@ class ApplicationServicesHandler(object): self.current_max = 0 self.is_processing = False + self.supported_protocols = None @defer.inlineCallbacks def notify_interested_services(self, current_id): @@ -177,10 +185,24 @@ class ApplicationServicesHandler(object): @defer.inlineCallbacks def get_3pe_protocols(self): + # The set of supported AS protocols and the metadata about them is + # effectively static during the lifetime of a homeserver process. We + # can look this up once and just cache it. + if self.supported_protocols: + defer.returnValue(self.supported_protocols) + services = yield self.store.get_app_services() - protocols = set() + protocols = {} for s in services: - protocols.update(s.protocols) + for p in s.protocols: + if p in self.PROTOCOL_META: + protocols[p] = self.PROTOCOL_META[p] + else: + # We don't know any metadata for it, but we'd best at least + # still declare that we know it exists + protocols[p] = {} + + self.supported_protocols = protocols defer.returnValue(protocols) @defer.inlineCallbacks diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py index ae7901551..bbc3e9b96 100644 --- a/synapse/rest/client/v2_alpha/thirdparty.py +++ b/synapse/rest/client/v2_alpha/thirdparty.py @@ -28,13 +28,6 @@ logger = logging.getLogger(__name__) class ThirdPartyProtocolsServlet(RestServlet): PATTERNS = client_v2_patterns("/thirdparty/protocols", releases=()) - META = { - # TODO(paul): Declare kinds of metadata in here - "gitter": { - "user_fields": ["username"], - } - } - def __init__(self, hs): super(ThirdPartyProtocolsServlet, self).__init__() @@ -46,19 +39,7 @@ class ThirdPartyProtocolsServlet(RestServlet): yield self.auth.get_user_by_req(request) protocols = yield self.appservice_handler.get_3pe_protocols() - - result = {} - # TODO(paul): Eventually this kind of metadata wants to come from the - # ASes themselves - for protocol in protocols: - if protocol in self.META: - result[protocol] = self.META[protocol] - else: - # We don't know any metadata for it, but we'd best at least - # still declare that we know it exists - result[protocol] = {} - - defer.returnValue((200, result)) + defer.returnValue((200, protocols)) class ThirdPartyUserServlet(RestServlet): From 5474824975b5372665b0921960a0c3887a49efdb Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 25 Aug 2016 15:10:06 +0100 Subject: [PATCH 5/9] Actually query over AS API for 3PE lookup metadata --- synapse/appservice/api.py | 20 ++++++++++++++++++++ synapse/handlers/appservice.py | 14 +------------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 07ec33b9a..1de0e7240 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -52,6 +52,13 @@ class ApplicationServiceApi(SimpleHttpClient): pushing. """ + PROTOCOL_META = { + # TODO(paul): Declare kinds of metadata in here + "gitter": { + "user_fields": ["username"], + } + } + def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) self.clock = hs.get_clock() @@ -131,6 +138,19 @@ class ApplicationServiceApi(SimpleHttpClient): logger.warning("query_3pe to %s threw exception %s", uri, ex) defer.returnValue([]) + @defer.inlineCallbacks + def get_3pe_protocol(self, service, protocol): + # TODO: cache + uri = "%s/thirdparty/protocol/%s" % (service.url, urllib.quote(protocol)) + try: + response = yield self.get_json(uri, {}) + defer.returnValue(response) + except Exception as ex: + logger.warning("query_3pe_protocol to %s threw exception %s", + uri, ex + ) + defer.returnValue({}) + @defer.inlineCallbacks def push_bulk(self, service, events, txn_id=None): events = self._serialize(events) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 1c5c5ec30..efb9da798 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -37,13 +37,6 @@ def log_failure(failure): class ApplicationServicesHandler(object): - PROTOCOL_META = { - # TODO(paul): Declare kinds of metadata in here - "gitter": { - "user_fields": ["username"], - } - } - def __init__(self, hs): self.store = hs.get_datastore() self.is_mine_id = hs.is_mine_id @@ -195,12 +188,7 @@ class ApplicationServicesHandler(object): protocols = {} for s in services: for p in s.protocols: - if p in self.PROTOCOL_META: - protocols[p] = self.PROTOCOL_META[p] - else: - # We don't know any metadata for it, but we'd best at least - # still declare that we know it exists - protocols[p] = {} + protocols[p] = yield self.appservice_api.get_3pe_protocol(s, p) self.supported_protocols = protocols defer.returnValue(protocols) From d0b8d49f714dca232f8a970e31042686ea82b982 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 25 Aug 2016 15:45:28 +0100 Subject: [PATCH 6/9] Kill PROTOCOL_META since I'm not using it any more --- synapse/appservice/api.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 1de0e7240..acef09609 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -52,13 +52,6 @@ class ApplicationServiceApi(SimpleHttpClient): pushing. """ - PROTOCOL_META = { - # TODO(paul): Declare kinds of metadata in here - "gitter": { - "user_fields": ["username"], - } - } - def __init__(self, hs): super(ApplicationServiceApi, self).__init__(hs) self.clock = hs.get_clock() From db7283cc6bf16a5640da3f650cc30996af29d291 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 25 Aug 2016 15:56:27 +0100 Subject: [PATCH 7/9] Implement a ResponseCache around 3PE lookup metadata lookups --- synapse/appservice/api.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index acef09609..9de5c6e47 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -17,6 +17,7 @@ from twisted.internet import defer from synapse.api.errors import CodeMessageException from synapse.http.client import SimpleHttpClient from synapse.events.utils import serialize_event +from synapse.util.caches.response_cache import ResponseCache from synapse.types import ThirdPartyEntityKind import logging @@ -25,6 +26,9 @@ import urllib logger = logging.getLogger(__name__) +HOUR_IN_MS = 60 * 60 * 1000 + + def _is_valid_3pe_result(r, field): if not isinstance(r, dict): return False @@ -56,6 +60,8 @@ class ApplicationServiceApi(SimpleHttpClient): super(ApplicationServiceApi, self).__init__(hs) self.clock = hs.get_clock() + self.protocol_meta_cache = ResponseCache(hs, timeout_ms=1*HOUR_IN_MS) + @defer.inlineCallbacks def query_user(self, service, user_id): uri = service.url + ("/users/%s" % urllib.quote(user_id)) @@ -131,18 +137,22 @@ class ApplicationServiceApi(SimpleHttpClient): logger.warning("query_3pe to %s threw exception %s", uri, ex) defer.returnValue([]) - @defer.inlineCallbacks def get_3pe_protocol(self, service, protocol): - # TODO: cache - uri = "%s/thirdparty/protocol/%s" % (service.url, urllib.quote(protocol)) - try: - response = yield self.get_json(uri, {}) - defer.returnValue(response) - except Exception as ex: - logger.warning("query_3pe_protocol to %s threw exception %s", - uri, ex - ) - defer.returnValue({}) + @defer.inlineCallbacks + def _get(): + uri = "%s/thirdparty/protocol/%s" % (service.url, urllib.quote(protocol)) + try: + defer.returnValue((yield self.get_json(uri, {}))) + except Exception as ex: + logger.warning("query_3pe_protocol to %s threw exception %s", + uri, ex + ) + defer.returnValue({}) + + key = (service.id, protocol) + return self.protocol_meta_cache.get(key) or ( + self.protocol_meta_cache.set(key, _get()) + ) @defer.inlineCallbacks def push_bulk(self, service, events, txn_id=None): From c435bfee9c2536b23bd2fbd0a590231a4bd2e050 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 25 Aug 2016 15:57:07 +0100 Subject: [PATCH 8/9] Don't need toplevel cache on 3PE lookup metadata any more --- synapse/handlers/appservice.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index efb9da798..94d781743 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -48,7 +48,6 @@ class ApplicationServicesHandler(object): self.current_max = 0 self.is_processing = False - self.supported_protocols = None @defer.inlineCallbacks def notify_interested_services(self, current_id): @@ -178,19 +177,12 @@ class ApplicationServicesHandler(object): @defer.inlineCallbacks def get_3pe_protocols(self): - # The set of supported AS protocols and the metadata about them is - # effectively static during the lifetime of a homeserver process. We - # can look this up once and just cache it. - if self.supported_protocols: - defer.returnValue(self.supported_protocols) - services = yield self.store.get_app_services() protocols = {} for s in services: for p in s.protocols: protocols[p] = yield self.appservice_api.get_3pe_protocol(s, p) - self.supported_protocols = protocols defer.returnValue(protocols) @defer.inlineCallbacks From adf53f04cea7eb9629896769aa8ca642a0a4646e Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 25 Aug 2016 16:00:31 +0100 Subject: [PATCH 9/9] appease pep8 --- synapse/appservice/api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 9de5c6e47..d28a9dff0 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -60,7 +60,7 @@ class ApplicationServiceApi(SimpleHttpClient): super(ApplicationServiceApi, self).__init__(hs) self.clock = hs.get_clock() - self.protocol_meta_cache = ResponseCache(hs, timeout_ms=1*HOUR_IN_MS) + self.protocol_meta_cache = ResponseCache(hs, timeout_ms=HOUR_IN_MS) @defer.inlineCallbacks def query_user(self, service, user_id): @@ -145,8 +145,7 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue((yield self.get_json(uri, {}))) except Exception as ex: logger.warning("query_3pe_protocol to %s threw exception %s", - uri, ex - ) + uri, ex) defer.returnValue({}) key = (service.id, protocol)