Standardise the module interface (#10062)

This PR adds a common configuration section for all modules (see docs). These modules are then loaded at startup by the homeserver. Modules register their hooks and web resources using the new `register_[...]_callbacks` and `register_web_resource` methods of the module API.
This commit is contained in:
Brendan Abolivier 2021-06-18 13:15:52 +02:00 committed by GitHub
parent 91fa9cca99
commit 1b3e398bea
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 768 additions and 187 deletions

View file

@ -35,6 +35,7 @@ from synapse.app import check_bind_error
from synapse.app.phone_stats_home import start_phone_stats_home
from synapse.config.homeserver import HomeServerConfig
from synapse.crypto import context_factory
from synapse.events.spamcheck import load_legacy_spam_checkers
from synapse.logging.context import PreserveLoggingContext
from synapse.metrics.background_process_metrics import wrap_as_background_process
from synapse.metrics.jemalloc import setup_jemalloc_stats
@ -330,6 +331,14 @@ async def start(hs: "synapse.server.HomeServer"):
# Start the tracer
synapse.logging.opentracing.init_tracer(hs) # type: ignore[attr-defined] # noqa
# Instantiate the modules so they can register their web resources to the module API
# before we start the listeners.
module_api = hs.get_module_api()
for module, config in hs.config.modules.loaded_modules:
module(config=config, api=module_api)
load_legacy_spam_checkers(hs)
# It is now safe to start your Synapse.
hs.start_listening()
hs.get_datastore().db_pool.start_profiling()

View file

@ -354,6 +354,10 @@ class GenericWorkerServer(HomeServer):
if name == "replication":
resources[REPLICATION_PREFIX] = ReplicationRestResource(self)
# Attach additional resources registered by modules.
resources.update(self._module_web_resources)
self._module_web_resources_consumed = True
root_resource = create_resource_tree(resources, OptionsResource())
_base.listen_tcp(

View file

@ -124,6 +124,10 @@ class SynapseHomeServer(HomeServer):
)
resources[path] = resource
# Attach additional resources registered by modules.
resources.update(self._module_web_resources)
self._module_web_resources_consumed = True
# try to find something useful to redirect '/' to
if WEB_CLIENT_PREFIX in resources:
root_resource = RootOptionsRedirectResource(WEB_CLIENT_PREFIX)

View file

@ -16,6 +16,7 @@ from synapse.config import (
key,
logger,
metrics,
modules,
oidc,
password_auth_providers,
push,
@ -85,6 +86,7 @@ class RootConfig:
thirdpartyrules: third_party_event_rules.ThirdPartyRulesConfig
tracer: tracer.TracerConfig
redis: redis.RedisConfig
modules: modules.ModulesConfig
config_classes: List = ...
def __init__(self) -> None: ...

View file

@ -1,5 +1,4 @@
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2018 New Vector Ltd
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -30,6 +29,7 @@ from .jwt import JWTConfig
from .key import KeyConfig
from .logger import LoggingConfig
from .metrics import MetricsConfig
from .modules import ModulesConfig
from .oidc import OIDCConfig
from .password_auth_providers import PasswordAuthProviderConfig
from .push import PushConfig
@ -56,6 +56,7 @@ from .workers import WorkerConfig
class HomeServerConfig(RootConfig):
config_classes = [
ModulesConfig,
ServerConfig,
TlsConfig,
FederationConfig,

49
synapse/config/modules.py Normal file
View file

@ -0,0 +1,49 @@
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Any, Dict, List, Tuple
from synapse.config._base import Config, ConfigError
from synapse.util.module_loader import load_module
class ModulesConfig(Config):
section = "modules"
def read_config(self, config: dict, **kwargs):
self.loaded_modules: List[Tuple[Any, Dict]] = []
configured_modules = config.get("modules") or []
for i, module in enumerate(configured_modules):
config_path = ("modules", "<item %i>" % i)
if not isinstance(module, dict):
raise ConfigError("expected a mapping", config_path)
self.loaded_modules.append(load_module(module, config_path))
def generate_config_section(self, **kwargs):
return """
## Modules ##
# Server admins can expand Synapse's functionality with external modules.
#
# See https://matrix-org.github.io/synapse/develop/modules.html for more
# documentation on how to configure or create custom modules for Synapse.
#
modules:
# - module: my_super_module.MySuperClass
# config:
# do_thing: true
# - module: my_other_super_module.SomeClass
# config: {}
"""

View file

@ -42,18 +42,3 @@ class SpamCheckerConfig(Config):
self.spam_checkers.append(load_module(spam_checker, config_path))
else:
raise ConfigError("spam_checker syntax is incorrect")
def generate_config_section(self, **kwargs):
return """\
# Spam checkers are third-party modules that can block specific actions
# of local users, such as creating rooms and registering undesirable
# usernames, as well as remote users by redacting incoming events.
#
spam_checker:
#- module: "my_custom_project.SuperSpamChecker"
# config:
# example_option: 'things'
#- module: "some_other_project.BadEventStopper"
# config:
# example_stop_events_from: ['@bad:example.com']
"""

View file

@ -15,7 +15,18 @@
import inspect
import logging
from typing import TYPE_CHECKING, Any, Collection, Dict, List, Optional, Tuple, Union
from typing import (
TYPE_CHECKING,
Any,
Awaitable,
Callable,
Collection,
Dict,
List,
Optional,
Tuple,
Union,
)
from synapse.rest.media.v1._base import FileInfo
from synapse.rest.media.v1.media_storage import ReadableFileWrapper
@ -29,20 +40,186 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
CHECK_EVENT_FOR_SPAM_CALLBACK = Callable[
["synapse.events.EventBase"],
Awaitable[Union[bool, str]],
]
USER_MAY_INVITE_CALLBACK = Callable[[str, str, str], Awaitable[bool]]
USER_MAY_CREATE_ROOM_CALLBACK = Callable[[str], Awaitable[bool]]
USER_MAY_CREATE_ROOM_ALIAS_CALLBACK = Callable[[str, RoomAlias], Awaitable[bool]]
USER_MAY_PUBLISH_ROOM_CALLBACK = Callable[[str, str], Awaitable[bool]]
CHECK_USERNAME_FOR_SPAM_CALLBACK = Callable[[Dict[str, str]], Awaitable[bool]]
LEGACY_CHECK_REGISTRATION_FOR_SPAM_CALLBACK = Callable[
[
Optional[dict],
Optional[str],
Collection[Tuple[str, str]],
],
Awaitable[RegistrationBehaviour],
]
CHECK_REGISTRATION_FOR_SPAM_CALLBACK = Callable[
[
Optional[dict],
Optional[str],
Collection[Tuple[str, str]],
Optional[str],
],
Awaitable[RegistrationBehaviour],
]
CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK = Callable[
[ReadableFileWrapper, FileInfo],
Awaitable[bool],
]
def load_legacy_spam_checkers(hs: "synapse.server.HomeServer"):
"""Wrapper that loads spam checkers configured using the old configuration, and
registers the spam checker hooks they implement.
"""
spam_checkers = [] # type: List[Any]
api = hs.get_module_api()
for module, config in hs.config.spam_checkers:
# Older spam checkers don't accept the `api` argument, so we
# try and detect support.
spam_args = inspect.getfullargspec(module)
if "api" in spam_args.args:
spam_checkers.append(module(config=config, api=api))
else:
spam_checkers.append(module(config=config))
# The known spam checker hooks. If a spam checker module implements a method
# which name appears in this set, we'll want to register it.
spam_checker_methods = {
"check_event_for_spam",
"user_may_invite",
"user_may_create_room",
"user_may_create_room_alias",
"user_may_publish_room",
"check_username_for_spam",
"check_registration_for_spam",
"check_media_file_for_spam",
}
for spam_checker in spam_checkers:
# Methods on legacy spam checkers might not be async, so we wrap them around a
# wrapper that will call maybe_awaitable on the result.
def async_wrapper(f: Optional[Callable]) -> Optional[Callable[..., Awaitable]]:
# f might be None if the callback isn't implemented by the module. In this
# case we don't want to register a callback at all so we return None.
if f is None:
return None
if f.__name__ == "check_registration_for_spam":
checker_args = inspect.signature(f)
if len(checker_args.parameters) == 3:
# Backwards compatibility; some modules might implement a hook that
# doesn't expect a 4th argument. In this case, wrap it in a function
# that gives it only 3 arguments and drops the auth_provider_id on
# the floor.
def wrapper(
email_threepid: Optional[dict],
username: Optional[str],
request_info: Collection[Tuple[str, str]],
auth_provider_id: Optional[str],
) -> Union[Awaitable[RegistrationBehaviour], RegistrationBehaviour]:
# We've already made sure f is not None above, but mypy doesn't
# do well across function boundaries so we need to tell it f is
# definitely not None.
assert f is not None
return f(
email_threepid,
username,
request_info,
)
f = wrapper
elif len(checker_args.parameters) != 4:
raise RuntimeError(
"Bad signature for callback check_registration_for_spam",
)
def run(*args, **kwargs):
# We've already made sure f is not None above, but mypy doesn't do well
# across function boundaries so we need to tell it f is definitely not
# None.
assert f is not None
return maybe_awaitable(f(*args, **kwargs))
return run
# Register the hooks through the module API.
hooks = {
hook: async_wrapper(getattr(spam_checker, hook, None))
for hook in spam_checker_methods
}
api.register_spam_checker_callbacks(**hooks)
class SpamChecker:
def __init__(self, hs: "synapse.server.HomeServer"):
self.spam_checkers = [] # type: List[Any]
api = hs.get_module_api()
def __init__(self):
self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = []
self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = []
self._user_may_create_room_callbacks: List[USER_MAY_CREATE_ROOM_CALLBACK] = []
self._user_may_create_room_alias_callbacks: List[
USER_MAY_CREATE_ROOM_ALIAS_CALLBACK
] = []
self._user_may_publish_room_callbacks: List[USER_MAY_PUBLISH_ROOM_CALLBACK] = []
self._check_username_for_spam_callbacks: List[
CHECK_USERNAME_FOR_SPAM_CALLBACK
] = []
self._check_registration_for_spam_callbacks: List[
CHECK_REGISTRATION_FOR_SPAM_CALLBACK
] = []
self._check_media_file_for_spam_callbacks: List[
CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK
] = []
for module, config in hs.config.spam_checkers:
# Older spam checkers don't accept the `api` argument, so we
# try and detect support.
spam_args = inspect.getfullargspec(module)
if "api" in spam_args.args:
self.spam_checkers.append(module(config=config, api=api))
else:
self.spam_checkers.append(module(config=config))
def register_callbacks(
self,
check_event_for_spam: Optional[CHECK_EVENT_FOR_SPAM_CALLBACK] = None,
user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None,
user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None,
user_may_create_room_alias: Optional[
USER_MAY_CREATE_ROOM_ALIAS_CALLBACK
] = None,
user_may_publish_room: Optional[USER_MAY_PUBLISH_ROOM_CALLBACK] = None,
check_username_for_spam: Optional[CHECK_USERNAME_FOR_SPAM_CALLBACK] = None,
check_registration_for_spam: Optional[
CHECK_REGISTRATION_FOR_SPAM_CALLBACK
] = None,
check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None,
):
"""Register callbacks from module for each hook."""
if check_event_for_spam is not None:
self._check_event_for_spam_callbacks.append(check_event_for_spam)
if user_may_invite is not None:
self._user_may_invite_callbacks.append(user_may_invite)
if user_may_create_room is not None:
self._user_may_create_room_callbacks.append(user_may_create_room)
if user_may_create_room_alias is not None:
self._user_may_create_room_alias_callbacks.append(
user_may_create_room_alias,
)
if user_may_publish_room is not None:
self._user_may_publish_room_callbacks.append(user_may_publish_room)
if check_username_for_spam is not None:
self._check_username_for_spam_callbacks.append(check_username_for_spam)
if check_registration_for_spam is not None:
self._check_registration_for_spam_callbacks.append(
check_registration_for_spam,
)
if check_media_file_for_spam is not None:
self._check_media_file_for_spam_callbacks.append(check_media_file_for_spam)
async def check_event_for_spam(
self, event: "synapse.events.EventBase"
@ -60,9 +237,10 @@ class SpamChecker:
True or a string if the event is spammy. If a string is returned it
will be used as the error message returned to the user.
"""
for spam_checker in self.spam_checkers:
if await maybe_awaitable(spam_checker.check_event_for_spam(event)):
return True
for callback in self._check_event_for_spam_callbacks:
res = await callback(event) # type: Union[bool, str]
if res:
return res
return False
@ -81,15 +259,8 @@ class SpamChecker:
Returns:
True if the user may send an invite, otherwise False
"""
for spam_checker in self.spam_checkers:
if (
await maybe_awaitable(
spam_checker.user_may_invite(
inviter_userid, invitee_userid, room_id
)
)
is False
):
for callback in self._user_may_invite_callbacks:
if await callback(inviter_userid, invitee_userid, room_id) is False:
return False
return True
@ -105,11 +276,8 @@ class SpamChecker:
Returns:
True if the user may create a room, otherwise False
"""
for spam_checker in self.spam_checkers:
if (
await maybe_awaitable(spam_checker.user_may_create_room(userid))
is False
):
for callback in self._user_may_create_room_callbacks:
if await callback(userid) is False:
return False
return True
@ -128,13 +296,8 @@ class SpamChecker:
Returns:
True if the user may create a room alias, otherwise False
"""
for spam_checker in self.spam_checkers:
if (
await maybe_awaitable(
spam_checker.user_may_create_room_alias(userid, room_alias)
)
is False
):
for callback in self._user_may_create_room_alias_callbacks:
if await callback(userid, room_alias) is False:
return False
return True
@ -151,13 +314,8 @@ class SpamChecker:
Returns:
True if the user may publish the room, otherwise False
"""
for spam_checker in self.spam_checkers:
if (
await maybe_awaitable(
spam_checker.user_may_publish_room(userid, room_id)
)
is False
):
for callback in self._user_may_publish_room_callbacks:
if await callback(userid, room_id) is False:
return False
return True
@ -177,15 +335,11 @@ class SpamChecker:
Returns:
True if the user is spammy.
"""
for spam_checker in self.spam_checkers:
# For backwards compatibility, only run if the method exists on the
# spam checker
checker = getattr(spam_checker, "check_username_for_spam", None)
if checker:
# Make a copy of the user profile object to ensure the spam checker
# cannot modify it.
if await maybe_awaitable(checker(user_profile.copy())):
return True
for callback in self._check_username_for_spam_callbacks:
# Make a copy of the user profile object to ensure the spam checker cannot
# modify it.
if await callback(user_profile.copy()):
return True
return False
@ -211,33 +365,13 @@ class SpamChecker:
Enum for how the request should be handled
"""
for spam_checker in self.spam_checkers:
# For backwards compatibility, only run if the method exists on the
# spam checker
checker = getattr(spam_checker, "check_registration_for_spam", None)
if checker:
# Provide auth_provider_id if the function supports it
checker_args = inspect.signature(checker)
if len(checker_args.parameters) == 4:
d = checker(
email_threepid,
username,
request_info,
auth_provider_id,
)
elif len(checker_args.parameters) == 3:
d = checker(email_threepid, username, request_info)
else:
logger.error(
"Invalid signature for %s.check_registration_for_spam. Denying registration",
spam_checker.__module__,
)
return RegistrationBehaviour.DENY
behaviour = await maybe_awaitable(d)
assert isinstance(behaviour, RegistrationBehaviour)
if behaviour != RegistrationBehaviour.ALLOW:
return behaviour
for callback in self._check_registration_for_spam_callbacks:
behaviour = await (
callback(email_threepid, username, request_info, auth_provider_id)
)
assert isinstance(behaviour, RegistrationBehaviour)
if behaviour != RegistrationBehaviour.ALLOW:
return behaviour
return RegistrationBehaviour.ALLOW
@ -275,13 +409,9 @@ class SpamChecker:
allowed.
"""
for spam_checker in self.spam_checkers:
# For backwards compatibility, only run if the method exists on the
# spam checker
checker = getattr(spam_checker, "check_media_file_for_spam", None)
if checker:
spam = await maybe_awaitable(checker(file_wrapper, file_info))
if spam:
return True
for callback in self._check_media_file_for_spam_callbacks:
spam = await callback(file_wrapper, file_info)
if spam:
return True
return False

View file

@ -195,7 +195,7 @@ class RegistrationHandler(BaseHandler):
bind_emails: list of emails to bind to this account.
by_admin: True if this registration is being made via the
admin api, otherwise False.
user_agent_ips: Tuples of IP addresses and user-agents used
user_agent_ips: Tuples of user-agents and IP addresses used
during the registration process.
auth_provider_id: The SSO IdP the user used, if any.
Returns:

View file

@ -16,6 +16,7 @@ import logging
from typing import TYPE_CHECKING, Any, Generator, Iterable, List, Optional, Tuple
from twisted.internet import defer
from twisted.web.resource import IResource
from synapse.events import EventBase
from synapse.http.client import SimpleHttpClient
@ -42,7 +43,7 @@ class ModuleApi:
can register new users etc if necessary.
"""
def __init__(self, hs, auth_handler):
def __init__(self, hs: "HomeServer", auth_handler):
self._hs = hs
self._store = hs.get_datastore()
@ -56,6 +57,33 @@ class ModuleApi:
self._http_client = hs.get_simple_http_client() # type: SimpleHttpClient
self._public_room_list_manager = PublicRoomListManager(hs)
self._spam_checker = hs.get_spam_checker()
#################################################################################
# The following methods should only be called during the module's initialisation.
@property
def register_spam_checker_callbacks(self):
"""Registers callbacks for spam checking capabilities."""
return self._spam_checker.register_callbacks
def register_web_resource(self, path: str, resource: IResource):
"""Registers a web resource to be served at the given path.
This function should be called during initialisation of the module.
If multiple modules register a resource for the same path, the module that
appears the highest in the configuration file takes priority.
Args:
path: The path to register the resource for.
resource: The resource to attach to this path.
"""
self._hs.register_module_web_resource(path, resource)
#########################################################################
# The following methods can be called by the module at any point in time.
@property
def http_client(self):
"""Allows making outbound HTTP requests to remote resources.

View file

@ -15,3 +15,4 @@
"""Exception types which are exposed as part of the stable module API"""
from synapse.api.errors import RedirectException, SynapseError # noqa: F401
from synapse.config._base import ConfigError # noqa: F401

View file

@ -1,6 +1,4 @@
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2017-2018 New Vector Ltd
# Copyright 2019 The Matrix.org Foundation C.I.C.
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -39,6 +37,7 @@ import twisted.internet.tcp
from twisted.internet import defer
from twisted.mail.smtp import sendmail
from twisted.web.iweb import IPolicyForHTTPS
from twisted.web.resource import IResource
from synapse.api.auth import Auth
from synapse.api.filtering import Filtering
@ -258,6 +257,38 @@ class HomeServer(metaclass=abc.ABCMeta):
self.datastores = None # type: Optional[Databases]
self._module_web_resources: Dict[str, IResource] = {}
self._module_web_resources_consumed = False
def register_module_web_resource(self, path: str, resource: IResource):
"""Allows a module to register a web resource to be served at the given path.
If multiple modules register a resource for the same path, the module that
appears the highest in the configuration file takes priority.
Args:
path: The path to register the resource for.
resource: The resource to attach to this path.
Raises:
SynapseError(500): A module tried to register a web resource after the HTTP
listeners have been started.
"""
if self._module_web_resources_consumed:
raise RuntimeError(
"Tried to register a web resource from a module after startup",
)
# Don't register a resource that's already been registered.
if path not in self._module_web_resources.keys():
self._module_web_resources[path] = resource
else:
logger.warning(
"Module tried to register a web resource for path %s but another module"
" has already registered a resource for this path.",
path,
)
def get_instance_id(self) -> str:
"""A unique ID for this synapse process instance.
@ -646,7 +677,7 @@ class HomeServer(metaclass=abc.ABCMeta):
@cache_in_self
def get_spam_checker(self) -> SpamChecker:
return SpamChecker(self)
return SpamChecker()
@cache_in_self
def get_third_party_event_rules(self) -> ThirdPartyEventRules:

View file

@ -51,21 +51,26 @@ def load_module(provider: dict, config_path: Iterable[str]) -> Tuple[Type, Any]:
# Load the module config. If None, pass an empty dictionary instead
module_config = provider.get("config") or {}
try:
provider_config = provider_class.parse_config(module_config)
except jsonschema.ValidationError as e:
raise json_error_to_config_error(e, itertools.chain(config_path, ("config",)))
except ConfigError as e:
raise _wrap_config_error(
"Failed to parse config for module %r" % (modulename,),
prefix=itertools.chain(config_path, ("config",)),
e=e,
)
except Exception as e:
raise ConfigError(
"Failed to parse config for module %r" % (modulename,),
path=itertools.chain(config_path, ("config",)),
) from e
if hasattr(provider_class, "parse_config"):
try:
provider_config = provider_class.parse_config(module_config)
except jsonschema.ValidationError as e:
raise json_error_to_config_error(
e, itertools.chain(config_path, ("config",))
)
except ConfigError as e:
raise _wrap_config_error(
"Failed to parse config for module %r" % (modulename,),
prefix=itertools.chain(config_path, ("config",)),
e=e,
)
except Exception as e:
raise ConfigError(
"Failed to parse config for module %r" % (modulename,),
path=itertools.chain(config_path, ("config",)),
) from e
else:
provider_config = module_config
return provider_class, provider_config