From f2449e2eba448b6973237ddb3ac6ae293e651613 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 28 Oct 2018 01:31:03 +0300 Subject: [PATCH] Start working on management API implementation --- example-config.yaml | 22 +++++--- maubot/__main__.py | 42 ++++++++++++--- maubot/client.py | 7 ++- maubot/config.py | 2 +- maubot/{plugin.py => instance.py} | 33 +++++++++--- maubot/loader/__init__.py | 2 +- maubot/loader/abc.py | 19 ++++++- maubot/loader/zip.py | 66 ++++++++++++++--------- maubot/management/api/__init__.py | 46 ++++++++++++++++ maubot/management/api/middleware.py | 67 +++++++++++++++++++++++ maubot/management/api/plugin.py | 83 +++++++++++++++++++++++++++++ maubot/management/api/responses.py | 38 +++++++++++++ maubot/plugin_base.py | 2 +- maubot/server.py | 4 +- 14 files changed, 379 insertions(+), 54 deletions(-) rename maubot/{plugin.py => instance.py} (86%) create mode 100644 maubot/management/api/middleware.py create mode 100644 maubot/management/api/plugin.py create mode 100644 maubot/management/api/responses.py diff --git a/example-config.yaml b/example-config.yaml index e76e430..9bcf2bf 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -5,24 +5,30 @@ # Postgres: postgres://username:password@hostname/dbname database: sqlite:///maubot.db -# The directory where plugin databases should be stored. -plugin_db_directory: ./plugins - -# If multiple directories have a plugin with the same name, the first directory is used. plugin_directories: -- ./plugins + # The directory where uploaded new plugins should be stored. + upload: ./plugins + # The directories from which plugins should be loaded. + # Duplicate plugin IDs will be moved to the trash. + load: + - ./plugins + # The directory where old plugin versions and conflicting plugins should be moved. + # Set to "delete" to delete files immediately. + trash: ./trash + # The directory where plugin databases should be stored. + db: ./plugins server: # The IP and port to listen to. hostname: 0.0.0.0 port: 29316 # The base management API path. - base_path: /_matrix/maubot + base_path: /_matrix/maubot/v1 # The base appservice API path. Use / for legacy appservice API and /_matrix/app/v1 for v1. appservice_base_path: /_matrix/app/v1 - # The shared secret to authorize users of the API. + # The shared secret to sign API access tokens. # Set to "generate" to generate and save a new token at startup. - shared_secret: generate + unshared_secret: generate admins: - "@admin:example.com" diff --git a/maubot/__main__.py b/maubot/__main__.py index fe9f168..3c2b42e 100644 --- a/maubot/__main__.py +++ b/maubot/__main__.py @@ -14,20 +14,23 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from sqlalchemy import orm +from time import time import sqlalchemy as sql import logging.config import argparse import asyncio +import signal import copy import sys -import signal +import os from .config import Config from .db import Base, init as init_db from .server import MaubotServer from .client import Client, init as init_client -from .loader import ZippedPluginLoader -from .plugin import PluginInstance, init as init_plugin_instance_class +from .loader import ZippedPluginLoader, MaubotZipImportError, IDConflictError +from .instance import PluginInstance, init as init_plugin_instance_class +from .management.api import init as init_management from .__meta__ import __version__ parser = argparse.ArgumentParser(description="A plugin-based Matrix bot system.", @@ -57,9 +60,36 @@ loop = asyncio.get_event_loop() init_db(db_session) init_client(loop) -init_plugin_instance_class(config) -server = MaubotServer(config, loop) -ZippedPluginLoader.load_all(*config["plugin_directories"]) +init_plugin_instance_class(db_session, config) +management_api = init_management(config, loop) +server = MaubotServer(config, management_api, loop) + +trash_path = config["plugin_directories.trash"] + + +def trash(file_path: str, new_name: Optional[str] = None) -> None: + if trash_path == "delete": + os.remove(file_path) + else: + new_name = new_name or f"{int(time())}-{os.path.basename(file_path)}" + os.rename(file_path, os.path.abspath(os.path.join(trash_path, new_name))) + + +ZippedPluginLoader.log.debug("Preloading plugins...") +for directory in config["plugin_directories.load"]: + for file in os.listdir(directory): + if not file.endswith(".mbp"): + continue + path = os.path.abspath(os.path.join(directory, file)) + try: + ZippedPluginLoader.get(path) + except MaubotZipImportError: + ZippedPluginLoader.log.exception(f"Failed to load plugin at {path}, trashing...") + trash(path) + except IDConflictError: + ZippedPluginLoader.log.warn(f"Duplicate plugin ID at {path}, trashing...") + trash(path) + plugins = PluginInstance.all() for plugin in plugins: diff --git a/maubot/client.py b/maubot/client.py index 807e7a2..e96c918 100644 --- a/maubot/client.py +++ b/maubot/client.py @@ -13,7 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Set, TYPE_CHECKING from aiohttp import ClientSession import asyncio import logging @@ -24,6 +24,9 @@ from mautrix.types import (UserID, SyncToken, FilterID, ContentURI, StrippedStat from .db import DBClient from .matrix import MaubotMatrixClient +if TYPE_CHECKING: + from .instance import PluginInstance + log = logging.getLogger("maubot.client") @@ -32,6 +35,7 @@ class Client: cache: Dict[UserID, 'Client'] = {} http_client: ClientSession = None + references: Set['PluginInstance'] db_instance: DBClient client: MaubotMatrixClient @@ -39,6 +43,7 @@ class Client: self.db_instance = db_instance self.cache[self.id] = self self.log = log.getChild(self.id) + self.references = set() self.client = MaubotMatrixClient(mxid=self.id, base_url=self.homeserver, token=self.access_token, client_session=self.http_client, log=self.log, loop=self.loop, store=self.db_instance) diff --git a/maubot/config.py b/maubot/config.py index 6858482..699003c 100644 --- a/maubot/config.py +++ b/maubot/config.py @@ -16,7 +16,7 @@ import random import string -from mautrix.util import BaseFileConfig, ConfigUpdateHelper +from mautrix.util.config import BaseFileConfig, ConfigUpdateHelper class Config(BaseFileConfig): diff --git a/maubot/plugin.py b/maubot/instance.py similarity index 86% rename from maubot/plugin.py rename to maubot/instance.py index 65875ef..0438713 100644 --- a/maubot/plugin.py +++ b/maubot/instance.py @@ -14,12 +14,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from typing import Dict, List, Optional +from sqlalchemy.orm import Session from ruamel.yaml.comments import CommentedMap from ruamel.yaml import YAML import logging import io -from mautrix.util import BaseProxyConfig, RecursiveDict +from mautrix.util.config import BaseProxyConfig, RecursiveDict from mautrix.types import UserID from .db import DBPlugin @@ -35,6 +36,7 @@ yaml.indent(4) class PluginInstance: + db: Session = None mb_config: Config = None cache: Dict[str, 'PluginInstance'] = {} plugin_directories: List[str] = [] @@ -44,13 +46,24 @@ class PluginInstance: client: Client plugin: Plugin config: BaseProxyConfig + running: bool def __init__(self, db_instance: DBPlugin): self.db_instance = db_instance self.log = logging.getLogger(f"maubot.plugin.{self.id}") self.config = None + self.running = False self.cache[self.id] = self + def to_dict(self) -> dict: + return { + "id": self.id, + "type": self.type, + "enabled": self.enabled, + "running": self.running, + "primary_user": self.primary_user, + } + def load(self) -> None: try: self.loader = PluginLoader.find(self.type) @@ -63,6 +76,13 @@ class PluginInstance: self.log.error(f"Failed to get client for user {self.primary_user}") self.enabled = False self.log.debug("Plugin instance dependencies loaded") + self.loader.references.add(self) + self.client.references.add(self) + + def delete(self) -> None: + self.loader.references.remove(self) + self.db.delete(self.db_instance) + # TODO delete plugin db def load_config(self) -> CommentedMap: return yaml.load(self.db_instance.config) @@ -90,14 +110,14 @@ class PluginInstance: self.save_config) self.plugin = cls(self.client.client, self.id, self.log, self.config, self.mb_config["plugin_db_directory"]) - self.loader.references |= {self} await self.plugin.start() + self.running = True self.log.info(f"Started instance of {self.loader.id} v{self.loader.version} " f"with user {self.client.id}") async def stop(self) -> None: self.log.debug("Stopping plugin instance...") - self.loader.references -= {self} + self.running = False await self.plugin.stop() self.plugin = None @@ -130,10 +150,6 @@ class PluginInstance: def type(self) -> str: return self.db_instance.type - @type.setter - def type(self, value: str) -> None: - self.db_instance.type = value - @property def enabled(self) -> bool: return self.db_instance.enabled @@ -153,5 +169,6 @@ class PluginInstance: # endregion -def init(config: Config): +def init(db: Session, config: Config): + PluginInstance.db = db PluginInstance.mb_config = config diff --git a/maubot/loader/__init__.py b/maubot/loader/__init__.py index 304cc57..e2a356e 100644 --- a/maubot/loader/__init__.py +++ b/maubot/loader/__init__.py @@ -1,2 +1,2 @@ -from .abc import PluginLoader, PluginClass +from .abc import PluginLoader, PluginClass, IDConflictError from .zip import ZippedPluginLoader, MaubotZipImportError diff --git a/maubot/loader/abc.py b/maubot/loader/abc.py index f41d848..6308a27 100644 --- a/maubot/loader/abc.py +++ b/maubot/loader/abc.py @@ -15,11 +15,12 @@ # along with this program. If not, see . from typing import TypeVar, Type, Dict, Set, TYPE_CHECKING from abc import ABC, abstractmethod +import asyncio from ..plugin_base import Plugin if TYPE_CHECKING: - from ..plugin import PluginInstance + from ..instance import PluginInstance PluginClass = TypeVar("PluginClass", bound=Plugin) @@ -42,6 +43,12 @@ class PluginLoader(ABC): def find(cls, plugin_id: str) -> 'PluginLoader': return cls.id_cache[plugin_id] + def to_dict(self) -> dict: + return { + "id": self.id, + "version": self.version, + } + @property @abstractmethod def source(self) -> str: @@ -51,6 +58,12 @@ class PluginLoader(ABC): def read_file(self, path: str) -> bytes: pass + async def stop_instances(self) -> None: + await asyncio.gather([instance.stop() for instance in self.references if instance.running]) + + async def start_instances(self) -> None: + await asyncio.gather([instance.start() for instance in self.references if instance.enabled]) + @abstractmethod def load(self) -> Type[PluginClass]: pass @@ -62,3 +75,7 @@ class PluginLoader(ABC): @abstractmethod def unload(self) -> None: pass + + @abstractmethod + def delete(self) -> None: + pass diff --git a/maubot/loader/zip.py b/maubot/loader/zip.py index a6a107e..7630c9e 100644 --- a/maubot/loader/zip.py +++ b/maubot/loader/zip.py @@ -13,7 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, List, Type +from typing import Dict, List, Type, Tuple from zipfile import ZipFile, BadZipFile import configparser import logging @@ -60,8 +60,15 @@ class ZippedPluginLoader(PluginLoader): self.id_cache[self.id] = self self.log.debug(f"Preloaded plugin {self.id} from {self.path}") + def to_dict(self) -> dict: + return { + **super().to_dict(), + "path": self.path + } + @classmethod def get(cls, path: str) -> 'ZippedPluginLoader': + path = os.path.abspath(path) try: return cls.path_cache[path] except KeyError: @@ -80,10 +87,11 @@ class ZippedPluginLoader(PluginLoader): def read_file(self, path: str) -> bytes: return self._file.read(path) - def _load_meta(self) -> None: + @staticmethod + def _open_meta(source) -> Tuple[ZipFile, configparser.ConfigParser]: try: - self._file = ZipFile(self.path) - data = self._file.read("maubot.ini") + file = ZipFile(source) + data = file.read("maubot.ini") except FileNotFoundError as e: raise MaubotZipImportError("Maubot plugin not found") from e except BadZipFile as e: @@ -92,7 +100,14 @@ class ZippedPluginLoader(PluginLoader): raise MaubotZipImportError("File does not contain a maubot plugin definition") from e config = configparser.ConfigParser() try: - config.read_string(data.decode("utf-8"), source=f"{self.path}/maubot.ini") + config.read_string(data.decode("utf-8")) + except (configparser.Error, KeyError, IndexError, ValueError) as e: + raise MaubotZipImportError("Maubot plugin definition in file is invalid") from e + return file, config + + @classmethod + def _read_meta(cls, config: configparser.ConfigParser) -> Tuple[str, str, List[str], str, str]: + try: meta = config["maubot"] meta_id = meta["ID"] version = meta["Version"] @@ -103,10 +118,21 @@ class ZippedPluginLoader(PluginLoader): main_module, main_class = main_class.split("/")[:2] except (configparser.Error, KeyError, IndexError, ValueError) as e: raise MaubotZipImportError("Maubot plugin definition in file is invalid") from e - if self.id and meta_id != self.id: + return meta_id, version, modules, main_class, main_module + + @classmethod + def verify_meta(cls, source) -> Tuple[str, str]: + _, config = cls._open_meta(source) + meta = cls._read_meta(config) + return meta[0], meta[1] + + def _load_meta(self) -> None: + file, config = self._open_meta(self.path) + meta = self._read_meta(config) + if self.id and meta[0] != self.id: raise MaubotZipImportError("Maubot plugin ID changed during reload") - self.id, self.version, self.modules = meta_id, version, modules - self.main_class, self.main_module = main_class, main_module + self.id, self.version, self.modules, self.main_class, self.main_module = meta + self._file = file def _get_importer(self, reset_cache: bool = False) -> zipimporter: try: @@ -161,7 +187,7 @@ class ZippedPluginLoader(PluginLoader): self._loaded = None self.log.debug(f"Unloaded plugin {self.id} at {self.path}") - def destroy(self) -> None: + def delete(self) -> None: self.unload() try: del self.path_cache[self.path] @@ -171,24 +197,12 @@ class ZippedPluginLoader(PluginLoader): del self.id_cache[self.id] except KeyError: pass - self.id = None - self.path = None - self.version = None - self.modules = None if self._importer: self._importer.remove_cache() self._importer = None self._loaded = None - - @classmethod - def load_all(cls, *args: str) -> None: - cls.log.debug("Preloading plugins...") - for directory in args: - for file in os.listdir(directory): - if not file.endswith(".mbp"): - continue - path = os.path.join(directory, file) - try: - ZippedPluginLoader.get(path) - except (MaubotZipImportError, IDConflictError): - cls.log.exception(f"Failed to load plugin at {path}") + os.remove(self.path) + self.id = None + self.path = None + self.version = None + self.modules = None diff --git a/maubot/management/api/__init__.py b/maubot/management/api/__init__.py index e69de29..dc08a5f 100644 --- a/maubot/management/api/__init__.py +++ b/maubot/management/api/__init__.py @@ -0,0 +1,46 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from aiohttp import web +from asyncio import AbstractEventLoop + +from mautrix.types import UserID +from mautrix.util.signed_token import sign_token, verify_token + +from ...config import Config + +routes = web.RouteTableDef() +config: Config = None + + +def is_valid_token(token: str) -> bool: + data = verify_token(config["server.unshared_secret"], token) + user_id = data.get("user_id", None) + return user_id is not None and user_id in config["admins"] + + +def create_token(user: UserID) -> str: + return sign_token(config["server.unshared_secret"], { + "user_id": user, + }) + + +def init(cfg: Config, loop: AbstractEventLoop) -> web.Application: + global config + config = cfg + from .middleware import auth, error, log + app = web.Application(loop=loop, middlewares=[auth, log, error]) + app.add_routes(routes) + return app diff --git a/maubot/management/api/middleware.py b/maubot/management/api/middleware.py new file mode 100644 index 0000000..f1b76ad --- /dev/null +++ b/maubot/management/api/middleware.py @@ -0,0 +1,67 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from typing import Callable, Awaitable +from aiohttp import web +import logging + +from .responses import ErrNoToken, ErrInvalidToken +from . import is_valid_token + +Handler = Callable[[web.Request], Awaitable[web.Response]] + +req_log = logging.getLogger("maubot.mgmt.request") +resp_log = logging.getLogger("maubot.mgmt.response") + + +@web.middleware +async def auth(request: web.Request, handler: Handler) -> web.Response: + token = request.headers.get("Authorization", "") + if not token or not token.startswith("Bearer "): + req_log.debug(f"Request missing auth: {request.remote} {request.method} {request.path}") + return ErrNoToken + if not is_valid_token(token[len("Bearer "):]): + req_log.debug(f"Request invalid auth: {request.remote} {request.method} {request.path}") + return ErrInvalidToken + return await handler(request) + + +@web.middleware +async def error(request: web.Request, handler: Handler) -> web.Response: + try: + return await handler(request) + except web.HTTPException as ex: + return web.json_response({ + "error": f"Unhandled HTTP {ex.status}", + "errcode": f"unhandled_http_{ex.status}", + }, status=ex.status) + + +req_no = 0 + + +def get_req_no(): + global req_no + req_no += 1 + return req_no + + +@web.middleware +async def log(request: web.Request, handler: Handler) -> web.Response: + local_req_no = get_req_no() + req_log.info(f"Request {local_req_no}: {request.remote} {request.method} {request.path}") + resp = await handler(request) + resp_log.info(f"Responded to {local_req_no} from {request.remote}: {resp}") + return resp diff --git a/maubot/management/api/plugin.py b/maubot/management/api/plugin.py new file mode 100644 index 0000000..c85f88b --- /dev/null +++ b/maubot/management/api/plugin.py @@ -0,0 +1,83 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from aiohttp import web +from io import BytesIO +import os.path + +from ...loader import PluginLoader, ZippedPluginLoader, MaubotZipImportError +from .responses import ErrPluginNotFound, ErrPluginInUse, RespDeleted +from . import routes, config + + +def _plugin_to_dict(plugin: PluginLoader) -> dict: + return { + **plugin.to_dict(), + "instances": [instance.to_dict() for instance in plugin.references] + } + + +@routes.get("/plugins") +async def get_plugins(_) -> web.Response: + return web.json_response([_plugin_to_dict(plugin) for plugin in PluginLoader.id_cache.values()]) + + +@routes.get("/plugin/{id}") +async def get_plugin(request: web.Request) -> web.Response: + plugin_id = request.match_info.get("id", None) + plugin = PluginLoader.id_cache.get(plugin_id, None) + if not plugin: + return ErrPluginNotFound + return web.json_response(_plugin_to_dict(plugin)) + + +@routes.delete("/plugin/{id}") +async def delete_plugin(request: web.Request) -> web.Response: + plugin_id = request.match_info.get("id", None) + plugin = PluginLoader.id_cache.get(plugin_id, None) + if not plugin: + return ErrPluginNotFound + elif len(plugin.references) > 0: + return ErrPluginInUse + plugin.delete() + return RespDeleted + + +@routes.post("/plugins/upload") +async def upload_plugin(request: web.Request) -> web.Response: + content = await request.read() + file = BytesIO(content) + try: + pid, version = ZippedPluginLoader.verify_meta(file) + except MaubotZipImportError as e: + return web.json_response({ + "error": str(e), + "errcode": "invalid_plugin", + }, status=web.HTTPBadRequest) + plugin = PluginLoader.id_cache.get(pid, None) + if not plugin: + path = os.path.join(config["plugin_directories.upload"], f"{pid}-{version}.mbp") + with open(path, "wb") as p: + p.write(content) + try: + ZippedPluginLoader.get(path) + except MaubotZipImportError as e: + trash(path) + return web.json_response({ + "error": str(e), + "errcode": "invalid_plugin", + }, status=web.HTTPBadRequest) + else: + pass diff --git a/maubot/management/api/responses.py b/maubot/management/api/responses.py new file mode 100644 index 0000000..bab2b01 --- /dev/null +++ b/maubot/management/api/responses.py @@ -0,0 +1,38 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from aiohttp import web + +ErrNoToken = web.json_response({ + "error": "Authorization token missing", + "errcode": "auth_token_missing", +}, status=web.HTTPUnauthorized) + +ErrInvalidToken = web.json_response({ + "error": "Invalid authorization token", + "errcode": "auth_token_invalid", +}, status=web.HTTPUnauthorized) + +ErrPluginNotFound = web.json_response({ + "error": "Plugin not found", + "errcode": "plugin_not_found", +}, status=web.HTTPNotFound) + +ErrPluginInUse = web.json_response({ + "error": "Plugin instances of this type still exist", + "errcode": "plugin_in_use", +}) + +RespDeleted = web.Response(status=204) diff --git a/maubot/plugin_base.py b/maubot/plugin_base.py index b28ab4a..08d1211 100644 --- a/maubot/plugin_base.py +++ b/maubot/plugin_base.py @@ -24,7 +24,7 @@ import sqlalchemy as sql if TYPE_CHECKING: from .client import MaubotMatrixClient from .command_spec import CommandSpec - from mautrix.util import BaseProxyConfig + from mautrix.util.config import BaseProxyConfig class Plugin(ABC): diff --git a/maubot/server.py b/maubot/server.py index 7a666ec..4bd7bd2 100644 --- a/maubot/server.py +++ b/maubot/server.py @@ -23,13 +23,15 @@ from .__meta__ import __version__ class MaubotServer: - def __init__(self, config: Config, loop: asyncio.AbstractEventLoop): + def __init__(self, config: Config, management: web.Application, + loop: asyncio.AbstractEventLoop) -> None: self.loop = loop or asyncio.get_event_loop() self.app = web.Application(loop=self.loop) self.config = config path = PathBuilder(config["server.base_path"]) self.add_route(Method.GET, path.version, self.version) + self.app.add_subapp(config["server.base_path"], management) as_path = PathBuilder(config["server.appservice_base_path"]) self.add_route(Method.PUT, as_path.transactions, self.handle_transaction)