Merge f863e7cefb into 65be63fdd2

Previously, the webapp handler would match without respect to the trailing slash, e.g. matching "foo"
for "foo2". This behavior is changed to respect the trailing slash.

Fixes #239

.github/workflows/python-lint.yml

@@ -6,16 +6,17 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v3
+    - uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
     - uses: isort/isort-action@master
       with:
         sortPaths: "./maubot"
     - uses: psf/black@stable
       with:
         src: "./maubot"
+        version: "24.2.0"
     - name: pre-commit
       run: |
         pip install pre-commit

.gitignore

@@ -13,6 +13,7 @@ __pycache__
 !example-config.yaml
 !.pre-commit-config.yaml
 
+/start
 logs/
 plugins/
 trash/

.gitlab-ci.yml

@@ -10,7 +10,7 @@ default:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 build frontend:
-  image: node:18-alpine
+  image: node:20-alpine
   stage: build frontend
   before_script: []
   variables:

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
         exclude_types: [markdown]
@@ -8,13 +8,13 @@ repos:
       - id: check-yaml
       - id: check-added-large-files
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 24.2.0
     hooks:
       - id: black
         language_version: python3
         files: ^maubot/.*\.pyi?$
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         files: ^maubot/.*\.pyi?$

CHANGELOG.md

@@ -1,3 +1,33 @@
+# v0.5.0 (2024-08-24)
+
+* Dropped Python 3.9 support.
+* Updated Docker image to Alpine 3.20.
+* Updated mautrix-python to 0.20.6 to support authenticated media.
+* Removed hard dependency on SQLAlchemy.
+* Fixed `main_class` to default to being loaded from the last module instead of
+  the first if a module name is not explicitly specified.
+  * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html),
+    and loading from the first module doesn't make sense due to import order.
+* Added simple scheduler utility for running background tasks periodically or
+  after a certain delay.
+* Added testing framework for plugins (thanks to [@abompard] in [#225]).
+* Changed `mbc build` to ignore directories declared in `modules` that are
+  missing an `__init__.py` file.
+  * Importing the modules at runtime would fail and break the plugin.
+    To include non-code resources outside modules in the mbp archive,
+    use `extra_files` instead.
+
+[#225]: https://github.com/maubot/maubot/issues/225
+[@abompard]: https://github.com/abompard
+
+# v0.4.2 (2023-09-20)
+
+* Updated Pillow to 10.0.1.
+* Updated Docker image to Alpine 3.18.
+* Added logging for errors for /whoami errors when adding new bot accounts.
+* Added support for using appservice tokens (including appservice encryption)
+  in standalone mode.
+
 # v0.4.1 (2023-03-15)
 
 * Added `in_thread` parameter to `evt.reply()` and `evt.respond()`.

Dockerfile

@@ -1,9 +1,9 @@
-FROM node:18 AS frontend-builder
+FROM node:20 AS frontend-builder
 
 COPY ./maubot/management/frontend /frontend
 RUN cd /frontend && yarn --prod && yarn build
 
-FROM alpine:3.18
+FROM alpine:3.20
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -11,7 +11,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -47,7 +46,7 @@ COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery tzlocal \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM alpine:3.18
+FROM alpine:3.20
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -6,7 +6,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -42,7 +41,7 @@ COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

README.md

@@ -22,7 +22,7 @@ All setup and usage instructions are located on
 Matrix room: [#maubot:maunium.net](https://matrix.to/#/#maubot:maunium.net)
 
 ## Plugins
-A list of plugins can be found at [plugins.maubot.xyz](https://plugins.maubot.xyz/).
+A list of plugins can be found at [plugins.mau.bot](https://plugins.mau.bot/).
 
 To add your plugin to the list, send a pull request to <https://github.com/maubot/plugins.maubot.xyz>.
 

dev-requirements.txt

@@ -1,3 +1,3 @@
 pre-commit>=2.10.1,<3
 isort>=5.10.1,<6
-black>=23,<24
+black>=24,<25

maubot/__meta__.py

@@ -1 +1 @@
-__version__ = "0.4.1"
+__version__ = "0.5.0"

maubot/cli/commands/build.py

@@ -93,10 +93,16 @@ def write_plugin(meta: PluginMeta, output: str | IO) -> None:
             if os.path.isfile(f"{module}.py"):
                 zip.write(f"{module}.py")
             elif module is not None and os.path.isdir(module):
-                zipdir(zip, module)
+                if os.path.isfile(f"{module}/__init__.py"):
+                    zipdir(zip, module)
+                else:
+                    print(
+                        Fore.YELLOW
+                        + f"Module {module} is missing __init__.py, skipping"
+                        + Fore.RESET
+                    )
             else:
                 print(Fore.YELLOW + f"Module {module} not found, skipping" + Fore.RESET)
-
         for pattern in meta.extra_files:
             for file in glob.iglob(pattern):
                 zip.write(file)

maubot/instance.py

@@ -25,7 +25,6 @@ import os.path
 
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-import sqlalchemy as sql
 
 from mautrix.types import UserID
 from mautrix.util import background_task
@@ -36,6 +35,7 @@ from mautrix.util.logging import TraceLogger
 
 from .client import Client
 from .db import DatabaseEngine, Instance as DBInstance
+from .lib.optionalalchemy import Engine, MetaData, create_engine
 from .lib.plugin_db import ProxyPostgresDatabase
 from .loader import DatabaseType, PluginLoader, ZippedPluginLoader
 from .plugin_base import Plugin
@@ -128,7 +128,7 @@ class PluginInstance(DBInstance):
         }
 
     def _introspect_sqlalchemy(self) -> dict:
-        metadata = sql.MetaData()
+        metadata = MetaData()
         metadata.reflect(self.inst_db)
         return {
             table.name: {
@@ -214,7 +214,7 @@ class PluginInstance(DBInstance):
 
     async def get_db_tables(self) -> dict:
         if self.inst_db_tables is None:
-            if isinstance(self.inst_db, sql.engine.Engine):
+            if isinstance(self.inst_db, Engine):
                 self.inst_db_tables = self._introspect_sqlalchemy()
             elif self.inst_db.scheme == Scheme.SQLITE:
                 self.inst_db_tables = await self._introspect_sqlite()
@@ -294,7 +294,7 @@ class PluginInstance(DBInstance):
                     "Instance database engine is marked as Postgres, but plugin uses legacy "
                     "database interface, which doesn't support postgres."
                 )
-            self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}")
+            self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}")
         elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
             if self.database_engine is None:
                 if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db:
@@ -329,7 +329,7 @@ class PluginInstance(DBInstance):
     async def stop_database(self) -> None:
         if isinstance(self.inst_db, Database):
             await self.inst_db.stop()
-        elif isinstance(self.inst_db, sql.engine.Engine):
+        elif isinstance(self.inst_db, Engine):
             self.inst_db.dispose()
         else:
             raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}")

maubot/lib/optionalalchemy.py

@@ -0,0 +1,19 @@
+try:
+    from sqlalchemy import MetaData, asc, create_engine, desc
+    from sqlalchemy.engine import Engine
+    from sqlalchemy.exc import IntegrityError, OperationalError
+except ImportError:
+
+    class FakeError(Exception):
+        pass
+
+    class FakeType:
+        def __init__(self, *args, **kwargs):
+            raise Exception("SQLAlchemy is not installed")
+
+    def create_engine(*args, **kwargs):
+        raise Exception("SQLAlchemy is not installed")
+
+    MetaData = Engine = FakeType
+    IntegrityError = OperationalError = FakeError
+    asc = desc = lambda a: a

maubot/loader/zip.py

@@ -31,7 +31,7 @@ from ..config import Config
 from ..lib.zipimport import ZipImportError, zipimporter
 from ..plugin_base import Plugin
 from .abc import IDConflictError, PluginClass, PluginLoader
-from .meta import PluginMeta
+from .meta import DatabaseType, PluginMeta
 
 current_version = Version(__version__)
 yaml = YAML()
@@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader):
         return file, meta
 
     @classmethod
-    def verify_meta(cls, source) -> tuple[str, Version]:
+    def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]:
         _, meta = cls._read_meta(source)
-        return meta.id, meta.version
+        return meta.id, meta.version, meta.database_type if meta.database else None
 
     def _load_meta(self) -> None:
         file, meta = self._read_meta(self.path)
@@ -167,7 +167,7 @@ class ZippedPluginLoader(PluginLoader):
         if "/" in meta.main_class:
             self.main_module, self.main_class = meta.main_class.split("/")[:2]
         else:
-            self.main_module = meta.modules[0]
+            self.main_module = meta.modules[-1]
             self.main_class = meta.main_class
         self._file = file
 

maubot/management/api/instance_database.py

@@ -19,12 +19,12 @@ from datetime import datetime
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy import asc, desc, engine, exc
 import aiosqlite
 
 from mautrix.util.async_db import Database
 
 from ...instance import PluginInstance
+from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc
 from .base import routes
 from .responses import resp
 
@@ -56,15 +56,17 @@ async def get_table(request: web.Request) -> web.Response:
     try:
         order = [tuple(order.split(":")) for order in request.query.getall("order")]
         order = [
-            (asc if sort.lower() == "asc" else desc)(table.columns[column])
+            (
-            if sort
+                (asc if sort.lower() == "asc" else desc)(table.columns[column])
-            else table.columns[column]
+                if sort
+                else table.columns[column]
+            )
             for column, sort in order
         ]
     except KeyError:
         order = []
     limit = int(request.query.get("limit", "100"))
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit))
 
 
@@ -82,7 +84,7 @@ async def query(request: web.Request) -> web.Response:
     except KeyError:
         return resp.query_missing
     rows_as_dict = data.get("rows_as_dict", False)
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict)
     elif isinstance(instance.inst_db, Database):
         try:
@@ -131,12 +133,12 @@ async def _execute_query_asyncpg(
 def _execute_query_sqlalchemy(
     instance: PluginInstance, sql_query: str, rows_as_dict: bool = False
 ) -> web.Response:
-    assert isinstance(instance.inst_db, engine.Engine)
+    assert isinstance(instance.inst_db, Engine)
     try:
         res = instance.inst_db.execute(sql_query)
-    except exc.IntegrityError as e:
+    except IntegrityError as e:
         return resp.sql_integrity_error(e, sql_query)
-    except exc.OperationalError as e:
+    except OperationalError as e:
         return resp.sql_operational_error(e, sql_query)
     data = {
         "ok": True,

maubot/management/api/plugin_upload.py

@@ -23,10 +23,17 @@ import traceback
 from aiohttp import web
 from packaging.version import Version
 
-from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader
+from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader
 from .base import get_config, routes
 from .responses import resp
 
+try:
+    import sqlalchemy
+
+    has_alchemy = True
+except ImportError:
+    has_alchemy = False
+
 log = logging.getLogger("maubot.server.upload")
 
 
@@ -36,9 +43,11 @@ async def put_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     if pid != plugin_id:
         return resp.pid_mismatch
     plugin = PluginLoader.id_cache.get(plugin_id, None)
@@ -55,9 +64,11 @@ async def upload_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     plugin = PluginLoader.id_cache.get(pid, None)
     if not plugin:
         return await upload_new_plugin(content, pid, version)

maubot/management/api/responses.py

@@ -15,13 +15,16 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
 from http import HTTPStatus
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy.exc import IntegrityError, OperationalError
 import aiosqlite
 
+if TYPE_CHECKING:
+    from sqlalchemy.exc import IntegrityError, OperationalError
+
 
 class _Response:
     @property
@@ -324,6 +327,16 @@ class _Response:
             }
         )
 
+    @property
+    def sqlalchemy_not_installed(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "This plugin requires a legacy database, but SQLAlchemy is not installed",
+                "errcode": "unsupported_plugin_database",
+            },
+            status=HTTPStatus.NOT_IMPLEMENTED,
+        )
+
     @property
     def table_not_found(self) -> web.Response:
         return web.json_response(

maubot/plugin_base.py

@@ -20,14 +20,17 @@ from abc import ABC
 from asyncio import AbstractEventLoop
 
 from aiohttp import ClientSession
-from sqlalchemy.engine.base import Engine
 from yarl import URL
 
 from mautrix.util.async_db import Database, UpgradeTable
 from mautrix.util.config import BaseProxyConfig
 from mautrix.util.logging import TraceLogger
 
+from .scheduler import BasicScheduler
+
 if TYPE_CHECKING:
+    from sqlalchemy.engine.base import Engine
+
     from .client import MaubotMatrixClient
     from .loader import BasePluginLoader
     from .plugin_server import PluginWebApp
@@ -40,6 +43,7 @@ class Plugin(ABC):
     log: TraceLogger
     loop: AbstractEventLoop
     loader: BasePluginLoader
+    sched: BasicScheduler
     config: BaseProxyConfig | None
     database: Engine | Database | None
     webapp: PluginWebApp | None
@@ -53,11 +57,12 @@ class Plugin(ABC):
         instance_id: str,
         log: TraceLogger,
         config: BaseProxyConfig | None,
-        database: Engine | None,
+        database: Engine | Database | None,
         webapp: PluginWebApp | None,
         webapp_url: str | None,
         loader: BasePluginLoader,
     ) -> None:
+        self.sched = BasicScheduler(log=log.getChild("scheduler"))
         self.client = client
         self.loop = loop
         self.http = http
@@ -117,6 +122,7 @@ class Plugin(ABC):
             self.client.remove_event_handler(event_type, func)
         if self.webapp is not None:
             self.webapp.clear()
+        self.sched.stop()
         await self.stop()
 
     async def stop(self) -> None:

maubot/scheduler.py

@@ -0,0 +1,159 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2024 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Awaitable, Callable
+import asyncio
+import logging
+
+
+class BasicScheduler:
+    background_loop: asyncio.Task | None
+    tasks: set[asyncio.Task]
+    log: logging.Logger
+
+    def __init__(self, log: logging.Logger) -> None:
+        self.log = log
+        self.tasks = set()
+
+    def _find_caller(self) -> str:
+        try:
+            file_name, line_number, function_name, _ = self.log.findCaller()
+            return f"{function_name} at {file_name}:{line_number}"
+        except ValueError:
+            return "unknown function"
+
+    def run_periodically(
+        self,
+        period: float | int,
+        func: Callable[[], Awaitable],
+        run_task_in_background: bool = False,
+        catch_errors: bool = True,
+    ) -> asyncio.Task:
+        """
+        Run a function periodically in the background.
+
+        Args:
+            period: The period in seconds between each call to the function.
+            func: The function to run. No parameters will be provided,
+                use :meth:`functools.partial` if you need to pass parameters.
+            run_task_in_background: If ``True``, the function will be run in a background task.
+                If ``False`` (the default), the loop will wait for the task to return before
+                sleeping for the next period.
+            catch_errors: Whether the scheduler should catch and log any errors.
+                If ``False``, errors will be raised, and the caller must await the returned task
+                to find errors. This parameter has no effect if ``run_task_in_background``
+                is ``True``.
+
+        Returns:
+            The asyncio task object representing the background loop.
+        """
+        task = asyncio.create_task(
+            self._call_periodically(
+                period,
+                func,
+                caller=self._find_caller(),
+                catch_errors=catch_errors,
+                run_task_in_background=run_task_in_background,
+            )
+        )
+        self._register_task(task)
+        return task
+
+    def run_later(
+        self, delay: float | int, coro: Awaitable, catch_errors: bool = True
+    ) -> asyncio.Task:
+        """
+        Run a coroutine after a delay.
+
+        Examples:
+            >>> self.sched.run_later(5, self.async_task(meow=True))
+
+        Args:
+            delay: The delay in seconds to await the coroutine after.
+            coro: The coroutine to await.
+            catch_errors: Whether the scheduler should catch and log any errors.
+                If ``False``, errors will be raised, and the caller must await the returned task
+                to find errors.
+
+        Returns:
+            The asyncio task object representing the scheduled task.
+        """
+        task = asyncio.create_task(
+            self._call_with_delay(
+                delay, coro, caller=self._find_caller(), catch_errors=catch_errors
+            )
+        )
+        self._register_task(task)
+        return task
+
+    def _register_task(self, task: asyncio.Task) -> None:
+        self.tasks.add(task)
+        task.add_done_callback(self.tasks.discard)
+
+    async def _call_periodically(
+        self,
+        period: float | int,
+        func: Callable[[], Awaitable],
+        caller: str,
+        catch_errors: bool,
+        run_task_in_background: bool,
+    ) -> None:
+        while True:
+            try:
+                await asyncio.sleep(period)
+                if run_task_in_background:
+                    self._register_task(
+                        asyncio.create_task(self._call_periodically_background(func(), caller))
+                    )
+                else:
+                    await func()
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                if catch_errors:
+                    self.log.exception(f"Uncaught error in background loop (created in {caller})")
+                else:
+                    raise
+
+    async def _call_periodically_background(self, coro: Awaitable, caller: str) -> None:
+        try:
+            await coro
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            self.log.exception(f"Uncaught error in background loop subtask (created in {caller})")
+
+    async def _call_with_delay(
+        self, delay: float | int, coro: Awaitable, caller: str, catch_errors: bool
+    ) -> None:
+        try:
+            await asyncio.sleep(delay)
+            await coro
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            if catch_errors:
+                self.log.exception(f"Uncaught error in scheduled task (created in {caller})")
+            else:
+                raise
+
+    def stop(self) -> None:
+        """
+        Stop all scheduled tasks and background loops.
+        """
+        for task in self.tasks:
+            task.cancel(msg="Scheduler stopped")

maubot/server.py

@@ -64,14 +64,14 @@ class MaubotServer:
             if request.path.startswith(path):
                 request = request.clone(
                     rel_url=request.rel_url.with_path(
-                        request.rel_url.path[len(path) :]
+                        request.rel_url.path[len(path) - 1 :]
                     ).with_query(request.query_string)
                 )
                 return await app.handle(request)
         return web.Response(status=404)
 
     def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]:
-        subpath = self.config["server.plugin_base_path"] + instance_id
+        subpath = self.config["server.plugin_base_path"] + instance_id + "/"
         url = self.config["server.public_url"] + subpath
         try:
             return self.plugin_routes[subpath], url
@@ -82,7 +82,7 @@ class MaubotServer:
 
     def remove_instance_webapp(self, instance_id: str) -> None:
         try:
-            subpath = self.config["server.plugin_base_path"] + instance_id
+            subpath = self.config["server.plugin_base_path"] + instance_id + "/"
             self.plugin_routes.pop(subpath).clear()
         except KeyError:
             return

maubot/standalone/Dockerfile

@@ -1,9 +1,8 @@
-FROM docker.io/alpine:3.18
+FROM docker.io/alpine:3.20
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-aiohttp \
-      py3-sqlalchemy \
       py3-attrs \
       py3-bcrypt \
       py3-cffi \
@@ -26,8 +25,8 @@ RUN cd /opt/maubot \
       python3-dev \
       libffi-dev \
       build-base \
-  && pip3 install -r requirements.txt -r optional-requirements.txt \
+  && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
   && apk del .build-deps
 
 COPY . /opt/maubot
-RUN cd /opt/maubot && pip3 install .
+RUN cd /opt/maubot && pip3 install --break-system-packages .

maubot/standalone/__main__.py

@@ -115,7 +115,7 @@ with open(args.meta, "r") as meta_file:
 if "/" in meta.main_class:
     module, main_class = meta.main_class.split("/", 1)
 else:
-    module = meta.modules[0]
+    module = meta.modules[-1]
     main_class = meta.main_class
 
 if args.meta != "maubot.yaml" and os.path.dirname(args.meta) != "":

maubot/testing/__init__.py

@@ -0,0 +1,17 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from .bot import TestBot, make_message  # noqa: F401
+from .fixtures import *  # noqa: F401,F403

maubot/testing/bot.py

@@ -0,0 +1,100 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import asyncio
+import time
+
+from attr import dataclass
+
+from maubot.matrix import MaubotMatrixClient, MaubotMessageEvent
+from mautrix.api import HTTPAPI
+from mautrix.types import (
+    EventContent,
+    EventType,
+    MessageEvent,
+    MessageType,
+    RoomID,
+    TextMessageEventContent,
+)
+
+
+@dataclass
+class MatrixEvent:
+    room_id: RoomID
+    event_type: EventType
+    content: EventContent
+    kwargs: dict
+
+
+class TestBot:
+    """A mocked bot used for testing purposes.
+
+    Send messages to the mock Matrix server with the ``send()`` method.
+    Look into the ``responded`` list to get what server has replied.
+    """
+
+    def __init__(self, mxid="@botname:example.com", mxurl="http://matrix.example.com"):
+        api = HTTPAPI(base_url=mxurl)
+        self.client = MaubotMatrixClient(api=api)
+        self.responded = []
+        self.client.mxid = mxid
+        self.client.send_message_event = self._mock_send_message_event
+
+    async def _mock_send_message_event(self, room_id, event_type, content, txn_id=None, **kwargs):
+        self.responded.append(
+            MatrixEvent(room_id=room_id, event_type=event_type, content=content, kwargs=kwargs)
+        )
+
+    async def dispatch(self, event_type: EventType, event):
+        tasks = self.client.dispatch_manual_event(event_type, event, force_synchronous=True)
+        return await asyncio.gather(*tasks)
+
+    async def send(
+        self,
+        content,
+        html=None,
+        room_id="testroom",
+        msg_type=MessageType.TEXT,
+        sender="@dummy:example.com",
+        timestamp=None,
+    ):
+        event = make_message(
+            content,
+            html=html,
+            room_id=room_id,
+            msg_type=msg_type,
+            sender=sender,
+            timestamp=timestamp,
+        )
+        await self.dispatch(EventType.ROOM_MESSAGE, MaubotMessageEvent(event, self.client))
+
+
+def make_message(
+    content,
+    html=None,
+    room_id="testroom",
+    msg_type=MessageType.TEXT,
+    sender="@dummy:example.com",
+    timestamp=None,
+):
+    """Make a Matrix message event."""
+    return MessageEvent(
+        type=EventType.ROOM_MESSAGE,
+        room_id=room_id,
+        event_id="test",
+        sender=sender,
+        timestamp=timestamp or int(time.time() * 1000),
+        content=TextMessageEventContent(msgtype=msg_type, body=content, formatted_body=html),
+    )

maubot/testing/fixtures.py

@@ -0,0 +1,135 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from pathlib import Path
+import asyncio
+import logging
+
+from ruamel.yaml import YAML
+import aiohttp
+import pytest
+import pytest_asyncio
+
+from maubot import Plugin
+from maubot.loader import PluginMeta
+from maubot.standalone.loader import FileSystemLoader
+from mautrix.util.async_db import Database
+from mautrix.util.config import BaseProxyConfig, RecursiveDict
+from mautrix.util.logging import TraceLogger
+
+from .bot import TestBot
+
+
+@pytest_asyncio.fixture
+async def maubot_test_bot():
+    return TestBot()
+
+
+@pytest.fixture
+def maubot_upgrade_table():
+    return None
+
+
+@pytest.fixture
+def maubot_plugin_path():
+    return Path(".")
+
+
+@pytest.fixture
+def maubot_plugin_meta(maubot_plugin_path):
+    yaml = YAML()
+    with open(maubot_plugin_path.joinpath("maubot.yaml")) as fh:
+        plugin_meta = PluginMeta.deserialize(yaml.load(fh.read()))
+    return plugin_meta
+
+
+@pytest_asyncio.fixture
+async def maubot_plugin_db(tmp_path, maubot_plugin_meta, maubot_upgrade_table):
+    if not maubot_plugin_meta.get("database", False):
+        return
+    db_path = tmp_path.joinpath("maubot-tests.db").as_posix()
+    db = Database.create(
+        f"sqlite:{db_path}",
+        upgrade_table=maubot_upgrade_table,
+        log=logging.getLogger("db"),
+    )
+    await db.start()
+    yield db
+    await db.stop()
+
+
+@pytest.fixture
+def maubot_plugin_class():
+    return Plugin
+
+
+@pytest.fixture
+def maubot_plugin_config_class():
+    return BaseProxyConfig
+
+
+@pytest.fixture
+def maubot_plugin_config_dict():
+    return {}
+
+
+@pytest.fixture
+def maubot_plugin_config_overrides():
+    return {}
+
+
+@pytest.fixture
+def maubot_plugin_config(
+    maubot_plugin_path,
+    maubot_plugin_config_class,
+    maubot_plugin_config_dict,
+    maubot_plugin_config_overrides,
+):
+    yaml = YAML()
+    with open(maubot_plugin_path.joinpath("base-config.yaml")) as fh:
+        base_config = RecursiveDict(yaml.load(fh))
+    maubot_plugin_config_dict.update(maubot_plugin_config_overrides)
+    return maubot_plugin_config_class(
+        load=lambda: maubot_plugin_config_dict,
+        load_base=lambda: base_config,
+        save=lambda c: None,
+    )
+
+
+@pytest_asyncio.fixture
+async def maubot_plugin(
+    maubot_test_bot,
+    maubot_plugin_db,
+    maubot_plugin_class,
+    maubot_plugin_path,
+    maubot_plugin_config,
+    maubot_plugin_meta,
+):
+    loader = FileSystemLoader(maubot_plugin_path, maubot_plugin_meta)
+    async with aiohttp.ClientSession() as http:
+        instance = maubot_plugin_class(
+            client=maubot_test_bot.client,
+            loop=asyncio.get_running_loop(),
+            http=http,
+            instance_id="tests",
+            log=TraceLogger("test"),
+            config=maubot_plugin_config,
+            database=maubot_plugin_db,
+            webapp=None,
+            webapp_url=None,
+            loader=loader,
+        )
+        await instance.internal_start()
+        yield instance

optional-requirements.txt

@@ -5,3 +5,10 @@
 python-olm>=3,<4
 pycryptodome>=3,<4
 unpaddedbase64>=1,<3
+
+#/testing
+pytest
+pytest-asyncio
+
+#/legacydb
+SQLAlchemy>1,<1.4

pyproject.toml

@@ -9,5 +9,5 @@ skip = ["maubot/management/frontend"]
 
 [tool.black]
 line-length = 99
-target-version = ["py38"]
+target-version = ["py310"]
 force-exclude = "maubot/management/frontend"

requirements.txt

@@ -1,16 +1,16 @@
-mautrix>=0.20.2,<0.21
+mautrix>=0.20.6,<0.21
 aiohttp>=3,<4
 yarl>=1,<2
-SQLAlchemy>=1,<1.4
+asyncpg>=0.20,<0.30
-asyncpg>=0.20,<0.28
+aiosqlite>=0.16,<0.21
-aiosqlite>=0.16,<0.19
 commonmark>=0.9,<1
-ruamel.yaml>=0.15.35,<0.18
+ruamel.yaml>=0.15.35,<0.19
 attrs>=18.1.0
 bcrypt>=3,<5
 packaging>=10
 
 click>=7,<9
 colorama>=0.4,<0.5
-questionary>=1,<2
+questionary>=1,<3
 jinja2>=2,<4
+setuptools

setup.py

@@ -41,7 +41,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.9",
+    python_requires="~=3.10",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -50,13 +50,15 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     entry_points="""
         [console_scripts]
         mbc=maubot.cli:app
+        [pytest11]
+        maubot=maubot.testing
     """,
     data_files=[
         (".", ["maubot/example-config.yaml"]),