Merge 8220bcedf6 into 472fb9f6ac

[skip ci]

.gitlab-ci.yml

@@ -10,7 +10,7 @@ default:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 build frontend:
-  image: node:18-alpine
+  image: node:20-alpine
   stage: build frontend
   before_script: []
   variables:

CHANGELOG.md

@@ -1,5 +1,9 @@
-# v0.5.0 (unreleased)
+# v0.5.0 (2024-08-24)
 
+* Dropped Python 3.9 support.
+* Updated Docker image to Alpine 3.20.
+* Updated mautrix-python to 0.20.6 to support authenticated media.
+* Removed hard dependency on SQLAlchemy.
 * Fixed `main_class` to default to being loaded from the last module instead of
   the first if a module name is not explicitly specified.
   * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html),

Dockerfile

@@ -1,9 +1,9 @@
-FROM node:18 AS frontend-builder
+FROM node:20 AS frontend-builder
 
 COPY ./maubot/management/frontend /frontend
 RUN cd /frontend && yarn --prod && yarn build
 
-FROM alpine:3.18
+FROM alpine:3.20
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -11,7 +11,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -34,20 +33,19 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        #py3-pillow \
+        py3-pillow \
         py3-magic \
         py3-feedparser \
         py3-dateutil \
         py3-lxml \
-        py3-semver \
+        py3-semver
-  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
 
 COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery tzlocal \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM alpine:3.18
+FROM alpine:3.20
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -6,7 +6,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -30,11 +29,10 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        #py3-pillow \
+        py3-pillow \
         py3-magic \
         py3-feedparser \
-        py3-lxml \
+        py3-lxml
-  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 #        py3-gitlab
 #        py3-semver
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
@@ -43,7 +41,7 @@ COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

maubot/__meta__.py

@@ -1 +1 @@
-__version__ = "0.4.2"
+__version__ = "0.5.0"

maubot/example-config.yaml

@@ -1,5 +1,4 @@
 # The full URI to the database. SQLite and Postgres are fully supported.
-# Other DBMSes supported by SQLAlchemy may or may not work.
 # Format examples:
 #   SQLite:   sqlite:filename.db
 #   Postgres: postgresql://username:password@hostname/dbname

maubot/handlers/command.py

@@ -403,6 +403,8 @@ class CustomArgument(Argument):
 
 class SimpleArgument(Argument):
     def match(self, val: str, **kwargs) -> Tuple[str, Any]:
+        if not val:
+            return "", None
         if self.pass_raw:
             return "", val
         res = re.split(r"\s", val, 1)[0]

maubot/instance.py

@@ -25,7 +25,6 @@ import os.path
 
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-import sqlalchemy as sql
 
 from mautrix.types import UserID
 from mautrix.util import background_task
@@ -36,6 +35,7 @@ from mautrix.util.logging import TraceLogger
 
 from .client import Client
 from .db import DatabaseEngine, Instance as DBInstance
+from .lib.optionalalchemy import Engine, MetaData, create_engine
 from .lib.plugin_db import ProxyPostgresDatabase
 from .loader import DatabaseType, PluginLoader, ZippedPluginLoader
 from .plugin_base import Plugin
@@ -128,7 +128,7 @@ class PluginInstance(DBInstance):
         }
 
     def _introspect_sqlalchemy(self) -> dict:
-        metadata = sql.MetaData()
+        metadata = MetaData()
         metadata.reflect(self.inst_db)
         return {
             table.name: {
@@ -214,7 +214,7 @@ class PluginInstance(DBInstance):
 
     async def get_db_tables(self) -> dict:
         if self.inst_db_tables is None:
-            if isinstance(self.inst_db, sql.engine.Engine):
+            if isinstance(self.inst_db, Engine):
                 self.inst_db_tables = self._introspect_sqlalchemy()
             elif self.inst_db.scheme == Scheme.SQLITE:
                 self.inst_db_tables = await self._introspect_sqlite()
@@ -294,7 +294,7 @@ class PluginInstance(DBInstance):
                     "Instance database engine is marked as Postgres, but plugin uses legacy "
                     "database interface, which doesn't support postgres."
                 )
-            self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}")
+            self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}")
         elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
             if self.database_engine is None:
                 if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db:
@@ -329,7 +329,7 @@ class PluginInstance(DBInstance):
     async def stop_database(self) -> None:
         if isinstance(self.inst_db, Database):
             await self.inst_db.stop()
-        elif isinstance(self.inst_db, sql.engine.Engine):
+        elif isinstance(self.inst_db, Engine):
             self.inst_db.dispose()
         else:
             raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}")

maubot/lib/optionalalchemy.py

@@ -0,0 +1,19 @@
+try:
+    from sqlalchemy import MetaData, asc, create_engine, desc
+    from sqlalchemy.engine import Engine
+    from sqlalchemy.exc import IntegrityError, OperationalError
+except ImportError:
+
+    class FakeError(Exception):
+        pass
+
+    class FakeType:
+        def __init__(self, *args, **kwargs):
+            raise Exception("SQLAlchemy is not installed")
+
+    def create_engine(*args, **kwargs):
+        raise Exception("SQLAlchemy is not installed")
+
+    MetaData = Engine = FakeType
+    IntegrityError = OperationalError = FakeError
+    asc = desc = lambda a: a

maubot/loader/zip.py

@@ -31,7 +31,7 @@ from ..config import Config
 from ..lib.zipimport import ZipImportError, zipimporter
 from ..plugin_base import Plugin
 from .abc import IDConflictError, PluginClass, PluginLoader
-from .meta import PluginMeta
+from .meta import DatabaseType, PluginMeta
 
 current_version = Version(__version__)
 yaml = YAML()
@@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader):
         return file, meta
 
     @classmethod
-    def verify_meta(cls, source) -> tuple[str, Version]:
+    def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]:
         _, meta = cls._read_meta(source)
-        return meta.id, meta.version
+        return meta.id, meta.version, meta.database_type if meta.database else None
 
     def _load_meta(self) -> None:
         file, meta = self._read_meta(self.path)

maubot/management/api/instance_database.py

@@ -19,12 +19,12 @@ from datetime import datetime
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy import asc, desc, engine, exc
 import aiosqlite
 
 from mautrix.util.async_db import Database
 
 from ...instance import PluginInstance
+from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc
 from .base import routes
 from .responses import resp
 
@@ -66,7 +66,7 @@ async def get_table(request: web.Request) -> web.Response:
     except KeyError:
         order = []
     limit = int(request.query.get("limit", "100"))
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit))
 
 
@@ -84,7 +84,7 @@ async def query(request: web.Request) -> web.Response:
     except KeyError:
         return resp.query_missing
     rows_as_dict = data.get("rows_as_dict", False)
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict)
     elif isinstance(instance.inst_db, Database):
         try:
@@ -133,12 +133,12 @@ async def _execute_query_asyncpg(
 def _execute_query_sqlalchemy(
     instance: PluginInstance, sql_query: str, rows_as_dict: bool = False
 ) -> web.Response:
-    assert isinstance(instance.inst_db, engine.Engine)
+    assert isinstance(instance.inst_db, Engine)
     try:
         res = instance.inst_db.execute(sql_query)
-    except exc.IntegrityError as e:
+    except IntegrityError as e:
         return resp.sql_integrity_error(e, sql_query)
-    except exc.OperationalError as e:
+    except OperationalError as e:
         return resp.sql_operational_error(e, sql_query)
     data = {
         "ok": True,

maubot/management/api/plugin_upload.py

@@ -23,10 +23,17 @@ import traceback
 from aiohttp import web
 from packaging.version import Version
 
-from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader
+from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader
 from .base import get_config, routes
 from .responses import resp
 
+try:
+    import sqlalchemy
+
+    has_alchemy = True
+except ImportError:
+    has_alchemy = False
+
 log = logging.getLogger("maubot.server.upload")
 
 
@@ -36,9 +43,11 @@ async def put_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     if pid != plugin_id:
         return resp.pid_mismatch
     plugin = PluginLoader.id_cache.get(plugin_id, None)
@@ -55,9 +64,11 @@ async def upload_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     plugin = PluginLoader.id_cache.get(pid, None)
     if not plugin:
         return await upload_new_plugin(content, pid, version)

maubot/management/api/responses.py

@@ -15,13 +15,16 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
 from http import HTTPStatus
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy.exc import IntegrityError, OperationalError
 import aiosqlite
 
+if TYPE_CHECKING:
+    from sqlalchemy.exc import IntegrityError, OperationalError
+
 
 class _Response:
     @property
@@ -324,6 +327,16 @@ class _Response:
             }
         )
 
+    @property
+    def sqlalchemy_not_installed(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "This plugin requires a legacy database, but SQLAlchemy is not installed",
+                "errcode": "unsupported_plugin_database",
+            },
+            status=HTTPStatus.NOT_IMPLEMENTED,
+        )
+
     @property
     def table_not_found(self) -> web.Response:
         return web.json_response(

maubot/plugin_base.py

@@ -20,7 +20,6 @@ from abc import ABC
 from asyncio import AbstractEventLoop
 
 from aiohttp import ClientSession
-from sqlalchemy.engine.base import Engine
 from yarl import URL
 
 from mautrix.util.async_db import Database, UpgradeTable
@@ -30,6 +29,8 @@ from mautrix.util.logging import TraceLogger
 from .scheduler import BasicScheduler
 
 if TYPE_CHECKING:
+    from sqlalchemy.engine.base import Engine
+
     from .client import MaubotMatrixClient
     from .loader import BasePluginLoader
     from .plugin_server import PluginWebApp
@@ -56,7 +57,7 @@ class Plugin(ABC):
         instance_id: str,
         log: TraceLogger,
         config: BaseProxyConfig | None,
-        database: Engine | None,
+        database: Engine | Database | None,
         webapp: PluginWebApp | None,
         webapp_url: str | None,
         loader: BasePluginLoader,

maubot/server.py

@@ -64,14 +64,14 @@ class MaubotServer:
             if request.path.startswith(path):
                 request = request.clone(
                     rel_url=request.rel_url.with_path(
-                        request.rel_url.path[len(path) :]
+                        request.rel_url.path[len(path) - 1 :]
                     ).with_query(request.query_string)
                 )
                 return await app.handle(request)
         return web.Response(status=404)
 
     def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]:
-        subpath = self.config["server.plugin_base_path"] + instance_id
+        subpath = self.config["server.plugin_base_path"] + instance_id + "/"
         url = self.config["server.public_url"] + subpath
         try:
             return self.plugin_routes[subpath], url
@@ -82,7 +82,7 @@ class MaubotServer:
 
     def remove_instance_webapp(self, instance_id: str) -> None:
         try:
-            subpath = self.config["server.plugin_base_path"] + instance_id
+            subpath = self.config["server.plugin_base_path"] + instance_id + "/"
             self.plugin_routes.pop(subpath).clear()
         except KeyError:
             return

maubot/standalone/Dockerfile

@@ -1,9 +1,8 @@
-FROM docker.io/alpine:3.18
+FROM docker.io/alpine:3.20
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-aiohttp \
-      py3-sqlalchemy \
       py3-attrs \
       py3-bcrypt \
       py3-cffi \
@@ -26,8 +25,8 @@ RUN cd /opt/maubot \
       python3-dev \
       libffi-dev \
       build-base \
-  && pip3 install -r requirements.txt -r optional-requirements.txt \
+  && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
   && apk del .build-deps
 
 COPY . /opt/maubot
-RUN cd /opt/maubot && pip3 install .
+RUN cd /opt/maubot && pip3 install --break-system-packages .

optional-requirements.txt

@@ -9,3 +9,6 @@ unpaddedbase64>=1,<3
 #/testing
 pytest
 pytest-asyncio
+
+#/legacydb
+SQLAlchemy>1,<1.4

requirements.txt

@@ -1,16 +1,16 @@
-mautrix>=0.20.2,<0.21
+mautrix>=0.20.6,<0.21
 aiohttp>=3,<4
 yarl>=1,<2
-SQLAlchemy>=1,<1.4
+asyncpg>=0.20,<0.30
-asyncpg>=0.20,<0.29
+aiosqlite>=0.16,<0.21
-aiosqlite>=0.16,<0.19
 commonmark>=0.9,<1
-ruamel.yaml>=0.15.35,<0.18
+ruamel.yaml>=0.15.35,<0.19
 attrs>=18.1.0
 bcrypt>=3,<5
 packaging>=10
 
 click>=7,<9
 colorama>=0.4,<0.5
-questionary>=1,<2
+questionary>=1,<3
 jinja2>=2,<4
+setuptools

setup.py

@@ -41,7 +41,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.9",
+    python_requires="~=3.10",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -50,9 +50,9 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     entry_points="""
         [console_scripts]