Merge f63be9059b into 91f214819a

.gitlab-ci.yml

@@ -10,7 +10,7 @@ default:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 build frontend:
-  image: node:20-alpine
+  image: node:18-alpine
   stage: build frontend
   before_script: []
   variables:

CHANGELOG.md

@@ -1,9 +1,5 @@
-# v0.5.0 (2024-08-24)
+# v0.5.0 (unreleased)
 
-* Dropped Python 3.9 support.
-* Updated Docker image to Alpine 3.20.
-* Updated mautrix-python to 0.20.6 to support authenticated media.
-* Removed hard dependency on SQLAlchemy.
 * Fixed `main_class` to default to being loaded from the last module instead of
   the first if a module name is not explicitly specified.
   * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html),

Dockerfile

@@ -1,9 +1,9 @@
-FROM node:20 AS frontend-builder
+FROM node:18 AS frontend-builder
 
 COPY ./maubot/management/frontend /frontend
 RUN cd /frontend && yarn --prod && yarn build
 
-FROM alpine:3.20
+FROM alpine:3.18
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -11,6 +11,7 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
+        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -33,19 +34,20 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        py3-pillow \
+        #py3-pillow \
         py3-magic \
         py3-feedparser \
         py3-dateutil \
         py3-lxml \
-        py3-semver
+        py3-semver \
+  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
 
 COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
+    && pip3 install -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery tzlocal \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM alpine:3.20
+FROM alpine:3.18
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -6,6 +6,7 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
+        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -29,10 +30,11 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        py3-pillow \
+        #py3-pillow \
         py3-magic \
         py3-feedparser \
-        py3-lxml
+        py3-lxml \
+  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 #        py3-gitlab
 #        py3-semver
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
@@ -41,7 +43,7 @@ COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
+    && pip3 install -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

maubot/__meta__.py

@@ -1 +1 @@
-__version__ = "0.5.0"
+__version__ = "0.4.2"

maubot/instance.py

@@ -25,6 +25,7 @@ import os.path
 
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
+import sqlalchemy as sql
 
 from mautrix.types import UserID
 from mautrix.util import background_task
@@ -35,7 +36,6 @@ from mautrix.util.logging import TraceLogger
 
 from .client import Client
 from .db import DatabaseEngine, Instance as DBInstance
-from .lib.optionalalchemy import Engine, MetaData, create_engine
 from .lib.plugin_db import ProxyPostgresDatabase
 from .loader import DatabaseType, PluginLoader, ZippedPluginLoader
 from .plugin_base import Plugin
@@ -128,7 +128,7 @@ class PluginInstance(DBInstance):
         }
 
     def _introspect_sqlalchemy(self) -> dict:
-        metadata = MetaData()
+        metadata = sql.MetaData()
         metadata.reflect(self.inst_db)
         return {
             table.name: {
@@ -214,7 +214,7 @@ class PluginInstance(DBInstance):
 
     async def get_db_tables(self) -> dict:
         if self.inst_db_tables is None:
-            if isinstance(self.inst_db, Engine):
+            if isinstance(self.inst_db, sql.engine.Engine):
                 self.inst_db_tables = self._introspect_sqlalchemy()
             elif self.inst_db.scheme == Scheme.SQLITE:
                 self.inst_db_tables = await self._introspect_sqlite()
@@ -294,7 +294,7 @@ class PluginInstance(DBInstance):
                     "Instance database engine is marked as Postgres, but plugin uses legacy "
                     "database interface, which doesn't support postgres."
                 )
-            self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}")
+            self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}")
         elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
             if self.database_engine is None:
                 if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db:
@@ -329,7 +329,7 @@ class PluginInstance(DBInstance):
     async def stop_database(self) -> None:
         if isinstance(self.inst_db, Database):
             await self.inst_db.stop()
-        elif isinstance(self.inst_db, Engine):
+        elif isinstance(self.inst_db, sql.engine.Engine):
             self.inst_db.dispose()
         else:
             raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}")

maubot/lib/optionalalchemy.py

@@ -1,19 +0,0 @@
-try:
-    from sqlalchemy import MetaData, asc, create_engine, desc
-    from sqlalchemy.engine import Engine
-    from sqlalchemy.exc import IntegrityError, OperationalError
-except ImportError:
-
-    class FakeError(Exception):
-        pass
-
-    class FakeType:
-        def __init__(self, *args, **kwargs):
-            raise Exception("SQLAlchemy is not installed")
-
-    def create_engine(*args, **kwargs):
-        raise Exception("SQLAlchemy is not installed")
-
-    MetaData = Engine = FakeType
-    IntegrityError = OperationalError = FakeError
-    asc = desc = lambda a: a

maubot/loader/zip.py

@@ -31,7 +31,7 @@ from ..config import Config
 from ..lib.zipimport import ZipImportError, zipimporter
 from ..plugin_base import Plugin
 from .abc import IDConflictError, PluginClass, PluginLoader
-from .meta import DatabaseType, PluginMeta
+from .meta import PluginMeta
 
 current_version = Version(__version__)
 yaml = YAML()
@@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader):
         return file, meta
 
     @classmethod
-    def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]:
+    def verify_meta(cls, source) -> tuple[str, Version]:
         _, meta = cls._read_meta(source)
-        return meta.id, meta.version, meta.database_type if meta.database else None
+        return meta.id, meta.version
 
     def _load_meta(self) -> None:
         file, meta = self._read_meta(self.path)

maubot/management/api/instance_database.py

@@ -19,12 +19,12 @@ from datetime import datetime
 
 from aiohttp import web
 from asyncpg import PostgresError
+from sqlalchemy import asc, desc, engine, exc
 import aiosqlite
 
 from mautrix.util.async_db import Database
 
 from ...instance import PluginInstance
-from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc
 from .base import routes
 from .responses import resp
 
@@ -66,7 +66,7 @@ async def get_table(request: web.Request) -> web.Response:
     except KeyError:
         order = []
     limit = int(request.query.get("limit", "100"))
-    if isinstance(instance.inst_db, Engine):
+    if isinstance(instance.inst_db, engine.Engine):
         return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit))
 
 
@@ -84,7 +84,7 @@ async def query(request: web.Request) -> web.Response:
     except KeyError:
         return resp.query_missing
     rows_as_dict = data.get("rows_as_dict", False)
-    if isinstance(instance.inst_db, Engine):
+    if isinstance(instance.inst_db, engine.Engine):
         return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict)
     elif isinstance(instance.inst_db, Database):
         try:
@@ -133,12 +133,12 @@ async def _execute_query_asyncpg(
 def _execute_query_sqlalchemy(
     instance: PluginInstance, sql_query: str, rows_as_dict: bool = False
 ) -> web.Response:
-    assert isinstance(instance.inst_db, Engine)
+    assert isinstance(instance.inst_db, engine.Engine)
     try:
         res = instance.inst_db.execute(sql_query)
-    except IntegrityError as e:
+    except exc.IntegrityError as e:
         return resp.sql_integrity_error(e, sql_query)
-    except OperationalError as e:
+    except exc.OperationalError as e:
         return resp.sql_operational_error(e, sql_query)
     data = {
         "ok": True,

maubot/management/api/plugin_upload.py

@@ -23,17 +23,10 @@ import traceback
 from aiohttp import web
 from packaging.version import Version
 
-from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader
+from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader
 from .base import get_config, routes
 from .responses import resp
 
-try:
-    import sqlalchemy
-
-    has_alchemy = True
-except ImportError:
-    has_alchemy = False
-
 log = logging.getLogger("maubot.server.upload")
 
 
@@ -43,11 +36,9 @@ async def put_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
+        pid, version = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
-    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
-        return resp.sqlalchemy_not_installed
     if pid != plugin_id:
         return resp.pid_mismatch
     plugin = PluginLoader.id_cache.get(plugin_id, None)
@@ -64,11 +55,9 @@ async def upload_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
+        pid, version = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
-    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
-        return resp.sqlalchemy_not_installed
     plugin = PluginLoader.id_cache.get(pid, None)
     if not plugin:
         return await upload_new_plugin(content, pid, version)

maubot/management/api/responses.py

@@ -15,16 +15,13 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
 from http import HTTPStatus
 
 from aiohttp import web
 from asyncpg import PostgresError
+from sqlalchemy.exc import IntegrityError, OperationalError
 import aiosqlite
 
-if TYPE_CHECKING:
-    from sqlalchemy.exc import IntegrityError, OperationalError
-
 
 class _Response:
     @property
@@ -327,16 +324,6 @@ class _Response:
             }
         )
 
-    @property
-    def sqlalchemy_not_installed(self) -> web.Response:
-        return web.json_response(
-            {
-                "error": "This plugin requires a legacy database, but SQLAlchemy is not installed",
-                "errcode": "unsupported_plugin_database",
-            },
-            status=HTTPStatus.NOT_IMPLEMENTED,
-        )
-
     @property
     def table_not_found(self) -> web.Response:
         return web.json_response(

maubot/plugin_base.py

@@ -20,6 +20,7 @@ from abc import ABC
 from asyncio import AbstractEventLoop
 
 from aiohttp import ClientSession
+from sqlalchemy.engine.base import Engine
 from yarl import URL
 
 from mautrix.util.async_db import Database, UpgradeTable
@@ -29,8 +30,6 @@ from mautrix.util.logging import TraceLogger
 from .scheduler import BasicScheduler
 
 if TYPE_CHECKING:
-    from sqlalchemy.engine.base import Engine
-
     from .client import MaubotMatrixClient
     from .loader import BasePluginLoader
     from .plugin_server import PluginWebApp
@@ -57,7 +56,7 @@ class Plugin(ABC):
         instance_id: str,
         log: TraceLogger,
         config: BaseProxyConfig | None,
-        database: Engine | Database | None,
+        database: Engine | None,
         webapp: PluginWebApp | None,
         webapp_url: str | None,
         loader: BasePluginLoader,

maubot/server.py

@@ -64,14 +64,14 @@ class MaubotServer:
             if request.path.startswith(path):
                 request = request.clone(
                     rel_url=request.rel_url.with_path(
-                        request.rel_url.path[len(path) - 1 :]
+                        request.rel_url.path[len(path) :]
                     ).with_query(request.query_string)
                 )
                 return await app.handle(request)
         return web.Response(status=404)
 
     def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]:
-        subpath = self.config["server.plugin_base_path"] + instance_id + "/"
+        subpath = self.config["server.plugin_base_path"] + instance_id
         url = self.config["server.public_url"] + subpath
         try:
             return self.plugin_routes[subpath], url
@@ -82,7 +82,7 @@ class MaubotServer:
 
     def remove_instance_webapp(self, instance_id: str) -> None:
         try:
-            subpath = self.config["server.plugin_base_path"] + instance_id + "/"
+            subpath = self.config["server.plugin_base_path"] + instance_id
             self.plugin_routes.pop(subpath).clear()
         except KeyError:
             return

maubot/standalone/Dockerfile

@@ -1,8 +1,9 @@
-FROM docker.io/alpine:3.20
+FROM docker.io/alpine:3.18
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-aiohttp \
+      py3-sqlalchemy \
       py3-attrs \
       py3-bcrypt \
       py3-cffi \
@@ -25,8 +26,8 @@ RUN cd /opt/maubot \
       python3-dev \
       libffi-dev \
       build-base \
-  && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
+  && pip3 install -r requirements.txt -r optional-requirements.txt \
   && apk del .build-deps
 
 COPY . /opt/maubot
-RUN cd /opt/maubot && pip3 install --break-system-packages .
+RUN cd /opt/maubot && pip3 install .

optional-requirements.txt

@@ -9,6 +9,3 @@ unpaddedbase64>=1,<3
 #/testing
 pytest
 pytest-asyncio
-
-#/legacydb
-SQLAlchemy>1,<1.4

requirements.txt

@@ -1,16 +1,16 @@
-mautrix>=0.20.6,<0.21
+mautrix>=0.20.2,<0.21
 aiohttp>=3,<4
 yarl>=1,<2
-asyncpg>=0.20,<0.30
-aiosqlite>=0.16,<0.21
+SQLAlchemy>=1,<1.4
+asyncpg>=0.20,<0.29
+aiosqlite>=0.16,<0.19
 commonmark>=0.9,<1
-ruamel.yaml>=0.15.35,<0.19
+ruamel.yaml>=0.15.35,<0.18
 attrs>=18.1.0
 bcrypt>=3,<5
 packaging>=10
 
 click>=7,<9
 colorama>=0.4,<0.5
-questionary>=1,<3
+questionary>=1,<2
 jinja2>=2,<4
-setuptools

setup.py

@@ -41,7 +41,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.10",
+    python_requires="~=3.9",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -50,9 +50,9 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
-        "Programming Language :: Python :: 3.12",
     ],
     entry_points="""
         [console_scripts]