Merge 860c17a1bd into 65be63fdd2

Previously, the webapp handler would match without respect to the trailing slash, e.g. matching "foo"
for "foo2". This behavior is changed to respect the trailing slash.

Fixes #239

.gitlab-ci.yml

@@ -10,7 +10,7 @@ default:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 build frontend:
-  image: node:18-alpine
+  image: node:20-alpine
   stage: build frontend
   before_script: []
   variables:

CHANGELOG.md

@@ -1,5 +1,9 @@
-# v0.5.0 (unreleased)
+# v0.5.0 (2024-08-24)
 
+* Dropped Python 3.9 support.
+* Updated Docker image to Alpine 3.20.
+* Updated mautrix-python to 0.20.6 to support authenticated media.
+* Removed hard dependency on SQLAlchemy.
 * Fixed `main_class` to default to being loaded from the last module instead of
   the first if a module name is not explicitly specified.
   * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html),

Dockerfile

@@ -1,9 +1,9 @@
-FROM node:18 AS frontend-builder
+FROM node:20 AS frontend-builder
 
 COPY ./maubot/management/frontend /frontend
 RUN cd /frontend && yarn --prod && yarn build
 
-FROM alpine:3.18
+FROM alpine:3.20
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -11,7 +11,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -34,20 +33,19 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        #py3-pillow \
+        py3-pillow \
         py3-magic \
         py3-feedparser \
         py3-dateutil \
         py3-lxml \
-        py3-semver \
+        py3-semver
-  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
 
 COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery tzlocal \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM alpine:3.18
+FROM alpine:3.20
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -6,7 +6,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -30,11 +29,10 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        #py3-pillow \
+        py3-pillow \
         py3-magic \
         py3-feedparser \
-        py3-lxml \
+        py3-lxml
-  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 #        py3-gitlab
 #        py3-semver
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
@@ -43,7 +41,7 @@ COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

maubot/__meta__.py

@@ -1 +1 @@
-__version__ = "0.4.2"
+__version__ = "0.5.0"

maubot/cli/cliq/cliq.py

@@ -35,7 +35,7 @@ from .validators import ClickValidator, Required
 def with_http(func):
     @functools.wraps(func)
     async def wrapper(*args, **kwargs):
-        async with aiohttp.ClientSession() as sess:
+        async with aiohttp.ClientSession(trust_env=True) as sess:
             try:
                 return await func(*args, sess=sess, **kwargs)
             except aiohttp.ClientError as e:
@@ -50,7 +50,7 @@ def with_authenticated_http(func):
         server, token = get_token(server)
         if not token:
             return
-        async with aiohttp.ClientSession(headers={"Authorization": f"Bearer {token}"}) as sess:
+        async with aiohttp.ClientSession(headers={"Authorization": f"Bearer {token}"}, trust_env=True) as sess:
             try:
                 return await func(*args, sess=sess, server=server, **kwargs)
             except aiohttp.ClientError as e:

maubot/client.py

@@ -139,7 +139,7 @@ class Client(DBClient):
         self._postinited = True
         self.cache[self.id] = self
         self.log = self.log.getChild(self.id)
-        self.http_client = ClientSession(loop=self.maubot.loop)
+        self.http_client = ClientSession(loop=self.maubot.loop, trust_env=True)
         self.references = set()
         self.started = False
         self.sync_ok = True

maubot/instance.py

@@ -25,7 +25,6 @@ import os.path
 
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-import sqlalchemy as sql
 
 from mautrix.types import UserID
 from mautrix.util import background_task
@@ -36,6 +35,7 @@ from mautrix.util.logging import TraceLogger
 
 from .client import Client
 from .db import DatabaseEngine, Instance as DBInstance
+from .lib.optionalalchemy import Engine, MetaData, create_engine
 from .lib.plugin_db import ProxyPostgresDatabase
 from .loader import DatabaseType, PluginLoader, ZippedPluginLoader
 from .plugin_base import Plugin
@@ -128,7 +128,7 @@ class PluginInstance(DBInstance):
         }
 
     def _introspect_sqlalchemy(self) -> dict:
-        metadata = sql.MetaData()
+        metadata = MetaData()
         metadata.reflect(self.inst_db)
         return {
             table.name: {
@@ -214,7 +214,7 @@ class PluginInstance(DBInstance):
 
     async def get_db_tables(self) -> dict:
         if self.inst_db_tables is None:
-            if isinstance(self.inst_db, sql.engine.Engine):
+            if isinstance(self.inst_db, Engine):
                 self.inst_db_tables = self._introspect_sqlalchemy()
             elif self.inst_db.scheme == Scheme.SQLITE:
                 self.inst_db_tables = await self._introspect_sqlite()
@@ -294,7 +294,7 @@ class PluginInstance(DBInstance):
                     "Instance database engine is marked as Postgres, but plugin uses legacy "
                     "database interface, which doesn't support postgres."
                 )
-            self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}")
+            self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}")
         elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
             if self.database_engine is None:
                 if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db:
@@ -329,7 +329,7 @@ class PluginInstance(DBInstance):
     async def stop_database(self) -> None:
         if isinstance(self.inst_db, Database):
             await self.inst_db.stop()
-        elif isinstance(self.inst_db, sql.engine.Engine):
+        elif isinstance(self.inst_db, Engine):
             self.inst_db.dispose()
         else:
             raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}")

maubot/lib/optionalalchemy.py

@@ -0,0 +1,19 @@
+try:
+    from sqlalchemy import MetaData, asc, create_engine, desc
+    from sqlalchemy.engine import Engine
+    from sqlalchemy.exc import IntegrityError, OperationalError
+except ImportError:
+
+    class FakeError(Exception):
+        pass
+
+    class FakeType:
+        def __init__(self, *args, **kwargs):
+            raise Exception("SQLAlchemy is not installed")
+
+    def create_engine(*args, **kwargs):
+        raise Exception("SQLAlchemy is not installed")
+
+    MetaData = Engine = FakeType
+    IntegrityError = OperationalError = FakeError
+    asc = desc = lambda a: a

maubot/loader/zip.py

@@ -31,7 +31,7 @@ from ..config import Config
 from ..lib.zipimport import ZipImportError, zipimporter
 from ..plugin_base import Plugin
 from .abc import IDConflictError, PluginClass, PluginLoader
-from .meta import PluginMeta
+from .meta import DatabaseType, PluginMeta
 
 current_version = Version(__version__)
 yaml = YAML()
@@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader):
         return file, meta
 
     @classmethod
-    def verify_meta(cls, source) -> tuple[str, Version]:
+    def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]:
         _, meta = cls._read_meta(source)
-        return meta.id, meta.version
+        return meta.id, meta.version, meta.database_type if meta.database else None
 
     def _load_meta(self) -> None:
         file, meta = self._read_meta(self.path)

maubot/management/api/client_proxy.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from aiohttp import client as http, web
+from urllib.request import getproxies
 
 from ...client import Client
 from .base import routes
@@ -45,8 +46,10 @@ async def proxy(request: web.Request) -> web.StreamResponse:
             headers["X-Forwarded-For"] = f"{host}:{port}"
 
     data = await request.read()
+    proxies = getproxies()
     async with http.request(
-        request.method, f"{client.homeserver}/{path}", headers=headers, params=query, data=data
+            request.method, f"{client.homeserver}/{path}", headers=headers, params=query, data=data,
+            proxy=proxies["https"] if "https" in proxies else None
     ) as proxy_resp:
         response = web.StreamResponse(status=proxy_resp.status, headers=proxy_resp.headers)
         await response.prepare(request)

maubot/management/api/instance_database.py

@@ -19,12 +19,12 @@ from datetime import datetime
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy import asc, desc, engine, exc
 import aiosqlite
 
 from mautrix.util.async_db import Database
 
 from ...instance import PluginInstance
+from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc
 from .base import routes
 from .responses import resp
 
@@ -66,7 +66,7 @@ async def get_table(request: web.Request) -> web.Response:
     except KeyError:
         order = []
     limit = int(request.query.get("limit", "100"))
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit))
 
 
@@ -84,7 +84,7 @@ async def query(request: web.Request) -> web.Response:
     except KeyError:
         return resp.query_missing
     rows_as_dict = data.get("rows_as_dict", False)
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict)
     elif isinstance(instance.inst_db, Database):
         try:
@@ -133,12 +133,12 @@ async def _execute_query_asyncpg(
 def _execute_query_sqlalchemy(
     instance: PluginInstance, sql_query: str, rows_as_dict: bool = False
 ) -> web.Response:
-    assert isinstance(instance.inst_db, engine.Engine)
+    assert isinstance(instance.inst_db, Engine)
     try:
         res = instance.inst_db.execute(sql_query)
-    except exc.IntegrityError as e:
+    except IntegrityError as e:
         return resp.sql_integrity_error(e, sql_query)
-    except exc.OperationalError as e:
+    except OperationalError as e:
         return resp.sql_operational_error(e, sql_query)
     data = {
         "ok": True,

maubot/management/api/plugin_upload.py

@@ -23,10 +23,17 @@ import traceback
 from aiohttp import web
 from packaging.version import Version
 
-from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader
+from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader
 from .base import get_config, routes
 from .responses import resp
 
+try:
+    import sqlalchemy
+
+    has_alchemy = True
+except ImportError:
+    has_alchemy = False
+
 log = logging.getLogger("maubot.server.upload")
 
 
@@ -36,9 +43,11 @@ async def put_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     if pid != plugin_id:
         return resp.pid_mismatch
     plugin = PluginLoader.id_cache.get(plugin_id, None)
@@ -55,9 +64,11 @@ async def upload_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     plugin = PluginLoader.id_cache.get(pid, None)
     if not plugin:
         return await upload_new_plugin(content, pid, version)

maubot/management/api/responses.py

@@ -15,13 +15,16 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
 from http import HTTPStatus
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy.exc import IntegrityError, OperationalError
 import aiosqlite
 
+if TYPE_CHECKING:
+    from sqlalchemy.exc import IntegrityError, OperationalError
+
 
 class _Response:
     @property
@@ -324,6 +327,16 @@ class _Response:
             }
         )
 
+    @property
+    def sqlalchemy_not_installed(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "This plugin requires a legacy database, but SQLAlchemy is not installed",
+                "errcode": "unsupported_plugin_database",
+            },
+            status=HTTPStatus.NOT_IMPLEMENTED,
+        )
+
     @property
     def table_not_found(self) -> web.Response:
         return web.json_response(

maubot/plugin_base.py

@@ -20,7 +20,6 @@ from abc import ABC
 from asyncio import AbstractEventLoop
 
 from aiohttp import ClientSession
-from sqlalchemy.engine.base import Engine
 from yarl import URL
 
 from mautrix.util.async_db import Database, UpgradeTable
@@ -30,6 +29,8 @@ from mautrix.util.logging import TraceLogger
 from .scheduler import BasicScheduler
 
 if TYPE_CHECKING:
+    from sqlalchemy.engine.base import Engine
+
     from .client import MaubotMatrixClient
     from .loader import BasePluginLoader
     from .plugin_server import PluginWebApp
@@ -56,7 +57,7 @@ class Plugin(ABC):
         instance_id: str,
         log: TraceLogger,
         config: BaseProxyConfig | None,
-        database: Engine | None,
+        database: Engine | Database | None,
         webapp: PluginWebApp | None,
         webapp_url: str | None,
         loader: BasePluginLoader,

maubot/server.py

@@ -64,14 +64,14 @@ class MaubotServer:
             if request.path.startswith(path):
                 request = request.clone(
                     rel_url=request.rel_url.with_path(
-                        request.rel_url.path[len(path) :]
+                        request.rel_url.path[len(path) - 1 :]
                     ).with_query(request.query_string)
                 )
                 return await app.handle(request)
         return web.Response(status=404)
 
     def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]:
-        subpath = self.config["server.plugin_base_path"] + instance_id
+        subpath = self.config["server.plugin_base_path"] + instance_id + "/"
         url = self.config["server.public_url"] + subpath
         try:
             return self.plugin_routes[subpath], url
@@ -82,7 +82,7 @@ class MaubotServer:
 
     def remove_instance_webapp(self, instance_id: str) -> None:
         try:
-            subpath = self.config["server.plugin_base_path"] + instance_id
+            subpath = self.config["server.plugin_base_path"] + instance_id + "/"
             self.plugin_routes.pop(subpath).clear()
         except KeyError:
             return

maubot/standalone/Dockerfile

@@ -1,9 +1,8 @@
-FROM docker.io/alpine:3.18
+FROM docker.io/alpine:3.20
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-aiohttp \
-      py3-sqlalchemy \
       py3-attrs \
       py3-bcrypt \
       py3-cffi \
@@ -26,8 +25,8 @@ RUN cd /opt/maubot \
       python3-dev \
       libffi-dev \
       build-base \
-  && pip3 install -r requirements.txt -r optional-requirements.txt \
+  && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
   && apk del .build-deps
 
 COPY . /opt/maubot
-RUN cd /opt/maubot && pip3 install .
+RUN cd /opt/maubot && pip3 install --break-system-packages .

maubot/standalone/__main__.py

@@ -235,7 +235,7 @@ if appservice_listener:
 
 
 async def main():
-    http_client = ClientSession(loop=loop)
+    http_client = ClientSession(loop=loop, trust_env=True)
 
     global client, bot
 

optional-requirements.txt

@@ -9,3 +9,6 @@ unpaddedbase64>=1,<3
 #/testing
 pytest
 pytest-asyncio
+
+#/legacydb
+SQLAlchemy>1,<1.4

requirements.txt

@@ -1,16 +1,16 @@
-mautrix>=0.20.2,<0.21
+mautrix>=0.20.6,<0.21
 aiohttp>=3,<4
 yarl>=1,<2
-SQLAlchemy>=1,<1.4
+asyncpg>=0.20,<0.30
-asyncpg>=0.20,<0.29
+aiosqlite>=0.16,<0.21
-aiosqlite>=0.16,<0.19
 commonmark>=0.9,<1
-ruamel.yaml>=0.15.35,<0.18
+ruamel.yaml>=0.15.35,<0.19
 attrs>=18.1.0
 bcrypt>=3,<5
 packaging>=10
 
 click>=7,<9
 colorama>=0.4,<0.5
-questionary>=1,<2
+questionary>=1,<3
 jinja2>=2,<4
+setuptools

setup.py

@@ -41,7 +41,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.9",
+    python_requires="~=3.10",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -50,9 +50,9 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     entry_points="""
         [console_scripts]