Merge remote-tracking branch 'upstream/release-v1.115'

Signed-off-by: Tulir Asokan <tulir@maunium.net>

.dockerignore

@@ -8,6 +8,7 @@
 !README.rst
 !pyproject.toml
 !poetry.lock
+!requirements.txt
 !Cargo.lock
 !Cargo.toml
 !build_rust.py

.gitlab-ci.yml

@@ -0,0 +1,19 @@
+image: docker:stable
+
+stages:
+- build
+
+build amd64:
+  stage: build
+  tags:
+  - amd64
+  only:
+  - master
+  before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+  - synversion=$(cat pyproject.toml | grep '^version =' | sed -E 's/^version = "(.+)"$/\1/')
+  - docker build --tag $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$synversion .
+  - docker push $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:$synversion
+  - docker rmi $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$synversion

Dockerfile

@@ -0,0 +1,61 @@
+ARG PYTHON_VERSION=3.11
+
+FROM docker.io/python:${PYTHON_VERSION}-slim as builder
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libffi-dev \
+    libjpeg-dev \
+    libpq-dev \
+    libssl-dev \
+    libwebp-dev \
+    libxml++2.6-dev \
+    libxslt1-dev \
+    zlib1g-dev \
+    openssl \
+    git \
+    curl \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV RUSTUP_HOME=/rust
+ENV CARGO_HOME=/cargo
+ENV PATH=/cargo/bin:/rust/bin:$PATH
+RUN mkdir /rust /cargo
+
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable
+
+COPY synapse /synapse/synapse/
+COPY rust /synapse/rust/
+COPY README.rst pyproject.toml requirements.txt build_rust.py /synapse/
+
+RUN pip install --prefix="/install" --no-warn-script-location --ignore-installed \
+        --no-deps -r /synapse/requirements.txt \
+ && pip install --prefix="/install" --no-warn-script-location \
+        --no-deps \
+	'git+https://github.com/maunium/synapse-simple-antispam#egg=synapse-simple-antispam' \
+	'git+https://github.com/devture/matrix-synapse-shared-secret-auth@2.0.3#egg=shared_secret_authenticator' \
+ && pip install --prefix="/install" --no-warn-script-location \
+        --no-deps /synapse
+
+FROM docker.io/python:${PYTHON_VERSION}-slim
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    libjpeg62-turbo \
+    libpq5 \
+    libwebp7 \
+    xmlsec1 \
+    libjemalloc2 \
+    openssl \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /install /usr/local
+
+VOLUME ["/data"]
+ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2"
+
+ENTRYPOINT ["python3", "-m", "synapse.app.homeserver"]
+CMD ["--keys-directory", "/data", "-c", "/data/homeserver.yaml"]
+
+HEALTHCHECK --start-period=5s --interval=1m --timeout=5s \
+  CMD curl -fSs http://localhost:8008/health || exit 1

README.md

@@ -0,0 +1,63 @@
+# Maunium Synapse
+This is a fork of [Synapse] to remove dumb limits and fix bugs that the
+upstream devs don't want to fix.
+
+The only official distribution is the docker image in the [GitLab container
+registry], but you can also install from source ([upstream instructions]).
+
+The master branch and `:latest` docker tag are upgraded to each upstream
+release candidate very soon after release (usually within 10 minutes†). There
+are also docker tags for each release, e.g. `:1.75.0`. If you don't want RCs,
+use the specific release tags.
+
+†If there are merge conflicts, the update may be delayed for up to a few days
+after the full release.
+
+[Synapse]: https://github.com/matrix-org/synapse
+[GitLab container registry]: https://mau.dev/maunium/synapse/container_registry
+[upstream instructions]: https://github.com/matrix-org/synapse/blob/develop/INSTALL.md#installing-from-source
+
+## List of changes
+* Default power level for room creator is 9001 instead of 100.
+* Room creator can specify a custom room ID with the `room_id` param in the
+  request body. If the room ID is already in use, it will return `M_CONFLICT`.
+* ~~URL previewer user agent includes `Bot` so Twitter previews work properly.~~
+  Upstreamed after over 2 years 🎉
+* ~~Local event creation concurrency is disabled to avoid unnecessary state
+  resolution.~~ Upstreamed after over 3 years 🎉
+* Register admin API can register invalid user IDs.
+* Docker image with jemalloc enabled by default.
+* Config option to allow specific users to send events without unnecessary
+  validation.
+* Config option to allow specific users to receive events that are usually
+  filtered away (e.g. `org.matrix.dummy_event` and `m.room.aliases`).
+* Config option to allow specific users to use timestamp massaging without
+  being appservice users.
+* Removed bad pusher URL validation.
+* webp images are thumbnailed to webp instead of jpeg to avoid losing
+  transparency.
+* Media repo `Cache-Control` header says `immutable` and 1 year for all media
+  that exists, as media IDs in Matrix are immutable.
+* Allowed sending custom data with read receipts.
+
+You can view the full list of changes on the [meow-patchset] branch.
+Additionally, historical patch sets are saved as `meow-patchset-vX` [tags].
+
+[meow-patchset]: https://mau.dev/maunium/synapse/-/compare/patchset-base...meow-patchset
+[tags]: https://mau.dev/maunium/synapse/-/tags?search=meow-patchset&sort=updated_desc
+
+## Configuration reference
+```yaml
+meow:
+   # List of users who aren't subject to unnecessary validation in the C-S API.
+   validation_override:
+   - "@you:example.com"
+   # List of users who will get org.matrix.dummy_event and m.room.aliases events down /sync
+   filter_override:
+   - "@you:example.com"
+   # Whether or not the admin API should be able to register invalid user IDs.
+   admin_api_register_invalid: true
+   # List of users who can use timestamp massaging without being appservices
+   timestamp_override:
+   - "@you:example.com"
+```

synapse/api/filtering.py

@@ -142,12 +142,12 @@ USER_FILTER_SCHEMA = {
 
 @FormatChecker.cls_checks("matrix_room_id")
 def matrix_room_id_validator(room_id: object) -> bool:
-    return isinstance(room_id, str) and RoomID.is_valid(room_id)
+    return isinstance(room_id, str) and (RoomID.is_valid(room_id) or room_id == "*")
 
 
 @FormatChecker.cls_checks("matrix_user_id")
 def matrix_user_id_validator(user_id: object) -> bool:
-    return isinstance(user_id, str) and UserID.is_valid(user_id)
+    return isinstance(user_id, str) and (UserID.is_valid(user_id) or user_id == "*")
 
 
 class Filtering:

synapse/config/_base.pyi

@@ -36,6 +36,7 @@ from synapse.config import (  # noqa: F401
     jwt,
     key,
     logger,
+    meow,
     metrics,
     modules,
     oembed,
@@ -92,6 +93,7 @@ class RootConfig:
     voip: voip.VoipConfig
     registration: registration.RegistrationConfig
     account_validity: account_validity.AccountValidityConfig
+    meow: meow.MeowConfig
     metrics: metrics.MetricsConfig
     api: api.ApiConfig
     appservice: appservice.AppServiceConfig

synapse/config/homeserver.py

@@ -19,6 +19,7 @@
 #
 #
 from ._base import RootConfig
+from .meow import MeowConfig
 from .account_validity import AccountValidityConfig
 from .api import ApiConfig
 from .appservice import AppServiceConfig
@@ -65,6 +66,7 @@ from .workers import WorkerConfig
 
 class HomeServerConfig(RootConfig):
     config_classes = [
+        MeowConfig,
         ModulesConfig,
         ServerConfig,
         RetentionConfig,

synapse/config/meow.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 Maunium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import Config
+
+
+class MeowConfig(Config):
+    """Meow Configuration
+    Configuration for disabling dumb limits in Synapse
+    """
+
+    section = "meow"
+
+    def read_config(self, config, **kwargs):
+        meow_config = config.get("meow", {})
+        self.validation_override = set(meow_config.get("validation_override", []))
+        self.filter_override = set(meow_config.get("filter_override", []))
+        self.timestamp_override = set(meow_config.get("timestamp_override", []))
+        self.admin_api_register_invalid = meow_config.get(
+            "admin_api_register_invalid", True
+        )

synapse/config/repository.py

@@ -54,10 +54,8 @@ THUMBNAIL_SIZE_YAML = """\
 THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP = {
     "image/jpeg": "jpeg",
     "image/jpg": "jpeg",
-    "image/webp": "jpeg",
-    # Thumbnails can only be jpeg or png. We choose png thumbnails for gif
-    # because it can have transparency.
-    "image/gif": "png",
+    "image/webp": "webp",
+    "image/gif": "webp",
     "image/png": "png",
 }
 
@@ -109,6 +107,10 @@ def parse_thumbnail_requirements(
                 requirement.append(
                     ThumbnailRequirement(width, height, method, "image/png")
                 )
+            elif thumbnail_format == "webp":
+                requirement.append(
+                    ThumbnailRequirement(width, height, method, "image/webp")
+                )
             else:
                 raise Exception(
                     "Unknown thumbnail mapping from %s to %s. This is a Synapse problem, please report!"

synapse/events/utils.py

@@ -507,6 +507,11 @@ def serialize_event(
             ):
                 d["unsigned"]["transaction_id"] = txn_id
 
+    # Beeper: include internal stream ordering as HS order unsigned hint
+    stream_ordering = getattr(e.internal_metadata, "stream_ordering", None)
+    if stream_ordering:
+        d["unsigned"]["com.beeper.hs.order"] = stream_ordering
+
     # invite_room_state and knock_room_state are a list of stripped room state events
     # that are meant to provide metadata about a room to an invitee/knocker. They are
     # intended to only be included in specific circumstances, such as down sync, and

synapse/events/validator.py

@@ -64,7 +64,7 @@ class EventValidator:
             event: The event to validate.
             config: The homeserver's configuration.
         """
-        self.validate_builder(event)
+        self.validate_builder(event, config)
 
         if event.format_version == EventFormatVersions.ROOM_V1_V2:
             EventID.from_string(event.event_id)
@@ -95,6 +95,12 @@ class EventValidator:
             # Note that only the client controlled portion of the event is
             # checked, since we trust the portions of the event we created.
             validate_canonicaljson(event.content)
+        if not 0 < event.origin_server_ts < 2**53:
+            raise SynapseError(400, "Event timestamp is out of range")
+
+        # meow: allow specific users to send potentially dangerous events.
+        if event.sender in config.meow.validation_override:
+            return
 
         if event.type == EventTypes.Aliases:
             if "aliases" in event.content:
@@ -193,7 +199,9 @@ class EventValidator:
                 errcode=Codes.BAD_JSON,
             )
 
-    def validate_builder(self, event: Union[EventBase, EventBuilder]) -> None:
+    def validate_builder(
+        self, event: Union[EventBase, EventBuilder], config: HomeServerConfig
+    ) -> None:
         """Validates that the builder/event has roughly the right format. Only
         checks values that we expect a proto event to have, rather than all the
         fields an event would have
@@ -211,6 +219,10 @@ class EventValidator:
         RoomID.from_string(event.room_id)
         UserID.from_string(event.sender)
 
+        # meow: allow specific users to send so-called invalid events
+        if event.sender in config.meow.validation_override:
+            return
+
         if event.type == EventTypes.Message:
             strings = ["body", "msgtype"]
 

synapse/handlers/directory.py

@@ -80,9 +80,11 @@ class DirectoryHandler:
     ) -> None:
         # general association creation for both human users and app services
 
-        for wchar in string.whitespace:
-            if wchar in room_alias.localpart:
-                raise SynapseError(400, "Invalid characters in room alias")
+        # meow: allow specific users to include anything in room aliases
+        if creator not in self.config.meow.validation_override:
+            for wchar in string.whitespace:
+                if wchar in room_alias.localpart:
+                    raise SynapseError(400, "Invalid characters in room alias")
 
         if ":" in room_alias.localpart:
             raise SynapseError(400, "Invalid character in room alias localpart: ':'.")
@@ -127,7 +129,10 @@ class DirectoryHandler:
         user_id = requester.user.to_string()
         room_alias_str = room_alias.to_string()
 
-        if len(room_alias_str) > MAX_ALIAS_LENGTH:
+        if (
+            user_id not in self.hs.config.meow.validation_override
+            and len(room_alias_str) > MAX_ALIAS_LENGTH
+        ):
             raise SynapseError(
                 400,
                 "Can't create aliases longer than %s characters" % MAX_ALIAS_LENGTH,
@@ -169,7 +174,7 @@ class DirectoryHandler:
 
             if not self.config.roomdirectory.is_alias_creation_allowed(
                 user_id, room_id, room_alias_str
-            ):
+            ) and not is_admin:
                 # Let's just return a generic message, as there may be all sorts of
                 # reasons why we said no. TODO: Allow configurable error messages
                 # per alias creation rule?
@@ -505,7 +510,7 @@ class DirectoryHandler:
 
             if not self.config.roomdirectory.is_publishing_room_allowed(
                 user_id, room_id, room_aliases
-            ):
+            ) and not await self.auth.is_server_admin(requester):
                 # Let's just return a generic message, as there may be all sorts of
                 # reasons why we said no. TODO: Allow configurable error messages
                 # per alias creation rule?

synapse/handlers/federation.py

@@ -1452,7 +1452,7 @@ class FederationHandler:
                 room_version_obj, event_dict
             )
 
-            EventValidator().validate_builder(builder)
+            EventValidator().validate_builder(builder, self.hs.config)
 
             # Try several times, it could fail with PartialStateConflictError
             # in send_membership_event, cf comment in except block.
@@ -1617,7 +1617,7 @@ class FederationHandler:
         builder = self.event_builder_factory.for_room_version(
             room_version_obj, event_dict
         )
-        EventValidator().validate_builder(builder)
+        EventValidator().validate_builder(builder, self.hs.config)
 
         (
             event,

synapse/handlers/message.py

@@ -672,7 +672,7 @@ class EventCreationHandler:
             room_version_obj, event_dict
         )
 
-        self.validator.validate_builder(builder)
+        self.validator.validate_builder(builder, self.config)
 
         is_exempt = await self._is_exempt_from_privacy_policy(builder, requester)
         if require_consent and not is_exempt:
@@ -1330,6 +1330,8 @@ class EventCreationHandler:
         Raises:
             SynapseError if the event is invalid.
         """
+        if event.sender in self.config.meow.validation_override:
+            return
 
         relation = relation_from_event(event)
         if not relation:
@@ -1749,7 +1751,8 @@ class EventCreationHandler:
 
             await self._maybe_kick_guest_users(event, context)
 
-            if event.type == EventTypes.CanonicalAlias:
+            validation_override = event.sender in self.config.meow.validation_override
+            if event.type == EventTypes.CanonicalAlias and not validation_override:
                 # Validate a newly added alias or newly added alt_aliases.
 
                 original_alias = None
@@ -2104,7 +2107,7 @@ class EventCreationHandler:
             builder = self.event_builder_factory.for_room_version(
                 original_event.room_version, third_party_result
             )
-            self.validator.validate_builder(builder)
+            self.validator.validate_builder(builder, self.config)
         except SynapseError as e:
             raise Exception(
                 "Third party rules module created an invalid event: " + e.msg,

synapse/handlers/read_marker.py

@@ -20,11 +20,12 @@
 #
 
 import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from synapse.api.constants import ReceiptTypes
 from synapse.api.errors import SynapseError
 from synapse.util.async_helpers import Linearizer
+from synapse.types import JsonDict
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -39,7 +40,11 @@ class ReadMarkerHandler:
         self.read_marker_linearizer = Linearizer(name="read_marker")
 
     async def received_client_read_marker(
-        self, room_id: str, user_id: str, event_id: str
+        self,
+        room_id: str,
+        user_id: str,
+        event_id: str,
+        extra_content: Optional[JsonDict] = None,
     ) -> None:
         """Updates the read marker for a given user in a given room if the event ID given
         is ahead in the stream relative to the current read marker.
@@ -71,7 +76,7 @@ class ReadMarkerHandler:
                     should_update = event_ordering > old_event_ordering
 
             if should_update:
-                content = {"event_id": event_id}
+                content = {"event_id": event_id, **(extra_content or {})}
                 await self.account_data_handler.add_account_data_to_room(
                     user_id, room_id, ReceiptTypes.FULLY_READ, content
                 )

synapse/handlers/receipts.py

@@ -181,6 +181,7 @@ class ReceiptsHandler:
         user_id: UserID,
         event_id: str,
         thread_id: Optional[str],
+        extra_content: Optional[JsonDict] = None,
     ) -> None:
         """Called when a client tells us a local user has read up to the given
         event_id in the room.
@@ -197,7 +198,7 @@ class ReceiptsHandler:
             user_id=user_id.to_string(),
             event_ids=[event_id],
             thread_id=thread_id,
-            data={"ts": int(self.clock.time_msec())},
+            data={"ts": int(self.clock.time_msec()), **(extra_content or {})},
         )
 
         is_new = await self._handle_new_receipts([receipt])

synapse/handlers/register.py

@@ -148,22 +148,25 @@ class RegistrationHandler:
         localpart: str,
         guest_access_token: Optional[str] = None,
         assigned_user_id: Optional[str] = None,
+        allow_invalid: bool = False,
         inhibit_user_in_use_error: bool = False,
     ) -> None:
-        if types.contains_invalid_mxid_characters(localpart):
-            raise SynapseError(
-                400,
-                "User ID can only contain characters a-z, 0-9, or '=_-./+'",
-                Codes.INVALID_USERNAME,
-            )
+        # meow: allow admins to register invalid user ids
+        if not allow_invalid:
+            if types.contains_invalid_mxid_characters(localpart):
+                raise SynapseError(
+                    400,
+                    "User ID can only contain characters a-z, 0-9, or '=_-./+'",
+                    Codes.INVALID_USERNAME,
+                )
 
-        if not localpart:
-            raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME)
+            if not localpart:
+                raise SynapseError(400, "User ID cannot be empty", Codes.INVALID_USERNAME)
 
-        if localpart[0] == "_":
-            raise SynapseError(
-                400, "User ID may not begin with _", Codes.INVALID_USERNAME
-            )
+            if localpart[0] == "_":
+                raise SynapseError(
+                    400, "User ID may not begin with _", Codes.INVALID_USERNAME
+                )
 
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
@@ -177,14 +180,16 @@ class RegistrationHandler:
                     "A different user ID has already been registered for this session",
                 )
 
-        self.check_user_id_not_appservice_exclusive(user_id)
+        # meow: allow admins to register reserved user ids and long user ids
+        if not allow_invalid:
+            self.check_user_id_not_appservice_exclusive(user_id)
 
-        if len(user_id) > MAX_USERID_LENGTH:
-            raise SynapseError(
-                400,
-                "User ID may not be longer than %s characters" % (MAX_USERID_LENGTH,),
-                Codes.INVALID_USERNAME,
-            )
+            if len(user_id) > MAX_USERID_LENGTH:
+                raise SynapseError(
+                    400,
+                    "User ID may not be longer than %s characters" % (MAX_USERID_LENGTH,),
+                    Codes.INVALID_USERNAME,
+                )
 
         users = await self.store.get_users_by_id_case_insensitive(user_id)
         if users:
@@ -290,7 +295,12 @@ class RegistrationHandler:
             await self.auth_blocking.check_auth_blocking(threepid=threepid)
 
         if localpart is not None:
-            await self.check_username(localpart, guest_access_token=guest_access_token)
+            allow_invalid = by_admin and self.hs.config.meow.admin_api_register_invalid
+            await self.check_username(
+                localpart,
+                guest_access_token=guest_access_token,
+                allow_invalid=allow_invalid,
+            )
 
             was_guest = guest_access_token is not None
 

synapse/handlers/room.py

@@ -894,11 +894,23 @@ class RoomCreationHandler:
 
         self._validate_room_config(config, visibility)
 
-        room_id = await self._generate_and_create_room_id(
-            creator_id=user_id,
-            is_public=is_public,
-            room_version=room_version,
-        )
+        if "room_id" in config:
+            room_id = config["room_id"]
+            try:
+                await self.store.store_room(
+                    room_id=room_id,
+                    room_creator_user_id=user_id,
+                    is_public=is_public,
+                    room_version=room_version,
+                )
+            except StoreError:
+                raise SynapseError(409, "Room ID already in use", errcode="M_CONFLICT")
+        else:
+            room_id = await self._generate_and_create_room_id(
+                creator_id=user_id,
+                is_public=is_public,
+                room_version=room_version,
+            )
 
         # Check whether this visibility value is blocked by a third party module
         allowed_by_third_party_rules = await (
@@ -915,7 +927,7 @@ class RoomCreationHandler:
                 room_aliases.append(room_alias.to_string())
             if not self.config.roomdirectory.is_publishing_room_allowed(
                 user_id, room_id, room_aliases
-            ):
+            ) and not is_requester_admin:
                 # allow room creation to continue but do not publish room
                 await self.store.set_room_is_public(room_id, False)
 
@@ -1190,7 +1202,7 @@ class RoomCreationHandler:
             # Please update the docs for `default_power_level_content_override` when
             # updating the `events` dict below
             power_level_content: JsonDict = {
-                "users": {creator_id: 100},
+                "users": {creator_id: 9001},
                 "users_default": 0,
                 "events": {
                     EventTypes.Name: 50,

synapse/handlers/room_member.py

@@ -797,26 +797,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
             content.pop("displayname", None)
             content.pop("avatar_url", None)
 
-        if len(content.get("displayname") or "") > MAX_DISPLAYNAME_LEN:
-            raise SynapseError(
-                400,
-                f"Displayname is too long (max {MAX_DISPLAYNAME_LEN})",
-                errcode=Codes.BAD_JSON,
-            )
-
-        if len(content.get("avatar_url") or "") > MAX_AVATAR_URL_LEN:
-            raise SynapseError(
-                400,
-                f"Avatar URL is too long (max {MAX_AVATAR_URL_LEN})",
-                errcode=Codes.BAD_JSON,
-            )
-
-        if "avatar_url" in content and content.get("avatar_url") is not None:
-            if not await self.profile_handler.check_avatar_size_and_mime_type(
-                content["avatar_url"],
-            ):
-                raise SynapseError(403, "This avatar is not allowed", Codes.FORBIDDEN)
-
         # The event content should *not* include the authorising user as
         # it won't be properly signed. Strip it out since it might come
         # back from a client updating a display name / avatar.

synapse/handlers/sync.py

@@ -1310,7 +1310,6 @@ class SyncHandler:
             for e in await sync_config.filter_collection.filter_room_state(
                 list(state.values())
             )
-            if e.type != EventTypes.Aliases  # until MSC2261 or alternative solution
         }
 
     async def _compute_state_delta_for_full_sync(

synapse/media/_base.py

@@ -86,6 +86,7 @@ INLINE_CONTENT_TYPES = [
     "text/csv",
     "application/json",
     "application/ld+json",
+    "application/pdf",
     # We allow some media files deemed as safe, which comes from the matrix-react-sdk.
     # https://github.com/matrix-org/matrix-react-sdk/blob/a70fcfd0bcf7f8c85986da18001ea11597989a7c/src/utils/blobs.ts#L51
     # SVGs are *intentionally* omitted.
@@ -229,7 +230,9 @@ def add_file_headers(
     # recommend caching as it's sensitive or private - or at least
     # select private. don't bother setting Expires as all our
     # clients are smart enough to be happy with Cache-Control
-    request.setHeader(b"Cache-Control", b"public,max-age=86400,s-maxage=86400")
+    request.setHeader(
+        b"Cache-Control", b"public,immutable,max-age=86400,s-maxage=86400"
+    )
 
     if file_size is not None:
         request.setHeader(b"Content-Length", b"%d" % (file_size,))

synapse/media/thumbnailer.py

@@ -65,7 +65,7 @@ class ThumbnailError(Exception):
 
 
 class Thumbnailer:
-    FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"}
+    FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG", "image/webp": "WEBP"}
 
     @staticmethod
     def set_limits(max_image_pixels: int) -> None:

synapse/push/httppusher.py

@@ -140,13 +140,6 @@ class HttpPusher(Pusher):
         url = self.data["url"]
         if not isinstance(url, str):
             raise PusherConfigException("'url' must be a string")
-        url_parts = urllib.parse.urlparse(url)
-        # Note that the specification also says the scheme must be HTTPS, but
-        # it isn't up to the homeserver to verify that.
-        if url_parts.path != "/_matrix/push/v1/notify":
-            raise PusherConfigException(
-                "'url' must have a path of '/_matrix/push/v1/notify'"
-            )
 
         self.url = url
         self.http_client = hs.get_proxied_blocklisted_http_client()

synapse/rest/client/read_marker.py

@@ -80,12 +80,16 @@ class ReadMarkerRestServlet(RestServlet):
             # TODO Add validation to reject non-string event IDs.
             if not event_id:
                 continue
+            extra_content = body.get(
+                receipt_type.replace("m.", "com.beeper.") + ".extra", None
+            )
 
             if receipt_type == ReceiptTypes.FULLY_READ:
                 await self.read_marker_handler.received_client_read_marker(
                     room_id,
                     user_id=requester.user.to_string(),
                     event_id=event_id,
+                    extra_content=extra_content,
                 )
             else:
                 await self.receipts_handler.received_client_receipt(
@@ -95,6 +99,7 @@ class ReadMarkerRestServlet(RestServlet):
                     event_id=event_id,
                     # Setting the thread ID is not possible with the /read_markers endpoint.
                     thread_id=None,
+                    extra_content=extra_content,
                 )
 
         return 200, {}

synapse/rest/client/receipts.py

@@ -73,7 +73,7 @@ class ReceiptRestServlet(RestServlet):
                 f"Receipt type must be {', '.join(self._known_receipt_types)}",
             )
 
-        body = parse_json_object_from_request(request)
+        body = parse_json_object_from_request(request, allow_empty_body=False)
 
         # Pull the thread ID, if one exists.
         thread_id = None
@@ -110,6 +110,7 @@ class ReceiptRestServlet(RestServlet):
                 room_id,
                 user_id=requester.user.to_string(),
                 event_id=event_id,
+                extra_content=body,
             )
         else:
             await self.receipts_handler.received_client_receipt(
@@ -118,6 +119,7 @@ class ReceiptRestServlet(RestServlet):
                 user_id=requester.user,
                 event_id=event_id,
                 thread_id=thread_id,
+                extra_content=body,
             )
 
         return 200, {}

synapse/rest/client/room.py

@@ -342,6 +342,7 @@ class RoomSendEventRestServlet(TransactionRestServlet):
         super().__init__(hs)
         self.event_creation_handler = hs.get_event_creation_handler()
         self.auth = hs.get_auth()
+        self.hs = hs
 
     def register(self, http_server: HttpServer) -> None:
         # /rooms/$roomid/send/$event_type[/$txn_id]
@@ -365,7 +366,10 @@ class RoomSendEventRestServlet(TransactionRestServlet):
             "sender": requester.user.to_string(),
         }
 
-        if requester.app_service:
+        if (
+            requester.app_service
+            or requester.user.to_string() in self.hs.config.meow.timestamp_override
+        ):
             origin_server_ts = parse_integer(request, "ts")
             if origin_server_ts is not None:
                 event_dict["origin_server_ts"] = origin_server_ts

synapse/storage/controllers/__init__.py

@@ -45,6 +45,7 @@ class StorageControllers:
         # rewrite all the existing code to split it into high vs low level
         # interfaces.
         self.main = stores.main
+        self.hs = hs
 
         self.purge_events = PurgeEventsStorageController(hs, stores)
         self.state = StateStorageController(hs, stores)

synapse/visibility.py

@@ -139,6 +139,10 @@ async def filter_events_for_client(
                 room_id
             ] = await storage.main.get_retention_policy_for_room(room_id)
 
+    # meow: let admins see secret events like org.matrix.dummy_event, m.room.aliases
+    #       and events expired by the retention policy.
+    filter_override = user_id in storage.hs.config.meow.filter_override
+
     def allowed(event: EventBase) -> Optional[EventBase]:
         state_after_event = event_id_to_state.get(event.event_id)
         filtered = _check_client_allowed_to_see_event(
@@ -152,6 +156,7 @@ async def filter_events_for_client(
             state=state_after_event,
             is_peeking=is_peeking,
             sender_erased=erased_senders.get(event.sender, False),
+            filter_override=filter_override,
         )
         if filtered is None:
             return None
@@ -328,6 +333,7 @@ def _check_client_allowed_to_see_event(
     retention_policy: RetentionPolicy,
     state: Optional[StateMap[EventBase]],
     sender_erased: bool,
+    filter_override: bool,
 ) -> Optional[EventBase]:
     """Check with the given user is allowed to see the given event
 
@@ -344,6 +350,7 @@ def _check_client_allowed_to_see_event(
         retention_policy: The retention policy of the room
         state: The state at the event, unless its an outlier
         sender_erased: Whether the event sender has been marked as "erased"
+        filter_override: meow
 
     Returns:
         None if the user cannot see this event at all
@@ -357,7 +364,7 @@ def _check_client_allowed_to_see_event(
     # because, if this is not the case, we're probably only checking if the users can
     # see events in the room at that point in the DAG, and that shouldn't be decided
     # on those checks.
-    if filter_send_to_client:
+    if filter_send_to_client and not filter_override:
         if (
             _check_filter_send_to_client(event, clock, retention_policy, sender_ignored)
             == _CheckFilter.DENIED
@@ -367,6 +374,9 @@ def _check_client_allowed_to_see_event(
                 event.event_id,
             )
             return None
+    # meow: even with filter_override, we want to filter ignored users
+    elif filter_send_to_client and not event.is_state() and sender_ignored:
+        return None
 
     if event.event_id in always_include_ids:
         return event