Add a MXCUri class to make working with mxc uri's easier. (#13162)

This commit is contained in:
Andrew Morgan 2022-09-15 13:57:16 +01:00 committed by GitHub
parent 957e3d74fc
commit 918c74bfb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 53 additions and 74 deletions

1
changelog.d/13162.misc Normal file
View File

@ -0,0 +1 @@
Bump the minimum dependency of `matrix_common` to 1.3.0 to make use of the `MXCUri` class. Use `MXCUri` to simplify media retention test code.

10
poetry.lock generated
View File

@ -524,11 +524,11 @@ python-versions = ">=3.7"
[[package]] [[package]]
name = "matrix-common" name = "matrix-common"
version = "1.2.1" version = "1.3.0"
description = "Common utilities for Synapse, Sydent and Sygnal" description = "Common utilities for Synapse, Sydent and Sygnal"
category = "main" category = "main"
optional = false optional = false
python-versions = ">=3.6" python-versions = ">=3.7"
[package.dependencies] [package.dependencies]
attrs = "*" attrs = "*"
@ -1625,7 +1625,7 @@ url_preview = ["lxml"]
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.7.1" python-versions = "^3.7.1"
content-hash = "79cfa09d59f9f8b5ef24318fb860df1915f54328692aa56d04331ecbdd92a8cb" content-hash = "1b14fc274d9e2a495a7f864150f3ffcf4d9f585e09a67e53301ae4ef3c2f3e48"
[metadata.files] [metadata.files]
attrs = [ attrs = [
@ -2113,8 +2113,8 @@ markupsafe = [
{file = "MarkupSafe-2.1.0.tar.gz", hash = "sha256:80beaf63ddfbc64a0452b841d8036ca0611e049650e20afcb882f5d3c266d65f"}, {file = "MarkupSafe-2.1.0.tar.gz", hash = "sha256:80beaf63ddfbc64a0452b841d8036ca0611e049650e20afcb882f5d3c266d65f"},
] ]
matrix-common = [ matrix-common = [
{file = "matrix_common-1.2.1-py3-none-any.whl", hash = "sha256:946709c405944a0d4b1d73207b77eb064b6dbfc5d70a69471320b06d8ce98b20"}, {file = "matrix_common-1.3.0-py3-none-any.whl", hash = "sha256:524e2785b9b03be4d15f3a8a6b857c5b6af68791ffb1b9918f0ad299abc4db20"},
{file = "matrix_common-1.2.1.tar.gz", hash = "sha256:a99dcf02a6bd95b24a5a61b354888a2ac92bf2b4b839c727b8dd9da2cdfa3853"}, {file = "matrix_common-1.3.0.tar.gz", hash = "sha256:62e121cccd9f243417b57ec37a76dc44aeb198a7a5c67afd6b8275992ff2abd1"},
] ]
matrix-synapse-ldap3 = [ matrix-synapse-ldap3 = [
{file = "matrix-synapse-ldap3-0.2.2.tar.gz", hash = "sha256:b388d95693486eef69adaefd0fd9e84463d52fe17b0214a00efcaa669b73cb74"}, {file = "matrix-synapse-ldap3-0.2.2.tar.gz", hash = "sha256:b388d95693486eef69adaefd0fd9e84463d52fe17b0214a00efcaa669b73cb74"},

View File

@ -164,7 +164,7 @@ typing-extensions = ">=3.10.0.1"
cryptography = ">=3.4.7" cryptography = ">=3.4.7"
# ijson 3.1.4 fixes a bug with "." in property names # ijson 3.1.4 fixes a bug with "." in property names
ijson = ">=3.1.4" ijson = ">=3.1.4"
matrix-common = "^1.2.1" matrix-common = "^1.3.0"
# We need packaging.requirements.Requirement, added in 16.1. # We need packaging.requirements.Requirement, added in 16.1.
packaging = ">=16.1" packaging = ">=16.1"
# At the time of writing, we only use functions from the version `importlib.metadata` # At the time of writing, we only use functions from the version `importlib.metadata`

View File

@ -19,6 +19,8 @@ import shutil
from io import BytesIO from io import BytesIO
from typing import IO, TYPE_CHECKING, Dict, List, Optional, Set, Tuple from typing import IO, TYPE_CHECKING, Dict, List, Optional, Set, Tuple
from matrix_common.types.mxc_uri import MXCUri
import twisted.internet.error import twisted.internet.error
import twisted.web.http import twisted.web.http
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred
@ -186,7 +188,7 @@ class MediaRepository:
content: IO, content: IO,
content_length: int, content_length: int,
auth_user: UserID, auth_user: UserID,
) -> str: ) -> MXCUri:
"""Store uploaded content for a local user and return the mxc URL """Store uploaded content for a local user and return the mxc URL
Args: Args:
@ -219,7 +221,7 @@ class MediaRepository:
await self._generate_thumbnails(None, media_id, media_id, media_type) await self._generate_thumbnails(None, media_id, media_id, media_type)
return "mxc://%s/%s" % (self.server_name, media_id) return MXCUri(self.server_name, media_id)
async def get_local_media( async def get_local_media(
self, request: SynapseRequest, media_id: str, name: Optional[str] self, request: SynapseRequest, media_id: str, name: Optional[str]

View File

@ -101,6 +101,8 @@ class UploadResource(DirectServeJsonResource):
# the default 404, as that would just be confusing. # the default 404, as that would just be confusing.
raise SynapseError(400, "Bad content") raise SynapseError(400, "Bad content")
logger.info("Uploaded content with URI %r", content_uri) logger.info("Uploaded content with URI '%s'", content_uri)
respond_with_json(request, 200, {"content_uri": content_uri}, send_cors=True) respond_with_json(
request, 200, {"content_uri": str(content_uri)}, send_cors=True
)

View File

@ -13,7 +13,9 @@
# limitations under the License. # limitations under the License.
import io import io
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional
from matrix_common.types.mxc_uri import MXCUri
from twisted.test.proto_helpers import MemoryReactor from twisted.test.proto_helpers import MemoryReactor
@ -63,9 +65,9 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase):
last_accessed_ms: Optional[int], last_accessed_ms: Optional[int],
is_quarantined: Optional[bool] = False, is_quarantined: Optional[bool] = False,
is_protected: Optional[bool] = False, is_protected: Optional[bool] = False,
) -> str: ) -> MXCUri:
# "Upload" some media to the local media store # "Upload" some media to the local media store
mxc_uri = self.get_success( mxc_uri: MXCUri = self.get_success(
media_repository.create_content( media_repository.create_content(
media_type="text/plain", media_type="text/plain",
upload_name=None, upload_name=None,
@ -75,13 +77,11 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase):
) )
) )
media_id = mxc_uri.split("/")[-1]
# Set the last recently accessed time for this media # Set the last recently accessed time for this media
if last_accessed_ms is not None: if last_accessed_ms is not None:
self.get_success( self.get_success(
self.store.update_cached_last_access_time( self.store.update_cached_last_access_time(
local_media=(media_id,), local_media=(mxc_uri.media_id,),
remote_media=(), remote_media=(),
time_ms=last_accessed_ms, time_ms=last_accessed_ms,
) )
@ -92,7 +92,7 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase):
self.get_success( self.get_success(
self.store.quarantine_media_by_id( self.store.quarantine_media_by_id(
server_name=self.hs.config.server.server_name, server_name=self.hs.config.server.server_name,
media_id=media_id, media_id=mxc_uri.media_id,
quarantined_by="@theadmin:test", quarantined_by="@theadmin:test",
) )
) )
@ -101,18 +101,18 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase):
# Mark this media as protected from quarantine # Mark this media as protected from quarantine
self.get_success( self.get_success(
self.store.mark_local_media_as_safe( self.store.mark_local_media_as_safe(
media_id=media_id, media_id=mxc_uri.media_id,
safe=True, safe=True,
) )
) )
return media_id return mxc_uri
def _cache_remote_media_and_set_attributes( def _cache_remote_media_and_set_attributes(
media_id: str, media_id: str,
last_accessed_ms: Optional[int], last_accessed_ms: Optional[int],
is_quarantined: Optional[bool] = False, is_quarantined: Optional[bool] = False,
) -> str: ) -> MXCUri:
# Pretend to cache some remote media # Pretend to cache some remote media
self.get_success( self.get_success(
self.store.store_cached_remote_media( self.store.store_cached_remote_media(
@ -146,7 +146,7 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase):
) )
) )
return media_id return MXCUri(self.remote_server_name, media_id)
# Start with the local media store # Start with the local media store
self.local_recently_accessed_media = _create_media_and_set_attributes( self.local_recently_accessed_media = _create_media_and_set_attributes(
@ -214,28 +214,16 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase):
# Remote media should be unaffected. # Remote media should be unaffected.
self._assert_if_mxc_uris_purged( self._assert_if_mxc_uris_purged(
purged=[ purged=[
( self.local_not_recently_accessed_media,
self.hs.config.server.server_name, self.local_never_accessed_media,
self.local_not_recently_accessed_media,
),
(self.hs.config.server.server_name, self.local_never_accessed_media),
], ],
not_purged=[ not_purged=[
(self.hs.config.server.server_name, self.local_recently_accessed_media), self.local_recently_accessed_media,
( self.local_not_recently_accessed_quarantined_media,
self.hs.config.server.server_name, self.local_not_recently_accessed_protected_media,
self.local_not_recently_accessed_quarantined_media, self.remote_recently_accessed_media,
), self.remote_not_recently_accessed_media,
( self.remote_not_recently_accessed_quarantined_media,
self.hs.config.server.server_name,
self.local_not_recently_accessed_protected_media,
),
(self.remote_server_name, self.remote_recently_accessed_media),
(self.remote_server_name, self.remote_not_recently_accessed_media),
(
self.remote_server_name,
self.remote_not_recently_accessed_quarantined_media,
),
], ],
) )
@ -261,49 +249,35 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase):
# Remote media accessed <30 days ago should still exist. # Remote media accessed <30 days ago should still exist.
self._assert_if_mxc_uris_purged( self._assert_if_mxc_uris_purged(
purged=[ purged=[
(self.remote_server_name, self.remote_not_recently_accessed_media), self.remote_not_recently_accessed_media,
], ],
not_purged=[ not_purged=[
(self.remote_server_name, self.remote_recently_accessed_media), self.remote_recently_accessed_media,
(self.hs.config.server.server_name, self.local_recently_accessed_media), self.local_recently_accessed_media,
( self.local_not_recently_accessed_media,
self.hs.config.server.server_name, self.local_not_recently_accessed_quarantined_media,
self.local_not_recently_accessed_media, self.local_not_recently_accessed_protected_media,
), self.remote_not_recently_accessed_quarantined_media,
( self.local_never_accessed_media,
self.hs.config.server.server_name,
self.local_not_recently_accessed_quarantined_media,
),
(
self.hs.config.server.server_name,
self.local_not_recently_accessed_protected_media,
),
(
self.remote_server_name,
self.remote_not_recently_accessed_quarantined_media,
),
(self.hs.config.server.server_name, self.local_never_accessed_media),
], ],
) )
def _assert_if_mxc_uris_purged( def _assert_if_mxc_uris_purged(
self, purged: Iterable[Tuple[str, str]], not_purged: Iterable[Tuple[str, str]] self, purged: Iterable[MXCUri], not_purged: Iterable[MXCUri]
) -> None: ) -> None:
def _assert_mxc_uri_purge_state( def _assert_mxc_uri_purge_state(mxc_uri: MXCUri, expect_purged: bool) -> None:
server_name: str, media_id: str, expect_purged: bool
) -> None:
"""Given an MXC URI, assert whether it has been purged or not.""" """Given an MXC URI, assert whether it has been purged or not."""
if server_name == self.hs.config.server.server_name: if mxc_uri.server_name == self.hs.config.server.server_name:
found_media_dict = self.get_success( found_media_dict = self.get_success(
self.store.get_local_media(media_id) self.store.get_local_media(mxc_uri.media_id)
) )
else: else:
found_media_dict = self.get_success( found_media_dict = self.get_success(
self.store.get_cached_remote_media(server_name, media_id) self.store.get_cached_remote_media(
mxc_uri.server_name, mxc_uri.media_id
)
) )
mxc_uri = f"mxc://{server_name}/{media_id}"
if expect_purged: if expect_purged:
self.assertIsNone( self.assertIsNone(
found_media_dict, msg=f"{mxc_uri} unexpectedly not purged" found_media_dict, msg=f"{mxc_uri} unexpectedly not purged"
@ -315,7 +289,7 @@ class MediaRetentionTestCase(unittest.HomeserverTestCase):
) )
# Assert that the given MXC URIs have either been correctly purged or not. # Assert that the given MXC URIs have either been correctly purged or not.
for server_name, media_id in purged: for mxc_uri in purged:
_assert_mxc_uri_purge_state(server_name, media_id, expect_purged=True) _assert_mxc_uri_purge_state(mxc_uri, expect_purged=True)
for server_name, media_id in not_purged: for mxc_uri in not_purged:
_assert_mxc_uri_purge_state(server_name, media_id, expect_purged=False) _assert_mxc_uri_purge_state(mxc_uri, expect_purged=False)