From 48fd7037727696f379205e6e60f7d7cb078da3e1 Mon Sep 17 00:00:00 2001 From: CortexPE Date: Mon, 21 Jun 2021 17:24:02 +0800 Subject: [PATCH 1/8] Patch #90 --- pantalaimon/daemon.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index b238ba7..71f474a 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -835,9 +835,7 @@ class ProxyDaemon: body=await response.read(), ) - def _get_upload_and_media_info(self, content_key, content): - content_uri = content[content_key] - + def _get_upload_and_media_info(self, content_uri: str): try: upload_info = self.upload_info[content_uri] except KeyError: @@ -847,7 +845,6 @@ class ProxyDaemon: self.upload_info[content_uri] = upload_info - content_uri = content[content_key] mxc = urlparse(content_uri) mxc_server = mxc.netloc.strip("/") mxc_path = mxc.path.strip("/") @@ -860,8 +857,8 @@ class ProxyDaemon: return upload_info, media_info - async def _map_decrypted_uri(self, content_key, content, request, client): - upload_info, media_info = self._get_upload_and_media_info(content_key, content) + async def _decrypt_uri(self, content_uri, client): + upload_info, media_info = self._get_upload_and_media_info(content_uri) if not upload_info or not media_info: raise NotDecryptedAvailableError @@ -877,7 +874,7 @@ class ProxyDaemon: decrypted_upload, _ = await client.upload( data_provider=BufferedReader(BytesIO(decrypted_file)), - content_type=response.content_type, + content_type=upload_info.mimetype, filename=upload_info.filename, encrypt=False, filesize=len(decrypted_file), @@ -886,9 +883,7 @@ class ProxyDaemon: if not isinstance(decrypted_upload, UploadResponse): raise NotDecryptedAvailableError - content[content_key] = decrypted_upload.content_uri - - return content + return decrypted_upload.content_uri async def send_message(self, request): access_token = self.get_access_token(request) @@ -930,9 +925,8 @@ class ProxyDaemon: or msgtype == "m.room.avatar" ): try: - content = await self._map_decrypted_uri( - "url", content, request, client - ) + content["url"] = await self._decrypt_uri(content["url"], client) + content["info"]["thumbnail_url"] = await self._decrypt_uri(content["info"]["thumbnail_url"], client) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) @@ -952,9 +946,7 @@ class ProxyDaemon: content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar" ): - upload_info, media_info = self._get_upload_and_media_info( - "url", content - ) + upload_info, media_info = self._get_upload_and_media_info(content["url"]) if not upload_info or not media_info: response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified @@ -973,7 +965,7 @@ class ProxyDaemon: thumb_upload_info, thumb_media_info, ) = self._get_upload_and_media_info( - "thumbnail_url", content["info"] + content["info"]["thumbnail_url"] ) if thumb_upload_info and thumb_media_info: thumb_media_info.to_thumbnail( @@ -1272,9 +1264,7 @@ class ProxyDaemon: return self._not_json try: - content = await self._map_decrypted_uri( - "avatar_url", content, request, client - ) + content["avatar_url"] = await self._decrypt_uri(content["avatar_url"], client) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) From c455b37b67573531fd0f96a292d67fac32173b09 Mon Sep 17 00:00:00 2001 From: CortexPE Date: Mon, 21 Jun 2021 17:32:51 +0800 Subject: [PATCH 2/8] prevent touching thumbnail info when not needed --- pantalaimon/daemon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 71f474a..a2f5a37 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -926,7 +926,8 @@ class ProxyDaemon: ): try: content["url"] = await self._decrypt_uri(content["url"], client) - content["info"]["thumbnail_url"] = await self._decrypt_uri(content["info"]["thumbnail_url"], client) + if "info" in content and "thumbnail_url" in content["info"]: + content["info"]["thumbnail_url"] = await self._decrypt_uri(content["info"]["thumbnail_url"], client) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) From 95bf21765708d4c85257f882cfbfde2487f1e55b Mon Sep 17 00:00:00 2001 From: CortexPE Date: Fri, 2 Jul 2021 19:45:40 +0800 Subject: [PATCH 3/8] patch KeyError on sync --- pantalaimon/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index b2d2a72..76d31a2 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -905,7 +905,7 @@ class PanClient(AsyncClient): self.handle_to_device_from_sync_body(body) - for room_id, room_dict in body["rooms"]["join"].items(): + for room_id, room_dict in body.get("rooms", {}).get("join", {}).items(): try: if not self.rooms[room_id].encrypted: logger.info( @@ -920,7 +920,7 @@ class PanClient(AsyncClient): # pan sync stream did. Let's assume that the room is encrypted. pass - for event in room_dict["timeline"]["events"]: + for event in room_dict.get("timeline", {}).get("events", []): if "type" not in event: continue From c36aca183b1cbac7eadeac028744151196c34ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 6 Jul 2021 10:41:23 +0200 Subject: [PATCH 4/8] pantalaimon: Bump our version --- CHANGELOG.md | 14 +++++++++++++- pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7403103..f3a6066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## 0.9.3 2021-05-14 +## 0.10.1 2021-07-06 + +### Fixed + +- [[#100]] Don't require the rooms dicts in the sync response +- [[#99]] Thumbnails not generating for media uploaded in unencrypted rooms + whole LRU cache when it shouldn't + +[#100]: https://github.com/matrix-org/pantalaimon/pull/100 +[#99]: https://github.com/matrix-org/pantalaimon/pull/99 + + +## 0.10.0 2021-05-14 ### Added diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 271a706..84e9429 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -262,7 +262,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.10.0", prog_name="pantalaimon") +@click.version_option(version="0.10.1", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 26344a1..7f76bf9 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.10.0", prog_name="panctl") +@click.version_option(version="0.10.1", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index a604a44..7f7d78f 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.10.0", + version="0.10.1", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 340dbf2eb6d2c3f3a20f985bdd25cca0934bec78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Jul 2021 10:44:03 +0200 Subject: [PATCH 5/8] pantalaimon: Prevent the sending of messages if the client didn't sync Pantalaimon depends on the room state to be present to decide correctly if a message should be encrypted or now. The room state is fetched from the server every time we restart, this means that failure to fetch room states from the server would hinder us from doing the correct decision. This patch prevents a downgrade to sending unencrypted messages if we're unable to sync at least once with the server. --- pantalaimon/client.py | 5 ++++- pantalaimon/daemon.py | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index 76d31a2..e0188f2 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -410,6 +410,10 @@ class PanClient(AsyncClient): except (asyncio.CancelledError, KeyboardInterrupt): return + @property + def has_been_synced(self) -> bool: + self.last_sync_token is not None + async def sync_tasks(self, response): if self.index: await self.index.commit_events() @@ -540,7 +544,6 @@ class PanClient(AsyncClient): timeout = 30000 sync_filter = {"room": {"state": {"lazy_load_members": True}}} next_batch = self.pan_store.load_token(self.server_name, self.user_id) - self.last_sync_token = next_batch # We don't store any room state so initial sync needs to be with the # full_state parameter. Subsequent ones are normal. diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index a2f5a37..bb9b8e5 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -902,7 +902,22 @@ class ProxyDaemon: room = client.rooms[room_id] encrypt = room.encrypted except KeyError: - return await self.forward_to_web(request, token=client.access_token) + if client.has_been_synced: + return await self.forward_to_web(request, token=client.access_token) + else: + logger.error( + "The internal Pantalaimon client did not manage " + "to sync with the server." + ) + return web.json_response( + { + "errcode": "M_UNKNOWN", + "error": "The pantalaimon client did not manage to sync with " + "the server", + }, + headers=CORS_HEADERS, + status=500, + ) # Don't encrypt reactions for now - they are weird and clients # need to support them like this. From f875499d6b0a9d968270872a4a81527517dd703e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Jul 2021 11:05:02 +0200 Subject: [PATCH 6/8] pantalaimon: Fix some formatting issues --- pantalaimon/daemon.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index bb9b8e5..bdba6a8 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -942,7 +942,9 @@ class ProxyDaemon: try: content["url"] = await self._decrypt_uri(content["url"], client) if "info" in content and "thumbnail_url" in content["info"]: - content["info"]["thumbnail_url"] = await self._decrypt_uri(content["info"]["thumbnail_url"], client) + content["info"]["thumbnail_url"] = await self._decrypt_uri( + content["info"]["thumbnail_url"], client + ) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) @@ -962,7 +964,9 @@ class ProxyDaemon: content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar" ): - upload_info, media_info = self._get_upload_and_media_info(content["url"]) + upload_info, media_info = self._get_upload_and_media_info( + content["url"] + ) if not upload_info or not media_info: response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified @@ -1280,7 +1284,9 @@ class ProxyDaemon: return self._not_json try: - content["avatar_url"] = await self._decrypt_uri(content["avatar_url"], client) + content["avatar_url"] = await self._decrypt_uri( + content["avatar_url"], client + ) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) From a9f6ad2c7e08c72e8274a6bdf2954edf1c2d6b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Jul 2021 12:25:39 +0200 Subject: [PATCH 7/8] pantalaimon: Don't forward messages if we think the room isn't joined --- pantalaimon/daemon.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index bdba6a8..bbec91a 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -897,13 +897,22 @@ class ProxyDaemon: room_id = request.match_info["room_id"] - # The room is not in the joined rooms list, just forward it. try: room = client.rooms[room_id] encrypt = room.encrypted except KeyError: + # The room is not in the joined rooms list, either the pan client + # didn't manage to sync the state or we're not joined, in either + # case send an error response. if client.has_been_synced: - return await self.forward_to_web(request, token=client.access_token) + return web.json_response( + { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event." + }, + headers=CORS_HEADERS, + status=403, + ) else: logger.error( "The internal Pantalaimon client did not manage " From e3be7bee3be0ceeae07b067ec4c90f5849b9763f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Jul 2021 13:48:16 +0200 Subject: [PATCH 8/8] pantalaimon: Bump our version --- CHANGELOG.md | 9 +++++++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 4 ++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a6066..c0dda50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.10.2 2021-07-14 + +### Fixed + +- [[#103]] Prevent E2EE downgrade on failed syncs + +[#103]: https://github.com/matrix-org/pantalaimon/pull/103 + + ## 0.10.1 2021-07-06 ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 84e9429..fe5e167 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -262,7 +262,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.10.1", prog_name="pantalaimon") +@click.version_option(version="0.10.2", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 7f76bf9..fdb9921 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.10.1", prog_name="panctl") +@click.version_option(version="0.10.2", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 7f7d78f..b13317a 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.10.1", + version="0.10.2", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.14, < 0.19" + "matrix-nio[e2e] >= 0.18, < 0.19" ], extras_require={ "ui": [