From 0be0b5301bd4ce985a34cbf5bacf1c5e570953f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 2 Mar 2020 16:20:33 +0100 Subject: [PATCH 001/146] client: Log the decrypted sender, room id and event id at the info level. --- pantalaimon/client.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index 47ce515..78d00e5 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -819,6 +819,13 @@ class PanClient(AsyncClient): try: decrypted_event = self.decrypt_event(event) logger.debug("Decrypted event: {}".format(decrypted_event)) + logger.info( + "Decrypted event from {} in {}, event id: {}".format( + decrypted_event.sender, + decrypted_event.room_id, + decrypted_event.event_id, + ) + ) event_dict.update(decrypted_event.source) event_dict["decrypted"] = True From 6c4fb03744c219c14778f75969d8197e2fdce9c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 16 Mar 2020 09:48:39 +0100 Subject: [PATCH 002/146] README: Link to the man pages from the readme. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 17b6cc6..af1563b 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,8 @@ Usage While pantalaimon is a daemon, it is meant to be run as your own user. It won't verify devices for you automatically, unless configured to do so, and requires -user interaction to verify, ignore or blacklist devices. +user interaction to verify, ignore or blacklist devices. A more complete +description of Pantalaimon can be found in the [man page](docs/man/pantalaimon.8.md). Pantalaimon requires a configuration file to run. The configuration file specifies one or more homeservers for pantalaimon to connect to. @@ -114,7 +115,7 @@ ListenPort = 8009 The configuration file should be placed in `~/.config/pantalaimon/pantalaimon.conf`. The full documentation for the pantalaimons configuration can be found in -the man page `pantalaimon(5)`. +the [man page](docs/man/pantalaimon.5.md) `pantalaimon(5)`. Now that pantalaimon is configured it can be run: From 4b6b61c4be6eebf34d0c203e8902f0a8e7d4c20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 21 Apr 2020 12:04:02 +0200 Subject: [PATCH 003/146] pantalaimon: Modify the media events so they contain the decrypted URLs as well. This closes 43. --- pantalaimon/client.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index 78d00e5..f20b792 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -827,6 +827,17 @@ class PanClient(AsyncClient): ) ) + if isinstance(decrypted_event, RoomEncryptedMedia): + decrypted_event.source["content"]["url"] = decrypted_event.url + + if ( + "info" in decrypted_event.source + and "thumbnail_file" in decrypted_event.source["info"] + ): + decrypted_event.source["content"]["info"][ + "thumbnail_url" + ] = decrypted_event.source["thumbnail_file"].get("url") + event_dict.update(decrypted_event.source) event_dict["decrypted"] = True event_dict["verified"] = decrypted_event.verified From 9e53315540c159aa80d8017dc0c763c41bcae0fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 22 Apr 2020 11:25:02 +0200 Subject: [PATCH 004/146] main: Add the v1 media endpoints. This uses the same logic as the r0 enpdpoints but some clients might be stuck using the v1 ones. --- pantalaimon/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index c56b17a..7db5347 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -74,6 +74,13 @@ async def init(data_dir, server_conf, send_queue, recv_queue): web.get("/.well-known/matrix/client", proxy.well_known), web.post("/_matrix/client/r0/search", proxy.search), web.options("/_matrix/client/r0/search", proxy.search_opts), + web.get( + "/_matrix/media/v1/download/{server_name}/{media_id}", proxy.download + ), + web.get( + "/_matrix/media/v1/download/{server_name}/{media_id}/{file_name}", + proxy.download, + ), web.get( "/_matrix/media/r0/download/{server_name}/{media_id}", proxy.download ), From a2994b0a55fbd95b8cc51ee75a032363093d842d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 22 Apr 2020 11:25:54 +0200 Subject: [PATCH 005/146] client: Simplify the thumbnail URL setting. --- pantalaimon/client.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index f20b792..e705fda 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -830,13 +830,10 @@ class PanClient(AsyncClient): if isinstance(decrypted_event, RoomEncryptedMedia): decrypted_event.source["content"]["url"] = decrypted_event.url - if ( - "info" in decrypted_event.source - and "thumbnail_file" in decrypted_event.source["info"] - ): + if decrypted_event.thumbnail_url: decrypted_event.source["content"]["info"][ "thumbnail_url" - ] = decrypted_event.source["thumbnail_file"].get("url") + ] = decrypted_event.thumbnail_url event_dict.update(decrypted_event.source) event_dict["decrypted"] = True From e74a0b92ee6bc65a54ea59e8650d65841b2b4d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 22 Apr 2020 11:45:55 +0200 Subject: [PATCH 006/146] pantalaimon: Add a changelog. --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f5bfe4e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- Added media endpoint handling to the /media/v1 path. + +### Fixed +- Modify media events so they contain the unencrypted URL fields as well. From 497ca67d3d72f6031ee5c3cf009c5c64f1c96c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 24 Apr 2020 11:53:41 +0200 Subject: [PATCH 007/146] setup.py: Pin our versions. --- setup.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 941b3b9..24a7102 100644 --- a/setup.py +++ b/setup.py @@ -18,14 +18,14 @@ setup( license="Apache License, Version 2.0", packages=find_packages(), install_requires=[ - "attrs", - "aiohttp", - "appdirs", - "click", - "keyring", - "logbook", - "peewee", - "janus", + "attrs <= 19.3.0", + "aiohttp <= 3.6.2", + "appdirs <= 1.4.3", + "click <= 7.1.1", + "keyring <= 21.2.0", + "logbook <= 1.5.3", + "peewee <= 3.1.13", + "janus <= 0.4.0", "cachetools >= 3.0.0" "prompt_toolkit>2<4", "typing;python_version<'3.5'", @@ -33,10 +33,10 @@ setup( ], extras_require={ "ui": [ - "dbus-python", - "PyGObject", - "pydbus", - "notify2", + "dbus-python <= 1.2.16", + "PyGObject <= 3.36.0", + "pydbus <= 0.6.0", + "notify2 <= 0.3.1", ] }, entry_points={ From 04569f9467a6f28ec32e7f2fa2f01723d84d91da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 24 Apr 2020 12:15:37 +0200 Subject: [PATCH 008/146] setup.py: Fix the required peewee version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 24a7102..6be223b 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( "click <= 7.1.1", "keyring <= 21.2.0", "logbook <= 1.5.3", - "peewee <= 3.1.13", + "peewee <= 3.13.3", "janus <= 0.4.0", "cachetools >= 3.0.0" "prompt_toolkit>2<4", From ed942187d4785c6a400d4b602d1bbd1d555bb8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Apr 2020 16:23:10 +0200 Subject: [PATCH 009/146] main: Turn the daemon startup function async. This removes removes the loop argument from the janus queue creation. This fixes #49. --- pantalaimon/main.py | 71 +++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 7db5347..270b957 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -132,24 +132,7 @@ async def message_router(receive_queue, send_queue, proxies): await proxy.receive_message(message) -@click.command( - help=( - "pantalaimon is a reverse proxy for matrix homeservers that " - "transparently encrypts and decrypts messages for clients that " - "connect to pantalaimon." - ) -) -@click.version_option(version="0.5.1", prog_name="pantalaimon") -@click.option( - "--log-level", - type=click.Choice(["error", "warning", "info", "debug"]), - default=None, -) -@click.option("--debug-encryption", is_flag=True) -@click.option("-c", "--config", type=click.Path(exists=True)) -@click.option("--data-path", type=click.Path(exists=True)) -@click.pass_context -def main(context, log_level, debug_encryption, config, data_path): +async def daemon(context, log_level, debug_encryption, config, data_path): loop = asyncio.get_event_loop() conf_dir = user_config_dir("pantalaimon", "") @@ -185,8 +168,8 @@ def main(context, log_level, debug_encryption, config, data_path): if UI_ENABLED: from pantalaimon.ui import GlibT - pan_queue = janus.Queue(loop=loop) - ui_queue = janus.Queue(loop=loop) + pan_queue = janus.Queue() + ui_queue = janus.Queue() glib_thread = GlibT( pan_queue.sync_q, @@ -197,7 +180,7 @@ def main(context, log_level, debug_encryption, config, data_path): ) glib_fut = loop.run_in_executor(None, glib_thread.run) - message_router_task = loop.create_task( + message_router_task = asyncio.create_task( message_router(ui_queue.async_q, pan_queue.async_q, proxies) ) @@ -209,11 +192,8 @@ def main(context, log_level, debug_encryption, config, data_path): message_router_task = None try: - for server_conf in pan_conf.servers.values(): - proxy, runner, site = loop.run_until_complete( - init(data_dir, server_conf, pan_queue, ui_queue) - ) + proxy, runner, site = await init(data_dir, server_conf, pan_queue, ui_queue) servers.append((proxy, runner, site)) proxies.append(proxy) @@ -227,6 +207,8 @@ def main(context, log_level, debug_encryption, config, data_path): home = os.path.expanduser("~") os.chdir(home) + event = asyncio.Event() + def handler(signum, frame): raise KeyboardInterrupt @@ -238,23 +220,48 @@ def main(context, log_level, debug_encryption, config, data_path): f"======== Starting daemon for homeserver " f"{proxy.name} on {site.name} ========" ) - loop.run_until_complete(site.start()) + await site.start() click.echo("(Press CTRL+C to quit)") - loop.run_forever() - except KeyboardInterrupt: + await event.wait() + except (KeyboardInterrupt, asyncio.CancelledError): for _, runner, _ in servers: - loop.run_until_complete(runner.cleanup()) + await runner.cleanup() if glib_fut: - loop.run_until_complete(wait_for_glib(glib_thread, glib_fut)) + await wait_for_glib(glib_thread, glib_fut) if message_router_task: message_router_task.cancel() - loop.run_until_complete(asyncio.wait({message_router_task})) + await asyncio.wait({message_router_task}) - loop.close() + raise +@click.command( + help=( + "pantalaimon is a reverse proxy for matrix homeservers that " + "transparently encrypts and decrypts messages for clients that " + "connect to pantalaimon." + ) +) +@click.version_option(version="0.5.1", prog_name="pantalaimon") +@click.option( + "--log-level", + type=click.Choice(["error", "warning", "info", "debug"]), + default=None, +) +@click.option("--debug-encryption", is_flag=True) +@click.option("-c", "--config", type=click.Path(exists=True)) +@click.option("--data-path", type=click.Path(exists=True)) +@click.pass_context +def main(context, log_level, debug_encryption, config, data_path): + event = asyncio.Event() + try: + asyncio.run(daemon(context, log_level, debug_encryption, config, data_path)) + except (KeyboardInterrupt, asyncio.CancelledError): + pass + + return if __name__ == "__main__": main() From ea042fa92f16dcaf09f9592becdade81c8e28014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Apr 2020 16:28:51 +0200 Subject: [PATCH 010/146] ui: Don't run the glib loop if it was shut down already. --- pantalaimon/ui.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pantalaimon/ui.py b/pantalaimon/ui.py index 154a15d..25eb2f8 100644 --- a/pantalaimon/ui.py +++ b/pantalaimon/ui.py @@ -646,6 +646,9 @@ if UI_ENABLED: self.notifications = False GLib.timeout_add(100, self.message_callback) + if not self.loop: + return + self.loop.run() def stop(self): From c732fd71410ee97e50df122dcfed99ffc883d1a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Apr 2020 16:32:36 +0200 Subject: [PATCH 011/146] setup.py: Bump our supported Janus version. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6be223b..3b3605b 100644 --- a/setup.py +++ b/setup.py @@ -25,9 +25,9 @@ setup( "keyring <= 21.2.0", "logbook <= 1.5.3", "peewee <= 3.13.3", - "janus <= 0.4.0", + "janus <= 0.5.0", "cachetools >= 3.0.0" - "prompt_toolkit>2<4", + "prompt_toolkit > 2 < 4", "typing;python_version<'3.5'", "matrix-nio[e2e] >= 0.8.0" ], From bf8bded35409b41df231fb37785ce5a867003dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Apr 2020 16:48:43 +0200 Subject: [PATCH 012/146] tests: Don't use the loop argument for janus queues. --- tests/conftest.py | 4 ++-- tests/pan_client_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f473ad8..c79de19 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -118,8 +118,8 @@ async def pan_proxy_server(tempdir, aiohttp_server): config = ServerConfig(server_name, urlparse("https://example.org")) - pan_queue = janus.Queue(loop=loop) - ui_queue = janus.Queue(loop=loop) + pan_queue = janus.Queue() + ui_queue = janus.Queue() proxy = ProxyDaemon( config.name, diff --git a/tests/pan_client_test.py b/tests/pan_client_test.py index 4bfa844..7202698 100644 --- a/tests/pan_client_test.py +++ b/tests/pan_client_test.py @@ -27,7 +27,7 @@ ALICE_ID = "@alice:example.org" @pytest.fixture async def client(tmpdir, loop): store = PanStore(tmpdir) - queue = janus.Queue(loop=loop) + queue = janus.Queue() conf = ServerConfig("example", "https://exapmle.org") conf.history_fetch_delay = 0.1 From d58b76146251992eedb8accf9cf328fa1c6b1996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Apr 2020 16:49:06 +0200 Subject: [PATCH 013/146] pantalaimon: Fix some lint issues. --- pantalaimon/index.py | 2 -- pantalaimon/main.py | 3 ++- pantalaimon/panctl.py | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pantalaimon/index.py b/pantalaimon/index.py index 0f10044..5c8e02b 100644 --- a/pantalaimon/index.py +++ b/pantalaimon/index.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from importlib import util - class InvalidQueryError(Exception): pass diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 270b957..4799f3a 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -237,6 +237,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): raise + @click.command( help=( "pantalaimon is a reverse proxy for matrix homeservers that " @@ -255,7 +256,6 @@ async def daemon(context, log_level, debug_encryption, config, data_path): @click.option("--data-path", type=click.Path(exists=True)) @click.pass_context def main(context, log_level, debug_encryption, config, data_path): - event = asyncio.Event() try: asyncio.run(daemon(context, log_level, debug_encryption, config, data_path)) except (KeyboardInterrupt, asyncio.CancelledError): @@ -263,5 +263,6 @@ def main(context, log_level, debug_encryption, config, data_path): return + if __name__ == "__main__": main() diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 54fc2b3..9d9eb2d 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -530,7 +530,7 @@ class PanCtl: ) if user_list: - print(f" - Pan users:") + print(" - Pan users:") user_string = "\n".join(user_list) print_formatted_text(HTML(user_string)) @@ -541,11 +541,11 @@ class PanCtl: for device in devices: if device["trust_state"] == "verified": - trust_state = f"Verified" + trust_state = "Verified" elif device["trust_state"] == "blacklisted": - trust_state = f"Blacklisted" + trust_state = "Blacklisted" elif device["trust_state"] == "ignored": - trust_state = f"Ignored" + trust_state = "Ignored" else: trust_state = "Unset" From e79acad2a9820bcac2f6b6c1316bb0f69a2a0406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 29 Apr 2020 16:57:15 +0200 Subject: [PATCH 014/146] setup.py: Revert the prompt toolkit version string. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3b3605b..3209f15 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( "peewee <= 3.13.3", "janus <= 0.5.0", "cachetools >= 3.0.0" - "prompt_toolkit > 2 < 4", + "prompt_toolkit>2<4", "typing;python_version<'3.5'", "matrix-nio[e2e] >= 0.8.0" ], From b732938489e1ebaa2f7711efd4c770521a84441f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 10 May 2020 19:12:09 +0200 Subject: [PATCH 015/146] pantalaimon: Bump the version. --- CHANGELOG.md | 3 ++- pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bfe4e..8500e43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## 0.6.0 2020-05-10 ### Added +- Add support for Janus 0.5.0. - Added media endpoint handling to the /media/v1 path. ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 4799f3a..cc8e289 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -245,7 +245,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.5.1", prog_name="pantalaimon") +@click.version_option(version="0.6.0", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 9d9eb2d..b1893ec 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.5.1", prog_name="panctl") +@click.version_option(version="0.6.0", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 3209f15..32dfbb0 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.5.1", + version="0.6.0", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 90eed20bef1d1b85dc21bb461ae0b8f5a3c7d30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 May 2020 09:56:20 +0200 Subject: [PATCH 016/146] pantalaimon: Bump the version so dockerhub picks it up. --- CHANGELOG.md | 5 +++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8500e43..ec38c09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.6.1 2020-05-12 + +### Fixed +- Bump the version to trigger a docker hub build with the latest nio release. + ## 0.6.0 2020-05-10 ### Added diff --git a/pantalaimon/main.py b/pantalaimon/main.py index cc8e289..b0c8cbb 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -245,7 +245,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.6.0", prog_name="pantalaimon") +@click.version_option(version="0.6.1", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index b1893ec..f6e0d2d 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.6.0", prog_name="panctl") +@click.version_option(version="0.6.1", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 32dfbb0..95f3b9c 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.6.0", + version="0.6.1", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From d9a4b91d70a7c6375ccd000bd94b223f49ba2c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 21 May 2020 12:38:48 +0200 Subject: [PATCH 017/146] setup.py: Don't pin the patch versions. This closes: #51. --- setup.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 95f3b9c..5b99ae0 100644 --- a/setup.py +++ b/setup.py @@ -18,25 +18,25 @@ setup( license="Apache License, Version 2.0", packages=find_packages(), install_requires=[ - "attrs <= 19.3.0", - "aiohttp <= 3.6.2", - "appdirs <= 1.4.3", - "click <= 7.1.1", - "keyring <= 21.2.0", - "logbook <= 1.5.3", - "peewee <= 3.13.3", - "janus <= 0.5.0", - "cachetools >= 3.0.0" + "attrs <= 19.3", + "aiohttp <= 3.6", + "appdirs <= 1.4", + "click <= 7.1", + "keyring <= 21.2", + "logbook <= 1.5", + "peewee <= 3.13", + "janus <= 0.5", + "cachetools >= 3.0" "prompt_toolkit>2<4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.8.0" + "matrix-nio[e2e] <= 0.12" ], extras_require={ "ui": [ - "dbus-python <= 1.2.16", - "PyGObject <= 3.36.0", - "pydbus <= 0.6.0", - "notify2 <= 0.3.1", + "dbus-python <= 1.2", + "PyGObject <= 3.36", + "pydbus <= 0.6", + "notify2 <= 0.3", ] }, entry_points={ From 56fd6b7a389f591175e630bf911c160b2e855cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 27 May 2020 17:10:55 +0200 Subject: [PATCH 018/146] pantalaimon: Bump the version. --- CHANGELOG.md | 5 +++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec38c09..ca3fa00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.6.2 2020-05-27 + +### Fixed +- Don't require exact patch versions for our deps. + ## 0.6.1 2020-05-12 ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index b0c8cbb..d6db3d1 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -245,7 +245,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.6.1", prog_name="pantalaimon") +@click.version_option(version="0.6.2", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index f6e0d2d..08efc91 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.6.1", prog_name="panctl") +@click.version_option(version="0.6.2", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 5b99ae0..eabb128 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.6.1", + version="0.6.2", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 4da77c3162dd5360bf7704c0edd124802a6d1b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 27 May 2020 18:43:55 +0200 Subject: [PATCH 019/146] pantalaimon: Fix the deps to not contain incompabilities with nio. --- setup.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index eabb128..8e2e263 100644 --- a/setup.py +++ b/setup.py @@ -18,15 +18,15 @@ setup( license="Apache License, Version 2.0", packages=find_packages(), install_requires=[ - "attrs <= 19.3", - "aiohttp <= 3.6", - "appdirs <= 1.4", - "click <= 7.1", - "keyring <= 21.2", - "logbook <= 1.5", - "peewee <= 3.13", - "janus <= 0.5", - "cachetools >= 3.0" + "attrs >= 19.3.0", + "aiohttp >= 3.6.2", + "appdirs >= 1.4.4", + "click >= 7.1.2", + "keyring >= 21.2.1", + "logbook >= 1.5.3", + "peewee >= 3.13.1", + "janus >= 0.5", + "cachetools >= 3.0.0" "prompt_toolkit>2<4", "typing;python_version<'3.5'", "matrix-nio[e2e] <= 0.12" From c05adace8cdf91c7fbed0ae191f2a40ea9106258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 27 May 2020 19:14:40 +0200 Subject: [PATCH 020/146] pantalaimon: Fix the version of our aiohttp dependency. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e2e263..ecfcf38 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( packages=find_packages(), install_requires=[ "attrs >= 19.3.0", - "aiohttp >= 3.6.2", + "aiohttp >= 3.6, < 4.0", "appdirs >= 1.4.4", "click >= 7.1.2", "keyring >= 21.2.1", From 4bd1d1767f284b48bb6f98cc388643a28fb7345b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 28 May 2020 16:29:26 +0200 Subject: [PATCH 021/146] pantalaimon: Bump the version. --- CHANGELOG.md | 5 +++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca3fa00..1955f72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.6.3 2020-05-28 + +### Fixed +- Fix our dep requirements to avoid incompatibilities between nio and pantalaimon. + ## 0.6.2 2020-05-27 ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index d6db3d1..0d9206a 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -245,7 +245,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.6.2", prog_name="pantalaimon") +@click.version_option(version="0.6.3", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 08efc91..9bc3f76 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.6.2", prog_name="panctl") +@click.version_option(version="0.6.3", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index ecfcf38..26c3b12 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.6.2", + version="0.6.3", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 007185dd7813ede55b4b450928572b4ec4ca991e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 21 Jun 2020 20:46:21 +0200 Subject: [PATCH 022/146] setup.py: Bump the maximal supported nio version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 26c3b12..46d08d8 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0" "prompt_toolkit>2<4", "typing;python_version<'3.5'", - "matrix-nio[e2e] <= 0.12" + "matrix-nio[e2e] <= 0.13" ], extras_require={ "ui": [ From fae9330a6b2ea7a2dd66b7a9a5799e9fd0bf67cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 21 Jun 2020 20:46:53 +0200 Subject: [PATCH 023/146] pantalaimon: Bump the version. --- CHANGELOG.md | 5 +++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1955f72..3c18b04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.6.4 2020-06-21 + +### Changed +- Bump the maximal supported nio version. + ## 0.6.3 2020-05-28 ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 0d9206a..99387aa 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -245,7 +245,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.6.3", prog_name="pantalaimon") +@click.version_option(version="0.6.4", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 9bc3f76..80c49dc 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.6.3", prog_name="panctl") +@click.version_option(version="0.6.4", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 46d08d8..43422f1 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.6.3", + version="0.6.4", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From da08ccac5a8eac92a1fa8c7572fb06c949de96d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Sun, 21 Jun 2020 21:40:29 +0200 Subject: [PATCH 024/146] setup.py: Bump the nio version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43422f1..2d7f2fd 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0" "prompt_toolkit>2<4", "typing;python_version<'3.5'", - "matrix-nio[e2e] <= 0.13" + "matrix-nio[e2e] <= 0.14" ], extras_require={ "ui": [ From a1ce95076ecd80c880028691feeced8d28cacad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 24 Jun 2020 15:04:37 +0200 Subject: [PATCH 025/146] daemon: Catch POST requests that try to send out messages. Synapse seems to accept a POST requests on the _matrix/client/r0/rooms/$ROOM/send/m.room.message path. While this is not specced in any way people might shoot themselves in the foot by sending unencrypted messages to a room if they, by accident, use the wrong HTTP method. This fixes: #56 --- pantalaimon/daemon.py | 3 ++- pantalaimon/main.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 637e373..8ad2b93 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -19,6 +19,7 @@ import urllib.parse import concurrent.futures from json import JSONDecodeError from typing import Any, Dict +from uuid import uuid4 import aiohttp import attr @@ -817,7 +818,7 @@ class ProxyDaemon: return await self.forward_to_web(request, token=client.access_token) msgtype = request.match_info["event_type"] - txnid = request.match_info["txnid"] + txnid = request.match_info.get("txnid", uuid4()) try: content = await request.json() diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 99387aa..a3ef545 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -69,6 +69,10 @@ async def init(data_dir, server_conf, send_queue, recv_queue): r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}", proxy.send_message, ), + web.post( + r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}", + proxy.send_message, + ), web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter), web.post("/.well-known/matrix/client", proxy.well_known), web.get("/.well-known/matrix/client", proxy.well_known), From 3ec8ee92ba9d0e505d2f616e644385e17b631429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 2 Jul 2020 12:14:29 +0200 Subject: [PATCH 026/146] setup.py: Allow patch versions of nio. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d7f2fd..5dbc36d 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0" "prompt_toolkit>2<4", "typing;python_version<'3.5'", - "matrix-nio[e2e] <= 0.14" + "matrix-nio[e2e] >= 0.14, < 0.15" ], extras_require={ "ui": [ From d388a21b9b1f17b7f52790f79dd571d8e75a4543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 2 Jul 2020 12:19:59 +0200 Subject: [PATCH 027/146] pantalaimon: Bump the version. --- CHANGELOG.md | 9 +++++++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c18b04..278a1ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.6.5 2020-07-02 + +### Fixed + +- [[a1ce950]] Allow to send messages using a POST request since Synapse seems to + allow it. + +[a1ce950]: https://github.com/matrix-org/pantalaimon/commit/a1ce95076ecd80c880028691feeced8d28cacad9 + ## 0.6.4 2020-06-21 ### Changed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index a3ef545..896d29e 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -249,7 +249,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.6.4", prog_name="pantalaimon") +@click.version_option(version="0.6.5", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 80c49dc..92a062d 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.6.4", prog_name="panctl") +@click.version_option(version="0.6.5", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 5dbc36d..2eea792 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.6.4", + version="0.6.5", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 24b371be62972a688e928e9b65b0b86f1c2e7b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 20 Jul 2020 14:26:07 +0200 Subject: [PATCH 028/146] daemon: Sanitize the GET /rooms/{room_id}/messages filters as well. Fractal started to apply filters on their scroll-back requests, this sadly doesn't include encrypted messages so the scroll-back appeared to be broken for encrypted rooms. This patch modifies the filter sanitization logic, since Fractal also sends out a new filter format that we didn't support, and applies it to the GET /rooms/{room_id}/messages endpoint as well. --- pantalaimon/daemon.py | 47 ++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 8ad2b93..0e1bd97 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -411,29 +411,34 @@ class ProxyDaemon: return access_token + def sanitize_subfilter(self, request_filter: Dict[Any, Any]): + types_filter = request_filter.get("types", None) + + if types_filter: + if "m.room.encrypted" not in types_filter: + types_filter.append("m.room.encrypted") + + not_types_filter = request_filter.get("not_types", None) + + if not_types_filter: + try: + not_types_filter.remove("m.room.encrypted") + except ValueError: + pass + def sanitize_filter(self, sync_filter): # type: (Dict[Any, Any]) -> Dict[Any, Any] """Make sure that a filter isn't filtering encrypted messages.""" sync_filter = dict(sync_filter) room_filter = sync_filter.get("room", None) + self.sanitize_subfilter(sync_filter) + if room_filter: timeline_filter = room_filter.get("timeline", None) if timeline_filter: - types_filter = timeline_filter.get("types", None) - - if types_filter: - if "m.room.encrypted" not in types_filter: - types_filter.append("m.room.encrypted") - - not_types_filter = timeline_filter.get("not_types", None) - - if not_types_filter: - try: - not_types_filter.remove("m.room.encrypted") - except ValueError: - pass + self.sanitize_subfilter(timeline_filter) return sync_filter @@ -763,8 +768,22 @@ class ProxyDaemon: if not client: return self._unknown_token + request_filter = request.query.get("filter", None) + query = CIMultiDict(request.query) + + if request_filter: + try: + request_filter = json.loads(request_filter) + except (JSONDecodeError, TypeError): + pass + + if isinstance(request_filter, dict): + request_filter = json.dumps(self.sanitize_filter(request_filter)) + + query["filter"] = request_filter + try: - response = await self.forward_request(request) + response = await self.forward_request(request, params=query) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) From 01e0c9363e18cbc18f8a3b012bb4f4df7902fdcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 20 Jul 2020 15:12:25 +0200 Subject: [PATCH 029/146] client: Store media info when decrypting instead of using a event callback. Event callbacks are only called for events that come through a sync resposne. Since events in the scrollback might be decryptable there's no reason to not store the media info for those as well. --- pantalaimon/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index e705fda..b2d2a72 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -208,7 +208,6 @@ class PanClient(AsyncClient): self.key_request_cb, (RoomKeyRequest, RoomKeyRequestCancellation) ) self.add_event_callback(self.undecrypted_event_cb, MegolmEvent) - self.add_event_callback(self.store_media_cb, RoomEncryptedMedia) self.add_event_callback( self.store_thumbnail_cb, (RoomEncryptedImage, RoomEncryptedVideo, RoomEncryptedFile), @@ -270,7 +269,7 @@ class PanClient(AsyncClient): self.media_info[(mxc_server, mxc_path)] = media self.pan_store.save_media(self.server_name, media) - def store_media_cb(self, room, event): + def store_event_media(self, event): try: mxc = urlparse(event.url) except ValueError: @@ -828,6 +827,8 @@ class PanClient(AsyncClient): ) if isinstance(decrypted_event, RoomEncryptedMedia): + self.store_event_media(decrypted_event) + decrypted_event.source["content"]["url"] = decrypted_event.url if decrypted_event.thumbnail_url: From d425e2d188aed32c3fe87cac210c0943fd51b085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 4 Aug 2020 15:32:07 +0200 Subject: [PATCH 030/146] pantalaimon: Increase the max POST size. --- pantalaimon/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 896d29e..2721008 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -58,7 +58,8 @@ async def init(data_dir, server_conf, send_queue, recv_queue): ssl=None if server_conf.ssl is True else False, ) - app = web.Application() + # 100 MB max POST size + app = web.Application(client_max_size=1024 ** 2 * 100) app.add_routes( [ From c13bd1329b2ad9a3fd2a6bef1b32439c8e00b575 Mon Sep 17 00:00:00 2001 From: Galen Abell Date: Fri, 14 Aug 2020 11:51:49 +0200 Subject: [PATCH 031/146] tests: Don't use keyring during tests Many systems don't have a keyring configured for DBus by default, which requires extra setup in order to run the proxy tests successfully. --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index c79de19..6103f77 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,7 +116,7 @@ async def pan_proxy_server(tempdir, aiohttp_server): server_name = faker.hostname() - config = ServerConfig(server_name, urlparse("https://example.org")) + config = ServerConfig(server_name, urlparse("https://example.org"), keyring=False) pan_queue = janus.Queue() ui_queue = janus.Queue() From 625bde33055e9d6907479f01052c0dc4c2ba9473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 2 Sep 2020 17:08:58 +0200 Subject: [PATCH 032/146] pantalaimon: Bump the version and update the changelog. --- CHANGELOG.md | 15 +++++++++++++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 4 ++-- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 278a1ea..30d7da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.7.0 2020-09-02 + +### Fixed + +- [[#60]] Sanitize the GET /rooms/{room_id}/messages filters as well. +- [[#62]] Store media info when decrypting instead of using a event callback. + +### Changed + +- [[d425e2d]] Increase the max POST size. + +[#62]: https://github.com/matrix-org/pantalaimon/pull/62 +[#60]: https://github.com/matrix-org/pantalaimon/pull/60 +[d425e2d]: https://github.com/matrix-org/pantalaimon/commit/d425e2d188aed32c3fe87cac210c0943fd51b085 + ## 0.6.5 2020-07-02 ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 2721008..0369294 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -250,7 +250,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.6.5", prog_name="pantalaimon") +@click.version_option(version="0.7.0", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 92a062d..7b15109 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.6.5", prog_name="panctl") +@click.version_option(version="0.7.0", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 2eea792..3f228c6 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.6.5", + version="0.7.0", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0" "prompt_toolkit>2<4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.14, < 0.15" + "matrix-nio[e2e] >= 0.14, < 0.16" ], extras_require={ "ui": [ From 73fa84a6e514cff4e1879814e520a2cf5c3d506f Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 15 Sep 2020 18:10:32 +0100 Subject: [PATCH 033/146] Do not try to login if the user authenticated via other means --- pantalaimon/daemon.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 0e1bd97..f695ef0 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -553,7 +553,7 @@ class ProxyDaemon: return user - async def start_pan_client(self, access_token, user, user_id, password): + async def start_pan_client(self, access_token, user, user_id, password, device_id=None): client = ClientInfo(user_id, access_token) self.client_info[access_token] = client self.store.save_server_user(self.name, user_id) @@ -578,11 +578,17 @@ class ProxyDaemon: store_class=self.client_store_class, media_info=self.media_info, ) - response = await pan_client.login(password, "pantalaimon") - if not isinstance(response, LoginResponse): - await pan_client.close() - return + if password == "": + # If password is blank, we cannot login normally and must + # fall back to using the provided device_id. + pan_client.restore_login(user_id, device_id, access_token) + else: + response = await pan_client.login(password, "pantalaimon") + + if not isinstance(response, LoginResponse): + await pan_client.close() + return logger.info(f"Succesfully started new background sync client for " f"{user_id}") @@ -646,13 +652,14 @@ class ProxyDaemon: if response.status == 200 and json_response: user_id = json_response.get("user_id", None) access_token = json_response.get("access_token", None) + device_id = json_response.get("device_id", None) if user_id and access_token: logger.info( f"User: {user} succesfully logged in, starting " f"a background sync client." ) - await self.start_pan_client(access_token, user, user_id, password) + await self.start_pan_client(access_token, user, user_id, password, device_id) return web.Response( status=response.status, From fdbb35d8069e5b86b377a5b56ef6654e8ca87498 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 16 Sep 2020 10:10:40 +0100 Subject: [PATCH 034/146] Add valuerror --- pantalaimon/daemon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index f695ef0..df32391 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -580,6 +580,8 @@ class ProxyDaemon: ) if password == "": + if device_id is None: + raise ValueError("Empty password provided, but device_id was also None") # If password is blank, we cannot login normally and must # fall back to using the provided device_id. pan_client.restore_login(user_id, device_id, access_token) From bb9408cbbe6625c9f6ae9f2ee5a4f1e07c5423d4 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 16 Sep 2020 10:12:55 +0100 Subject: [PATCH 035/146] Add changelog entry --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30d7da8..45b2139 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.X.X XXXX-XX-XX + +### Changed + +- [[#69]] If no password is provided to /login, the daemon will re-use the original login response. + +[#69]: https://github.com/matrix-org/pantalaimon/pull/69 + ## 0.7.0 2020-09-02 ### Fixed From 3c5e42896fc2a313e2903e0ed00430507cf75587 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 16 Sep 2020 11:59:46 +0100 Subject: [PATCH 036/146] log line --- pantalaimon/daemon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index df32391..ab3a936 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -581,7 +581,8 @@ class ProxyDaemon: if password == "": if device_id is None: - raise ValueError("Empty password provided, but device_id was also None") + logger.warn(f"Empty password provided and device_id was also None") + return # If password is blank, we cannot login normally and must # fall back to using the provided device_id. pan_client.restore_login(user_id, device_id, access_token) From 908c1a16be8f8a52f6acac4cd376815d06d0a9d7 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 16 Sep 2020 15:01:16 +0100 Subject: [PATCH 037/146] Fix logger.warn error Co-authored-by: poljar --- pantalaimon/daemon.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index ab3a936..9efee6b 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -581,7 +581,10 @@ class ProxyDaemon: if password == "": if device_id is None: - logger.warn(f"Empty password provided and device_id was also None") + logger.warn( + "Empty password provided and device_id was also None, not " + "starting background sync client " + ) return # If password is blank, we cannot login normally and must # fall back to using the provided device_id. From 5ca755a9445e8a8106e75905ddcb66fbf5611e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 30 Sep 2020 11:22:00 +0200 Subject: [PATCH 038/146] setup.py: Specify our UI deps better. This fixes: #68. --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 3f228c6..f64519b 100644 --- a/setup.py +++ b/setup.py @@ -33,10 +33,10 @@ setup( ], extras_require={ "ui": [ - "dbus-python <= 1.2", - "PyGObject <= 3.36", - "pydbus <= 0.6", - "notify2 <= 0.3", + "dbus-python >= 1.2, < 1.3", + "PyGObject >= 3.36, < 3.37", + "pydbus >= 0.6, < 0.7", + "notify2 >= 0.3, < 0.4", ] }, entry_points={ From d7089b91bd9afc0b1683ecd8f66a81f750e1efe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 30 Sep 2020 11:26:32 +0200 Subject: [PATCH 039/146] daemon: Format the file. --- pantalaimon/daemon.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 9efee6b..55b67a2 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -553,7 +553,9 @@ class ProxyDaemon: return user - async def start_pan_client(self, access_token, user, user_id, password, device_id=None): + async def start_pan_client( + self, access_token, user, user_id, password, device_id=None + ): client = ClientInfo(user_id, access_token) self.client_info[access_token] = client self.store.save_server_user(self.name, user_id) @@ -665,7 +667,9 @@ class ProxyDaemon: f"User: {user} succesfully logged in, starting " f"a background sync client." ) - await self.start_pan_client(access_token, user, user_id, password, device_id) + await self.start_pan_client( + access_token, user, user_id, password, device_id + ) return web.Response( status=response.status, From 9c65c060754c2c1fa7bdf2a5acc4812a260158dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 30 Sep 2020 11:26:46 +0200 Subject: [PATCH 040/146] pantalaimon: Bump the version. --- CHANGELOG.md | 2 +- pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45b2139..0a05ffd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## 0.X.X XXXX-XX-XX +## 0.8.0 2020-09-30 ### Changed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 0369294..12311ca 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -250,7 +250,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.7.0", prog_name="pantalaimon") +@click.version_option(version="0.8.0", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 7b15109..e551b89 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.7.0", prog_name="panctl") +@click.version_option(version="0.8.0", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index f64519b..0188b42 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.7.0", + version="0.8.0", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 1219ddfc48551b4c73d5f75e09e95b1a531cc87c Mon Sep 17 00:00:00 2001 From: mawalu Date: Sat, 17 Oct 2020 10:39:34 +0200 Subject: [PATCH 041/146] Use correct homeserver URL in example description --- docs/man/pantalaimon.5 | 2 +- docs/man/pantalaimon.5.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/man/pantalaimon.5 b/docs/man/pantalaimon.5 index 79651ba..448ef3a 100644 --- a/docs/man/pantalaimon.5 +++ b/docs/man/pantalaimon.5 @@ -128,7 +128,7 @@ Default location of the configuration file. The following example shows a configured pantalaimon proxy with the name .Em Clocktown , the homeserver URL is set to -.Em https://example.org , +.Em https://localhost:8448 , the pantalaimon proxy is listening for client connections on the address .Em localhost , and port diff --git a/docs/man/pantalaimon.5.md b/docs/man/pantalaimon.5.md index b140daf..fc197ec 100644 --- a/docs/man/pantalaimon.5.md +++ b/docs/man/pantalaimon.5.md @@ -111,7 +111,7 @@ overridden using appropriate environment variables. The following example shows a configured pantalaimon proxy with the name *Clocktown*, the homeserver URL is set to -*https://example.org*, +*https://localhost:8448*, the pantalaimon proxy is listening for client connections on the address *localhost*, and port From ec00d06c162b8c5bc88197fce04305a4ce522747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 2 Dec 2020 11:24:28 +0100 Subject: [PATCH 042/146] setup.py: Add a missing comma. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0188b42..c70017f 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( "logbook >= 1.5.3", "peewee >= 3.13.1", "janus >= 0.5", - "cachetools >= 3.0.0" + "cachetools >= 3.0.0", "prompt_toolkit>2<4", "typing;python_version<'3.5'", "matrix-nio[e2e] >= 0.14, < 0.16" From 726e96944d0a157bb5c244bc5a675ecbb16d495a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 2 Dec 2020 12:42:42 +0100 Subject: [PATCH 043/146] store: Add a missing use-database decorator. --- pantalaimon/store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 910df64..b73cb0f 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -226,6 +226,7 @@ class PanStore: user=user, room_id=task.room_id, token=task.token ).execute() + @use_database def load_fetcher_tasks(self, server, pan_user): server = Servers.get(name=server) user = ServerUsers.get(server=server, user_id=pan_user) From 5b1e220f5ea6944cb606e54303cfc30349fbe130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 2 Dec 2020 12:43:20 +0100 Subject: [PATCH 044/146] tox: Bump our Python versions. --- .travis.yml | 12 ++++++------ setup.py | 4 ++-- tox.ini | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index ccd725b..84318a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,14 +16,14 @@ before_install: matrix: include: - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - - python: 3.7 + - python: 3.8 + env: TOXENV=py38 + - python: 3.9 + env: TOXENV=py39 + - python: 3.9 env: TOXENV=coverage -install: pip install tox-travis PyGObject dbus-python aioresponses +install: pip install tox-travis aioresponses script: tox after_success: diff --git a/setup.py b/setup.py index c70017f..5cbad28 100644 --- a/setup.py +++ b/setup.py @@ -27,14 +27,14 @@ setup( "peewee >= 3.13.1", "janus >= 0.5", "cachetools >= 3.0.0", - "prompt_toolkit>2<4", + "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", "matrix-nio[e2e] >= 0.14, < 0.16" ], extras_require={ "ui": [ "dbus-python >= 1.2, < 1.3", - "PyGObject >= 3.36, < 3.37", + "PyGObject >= 3.36, < 3.39", "pydbus >= 0.6, < 0.7", "notify2 >= 0.3, < 0.4", ] diff --git a/tox.ini b/tox.ini index fb97e24..2fb9dd8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,11 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py36,py37,coverage +envlist = py38,py39,coverage [testenv] basepython = - py36: python3.6 - py37: python3.7 - py3: python3.7 + py38: python3.8 + py39: python3.9 + py3: python3.9 deps = -rtest-requirements.txt install_command = pip install {opts} {packages} @@ -15,7 +15,7 @@ commands = pytest usedevelop = True [testenv:coverage] -basepython = python3.7 +basepython = python3.9 commands = pytest --cov=pantalaimon --cov-report term-missing coverage xml From e5922da6ec99b3321b7fddb3fdd51957e867a7e7 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 02:37:27 +0100 Subject: [PATCH 045/146] WIP: first commit --- pantalaimon/daemon.py | 113 ++++++++++++++++++++++++++++++++++++------ pantalaimon/main.py | 4 ++ pantalaimon/store.py | 26 ++++++++++ 3 files changed, 128 insertions(+), 15 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 55b67a2..3522db5 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -35,6 +35,7 @@ from nio import ( OlmTrustError, SendRetryError, DownloadResponse, + UploadResponse, ) from nio.crypto import decrypt_attachment @@ -851,9 +852,46 @@ class ProxyDaemon: # The room isn't encrypted just forward the message. if not encrypt: + content_msgtype = "" + msgtype = request.match_info["event_type"] + if content_msgtype == "m.image" or content_msgtype == "m.video" \ + or content_msgtype == "m.audio" or content_msgtype == "m.file": + content_uri = request.match_info["content"]["url"] + + upload = self.store.load_upload(content_uri) + if upload is None: + return await self.forward_to_web(request, token=client.access_token) + + server_name = request.match_info["server_name"] + media_id = content_uri + file_name = request.match_info.get("file_name") + + response, decrypted_file, error = self._load_media(server_name, media_id, file_name, request) + + if response is None and decrypted_file is None and error is None: + return await self.forward_to_web(request, token=client.access_token) + if error is ClientConnectionError: + return await self.forward_to_web(request, token=client.access_token) + if error is KeyError: + return await self.forward_to_web(request, token=client.access_token) + + if not isinstance(response, DownloadResponse): + return await self.forward_to_web(request, token=client.access_token) + + decrypted_upload = client.upload( + data_provider=decrypted_file, + content_type=response.content_type, + filename=file_name, + encrypt=False, + ) + + if not isinstance(response, UploadResponse): + return await self.forward_to_web(request, token=client.access_token) + + request.match_info["content"]["url"] = decrypted_upload.content_uri + return await self.forward_to_web(request, token=client.access_token) - msgtype = request.match_info["event_type"] txnid = request.match_info.get("txnid", uuid4()) try: @@ -1039,11 +1077,36 @@ class ProxyDaemon: return web.json_response(result, headers=CORS_HEADERS, status=200) - async def download(self, request): - server_name = request.match_info["server_name"] - media_id = request.match_info["media_id"] - file_name = request.match_info.get("file_name") + async def upload(self, request): + file_name = request.query.get("filename", "") + content_type = request.headers.get("Content-Type", "application/octet-stream") + client = next(iter(self.pan_clients.values())) + try: + response = await client.upload( + data_provider=await request.read, + content_type=content_type, + filename=file_name, + encrypt=True, + ) + + if not isinstance(response, UploadResponse): + return web.Response( + status=response.transport_response.status, + content_type=response.transport_response.content_type, + headers=CORS_HEADERS, + body=await response.transport_response.read(), + ) + + self.store.save_upload(response.content_uri) + + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except SendRetryError as e: + return web.Response(status=503, text=str(e)) + + + def _load_media(self, server_name, media_id, file_name, request): try: media_info = self.media_info[(server_name, media_id)] except KeyError: @@ -1062,25 +1125,19 @@ class ProxyDaemon: logger.warn( f"Media info for {server_name}/{media_id} doesn't contain a key or hash." ) - return await self.forward_to_web(request) - + return None, None, KeyError if not self.pan_clients: - return await self.forward_to_web(request) + return None, None, None client = next(iter(self.pan_clients.values())) try: response = await client.download(server_name, media_id, file_name) except ClientConnectionError as e: - return web.Response(status=500, text=str(e)) + return None, None, e if not isinstance(response, DownloadResponse): - return web.Response( - status=response.transport_response.status, - content_type=response.transport_response.content_type, - headers=CORS_HEADERS, - body=await response.transport_response.read(), - ) + return response, None, None logger.info(f"Decrypting media {server_name}/{media_id}") @@ -1090,6 +1147,31 @@ class ProxyDaemon: pool, decrypt_attachment, response.body, key, hash, media_info.iv ) + return response, decrypted_file, None + + + async def download(self, request): + server_name = request.match_info["server_name"] + media_id = request.match_info["media_id"] + file_name = request.match_info.get("file_name") + + response, decrypted_file, error = self._load_media(server_name, media_id, file_name, request) + + if response is None and decrypted_file is None and error is None: + return await self.forward_to_web(request) + if error is ClientConnectionError: + return web.Response(status=500, text=str(error)) + if error is KeyError: + return await self.forward_to_web(request) + + if not isinstance(response, DownloadResponse): + return web.Response( + status=response.transport_response.status, + content_type=response.transport_response.content_type, + headers=CORS_HEADERS, + body=await response.transport_response.read(), + ) + return web.Response( status=response.transport_response.status, content_type=response.transport_response.content_type, @@ -1097,6 +1179,7 @@ class ProxyDaemon: body=decrypted_file, ) + async def well_known(self, _): """Intercept well-known requests diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 12311ca..b75ae21 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -93,6 +93,10 @@ async def init(data_dir, server_conf, send_queue, recv_queue): "/_matrix/media/r0/download/{server_name}/{media_id}/{file_name}", proxy.download, ), + web.post( + r"/_matrix/media/r0/upload", + proxy.upload, + ), ] ) app.router.add_route("*", "/" + "{proxyPath:.*}", proxy.router) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index b73cb0f..24e447d 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -48,6 +48,11 @@ class MediaInfo: hashes = attr.ib(type=dict) +@attr.s +class UploadInfo: + content_uri = attr.ib(type=str) + + class DictField(TextField): def python_value(self, value): # pragma: no cover return json.loads(value) @@ -112,6 +117,10 @@ class PanMediaInfo(Model): class Meta: constraints = [SQL("UNIQUE(server_id, mxc_server, mxc_path)")] +class PanUploadInfo(Model): + content_uri = TextField() + class Meta: + constraints = [SQL("UNIQUE(content_uri)")] @attr.s class ClientInfo: @@ -162,6 +171,23 @@ class PanStore: except DoesNotExist: return None + @use_database + def save_upload(self, content_uri): + PanUploadInfo.insert( + content_uri=content_uri, + ).on_conflict_ignore().execute() + + @use_database + def load_upload(self, content_uri): + u = PanUploadInfo.get_or_none( + PanUploadInfo.content_uri == content_uri, + ) + + if not u: + return None + + return UploadInfo(u.content_uri) + @use_database def save_media(self, server, media): server = Servers.get(name=server) From 5273b8357418bdcfeded260958b19f0196bece9c Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 02:47:28 +0100 Subject: [PATCH 046/146] use mxc data in send_message --- pantalaimon/daemon.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 3522db5..8235e3e 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -19,6 +19,7 @@ import urllib.parse import concurrent.futures from json import JSONDecodeError from typing import Any, Dict +from urllib.parse import urlparse from uuid import uuid4 import aiohttp @@ -862,8 +863,9 @@ class ProxyDaemon: if upload is None: return await self.forward_to_web(request, token=client.access_token) - server_name = request.match_info["server_name"] - media_id = content_uri + mxc = urlparse(content_uri) + server_name = mxc.netloc.strip("/") + media_id = mxc.path.strip("/") file_name = request.match_info.get("file_name") response, decrypted_file, error = self._load_media(server_name, media_id, file_name, request) From ce0fa21f941c54c5649e1b147ab68bc39495ba95 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 09:44:05 +0100 Subject: [PATCH 047/146] fix await --- pantalaimon/daemon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 8235e3e..1e731c3 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -868,7 +868,7 @@ class ProxyDaemon: media_id = mxc.path.strip("/") file_name = request.match_info.get("file_name") - response, decrypted_file, error = self._load_media(server_name, media_id, file_name, request) + response, decrypted_file, error = await self._load_media(server_name, media_id, file_name, request) if response is None and decrypted_file is None and error is None: return await self.forward_to_web(request, token=client.access_token) @@ -1108,7 +1108,7 @@ class ProxyDaemon: return web.Response(status=503, text=str(e)) - def _load_media(self, server_name, media_id, file_name, request): + async def _load_media(self, server_name, media_id, file_name, request): try: media_info = self.media_info[(server_name, media_id)] except KeyError: @@ -1116,7 +1116,7 @@ class ProxyDaemon: if not media_info: logger.info(f"No media info found for {server_name}/{media_id}") - return await self.forward_to_web(request) + return None, None, None self.media_info[(server_name, media_id)] = media_info @@ -1157,7 +1157,7 @@ class ProxyDaemon: media_id = request.match_info["media_id"] file_name = request.match_info.get("file_name") - response, decrypted_file, error = self._load_media(server_name, media_id, file_name, request) + response, decrypted_file, error = await self._load_media(server_name, media_id, file_name, request) if response is None and decrypted_file is None and error is None: return await self.forward_to_web(request) From 1168bcf7ffb9900330d456306fcef89065a5215f Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 17:12:21 +0100 Subject: [PATCH 048/146] CR fixes --- pantalaimon/daemon.py | 151 ++++++++++++++++++++++++------------------ pantalaimon/store.py | 17 ++++- tests/store_test.py | 18 ++++- 3 files changed, 120 insertions(+), 66 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 1e731c3..6130df7 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -17,6 +17,7 @@ import json import os import urllib.parse import concurrent.futures +from io import BufferedReader, BytesIO from json import JSONDecodeError from typing import Any, Dict from urllib.parse import urlparse @@ -50,7 +51,7 @@ from pantalaimon.client import ( ) from pantalaimon.index import INDEXING_ENABLED, InvalidQueryError from pantalaimon.log import logger -from pantalaimon.store import ClientInfo, PanStore +from pantalaimon.store import ClientInfo, PanStore, MediaInfo from pantalaimon.thread_messages import ( AcceptSasMessage, CancelSasMessage, @@ -826,6 +827,52 @@ class ProxyDaemon: body=await response.read(), ) + def _map_media_upload(self, content, request, client): + content_uri = content["url"] + + upload = self.store.load_upload(content_uri) + if upload is None: + return await self.forward_to_web(request, token=client.access_token) + + mxc = urlparse(content_uri) + mxc_server = mxc.netloc.strip("/") + mxc_path = mxc.path.strip("/") + file_name = request.match_info.get("file_name") + + logger.info(f"Adding media info for {mxc_server}/{mxc_path} to the store") + + media = MediaInfo(mxc_server, mxc_path, upload.key, upload.iv, upload.hashes) + self.media_info[(mxc_server, mxc_path)] = media + client = next(iter(self.pan_clients.values())) + self.store.save_media(client.server_name, media) + + try: + response, decrypted_file, error = await self._load_media(mxc_server, mxc_path, file_name, request) + + if response is None and decrypted_file is None: + return await self.forward_to_web(request, token=client.access_token) + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except KeyError: + return await self.forward_to_web(request, token=client.access_token) + + if not isinstance(response, DownloadResponse): + return await self.forward_to_web(request, token=client.access_token) + + decrypted_upload = await client.upload( + data_provider=BufferedReader(BytesIO(decrypted_file)), + content_type=response.content_type, + filename=file_name, + encrypt=False, + ) + + if not isinstance(decrypted_upload[0], UploadResponse): + raise ValueError + + content["url"] = decrypted_upload[0].content_uri + + return content + async def send_message(self, request): access_token = self.get_access_token(request) @@ -851,56 +898,27 @@ class ProxyDaemon: if request.match_info["event_type"] == "m.reaction": encrypt = False + msgtype = request.match_info["event_type"] + # The room isn't encrypted just forward the message. if not encrypt: - content_msgtype = "" - msgtype = request.match_info["event_type"] - if content_msgtype == "m.image" or content_msgtype == "m.video" \ - or content_msgtype == "m.audio" or content_msgtype == "m.file": - content_uri = request.match_info["content"]["url"] + content = "" + try: + content = await request.json() + except (JSONDecodeError, ContentTypeError): + return self._not_json - upload = self.store.load_upload(content_uri) - if upload is None: + content_msgtype = content["msgtype"] + if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"]: + try: + content = self._map_media_upload(content, request, client) + except ValueError: return await self.forward_to_web(request, token=client.access_token) - mxc = urlparse(content_uri) - server_name = mxc.netloc.strip("/") - media_id = mxc.path.strip("/") - file_name = request.match_info.get("file_name") - - response, decrypted_file, error = await self._load_media(server_name, media_id, file_name, request) - - if response is None and decrypted_file is None and error is None: - return await self.forward_to_web(request, token=client.access_token) - if error is ClientConnectionError: - return await self.forward_to_web(request, token=client.access_token) - if error is KeyError: - return await self.forward_to_web(request, token=client.access_token) - - if not isinstance(response, DownloadResponse): - return await self.forward_to_web(request, token=client.access_token) - - decrypted_upload = client.upload( - data_provider=decrypted_file, - content_type=response.content_type, - filename=file_name, - encrypt=False, - ) - - if not isinstance(response, UploadResponse): - return await self.forward_to_web(request, token=client.access_token) - - request.match_info["content"]["url"] = decrypted_upload.content_uri - return await self.forward_to_web(request, token=client.access_token) txnid = request.match_info.get("txnid", uuid4()) - try: - content = await request.json() - except (JSONDecodeError, ContentTypeError): - return self._not_json - async def _send(ignore_unverified=False): try: response = await client.room_send( @@ -1084,30 +1102,37 @@ class ProxyDaemon: content_type = request.headers.get("Content-Type", "application/octet-stream") client = next(iter(self.pan_clients.values())) + body = await request.read() try: response = await client.upload( - data_provider=await request.read, + data_provider=BufferedReader(BytesIO(body)), content_type=content_type, filename=file_name, encrypt=True, ) - if not isinstance(response, UploadResponse): + if not isinstance(response[0], UploadResponse): return web.Response( - status=response.transport_response.status, - content_type=response.transport_response.content_type, + status=response[0].transport_response.status, + content_type=response[0].transport_response.content_type, headers=CORS_HEADERS, - body=await response.transport_response.read(), + body=await response[0].transport_response.read(), ) - self.store.save_upload(response.content_uri) + self.store.save_upload(response[0].content_uri, response[1]) + + return web.Response( + status=response[0].transport_response.status, + content_type=response[0].transport_response.content_type, + headers=CORS_HEADERS, + body=await response[0].transport_response.read(), + ) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) except SendRetryError as e: return web.Response(status=503, text=str(e)) - async def _load_media(self, server_name, media_id, file_name, request): try: media_info = self.media_info[(server_name, media_id)] @@ -1116,30 +1141,30 @@ class ProxyDaemon: if not media_info: logger.info(f"No media info found for {server_name}/{media_id}") - return None, None, None + return None, None self.media_info[(server_name, media_id)] = media_info try: key = media_info.key["k"] hash = media_info.hashes["sha256"] - except KeyError: + except KeyError as e: logger.warn( f"Media info for {server_name}/{media_id} doesn't contain a key or hash." ) - return None, None, KeyError + raise e if not self.pan_clients: - return None, None, None + return None, None client = next(iter(self.pan_clients.values())) try: response = await client.download(server_name, media_id, file_name) except ClientConnectionError as e: - return None, None, e + raise e if not isinstance(response, DownloadResponse): - return response, None, None + return response, None logger.info(f"Decrypting media {server_name}/{media_id}") @@ -1149,7 +1174,7 @@ class ProxyDaemon: pool, decrypt_attachment, response.body, key, hash, media_info.iv ) - return response, decrypted_file, None + return response, decrypted_file async def download(self, request): @@ -1157,13 +1182,14 @@ class ProxyDaemon: media_id = request.match_info["media_id"] file_name = request.match_info.get("file_name") - response, decrypted_file, error = await self._load_media(server_name, media_id, file_name, request) + try: + response, decrypted_file, error = await self._load_media(server_name, media_id, file_name, request) - if response is None and decrypted_file is None and error is None: - return await self.forward_to_web(request) - if error is ClientConnectionError: - return web.Response(status=500, text=str(error)) - if error is KeyError: + if response is None and decrypted_file is None: + return await self.forward_to_web(request) + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except KeyError: return await self.forward_to_web(request) if not isinstance(response, DownloadResponse): @@ -1181,7 +1207,6 @@ class ProxyDaemon: body=decrypted_file, ) - async def well_known(self, _): """Intercept well-known requests diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 24e447d..4e93f71 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -51,6 +51,9 @@ class MediaInfo: @attr.s class UploadInfo: content_uri = attr.ib(type=str) + key = attr.ib(type=dict) + iv = attr.ib(type=str) + hashes = attr.ib(type=dict) class DictField(TextField): @@ -117,11 +120,17 @@ class PanMediaInfo(Model): class Meta: constraints = [SQL("UNIQUE(server_id, mxc_server, mxc_path)")] + class PanUploadInfo(Model): content_uri = TextField() + key = DictField() + hashes = DictField() + iv = TextField() + class Meta: constraints = [SQL("UNIQUE(content_uri)")] + @attr.s class ClientInfo: user_id = attr.ib(type=str) @@ -144,6 +153,7 @@ class PanStore: PanSyncTokens, PanFetcherTasks, PanMediaInfo, + PanUploadInfo, ] def __attrs_post_init__(self): @@ -172,9 +182,12 @@ class PanStore: return None @use_database - def save_upload(self, content_uri): + def save_upload(self, content_uri, media): PanUploadInfo.insert( content_uri=content_uri, + key=media["key"], + iv=media["iv"], + hashes=media["hashes"], ).on_conflict_ignore().execute() @use_database @@ -186,7 +199,7 @@ class PanStore: if not u: return None - return UploadInfo(u.content_uri) + return UploadInfo(u.content_uri, u.key, u.iv, u.hashes) @use_database def save_media(self, server, media): diff --git a/tests/store_test.py b/tests/store_test.py index 2f31587..5ea5f96 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -8,7 +8,7 @@ from nio import RoomMessage, RoomEncryptedMedia from urllib.parse import urlparse from conftest import faker from pantalaimon.index import INDEXING_ENABLED -from pantalaimon.store import FetchTask, MediaInfo +from pantalaimon.store import FetchTask, MediaInfo, UploadInfo TEST_ROOM = "!SVkFJHzfwvuaIEawgC:localhost" TEST_ROOM2 = "!testroom:localhost" @@ -177,3 +177,19 @@ class TestClass(object): media_info = media_cache[(mxc_server, mxc_path)] assert media_info == media assert media_info == panstore.load_media(server_name, mxc_server, mxc_path) + + def test_upload_storage(self, panstore): + event = self.encrypted_media_event + + assert not panstore.load_upload(event.url) + + upload = UploadInfo(event.url, event.key, event.iv, event.hashes) + + panstore.save_upload(event.url) + + upload_cache = panstore.load_upload(event.url) + + assert (event.url) in upload_cache + upload_info = upload_cache[(event.url)] + assert upload_info == upload + assert upload_info == panstore.load_upload(event.url) From 8a40b9e6bc3445f0a6c83077a44dce61bba8bc3e Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 17:21:35 +0100 Subject: [PATCH 049/146] refinements --- pantalaimon/daemon.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 6130df7..52d5628 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -827,7 +827,7 @@ class ProxyDaemon: body=await response.read(), ) - def _map_media_upload(self, content, request, client): + async def _map_media_upload(self, content, request, client): content_uri = content["url"] upload = self.store.load_upload(content_uri) @@ -843,8 +843,7 @@ class ProxyDaemon: media = MediaInfo(mxc_server, mxc_path, upload.key, upload.iv, upload.hashes) self.media_info[(mxc_server, mxc_path)] = media - client = next(iter(self.pan_clients.values())) - self.store.save_media(client.server_name, media) + self.store.save_media(self.name, media) try: response, decrypted_file, error = await self._load_media(mxc_server, mxc_path, file_name, request) @@ -859,7 +858,7 @@ class ProxyDaemon: if not isinstance(response, DownloadResponse): return await self.forward_to_web(request, token=client.access_token) - decrypted_upload = await client.upload( + decrypted_upload, maybe_keys = await client.upload( data_provider=BufferedReader(BytesIO(decrypted_file)), content_type=response.content_type, filename=file_name, @@ -911,7 +910,7 @@ class ProxyDaemon: content_msgtype = content["msgtype"] if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"]: try: - content = self._map_media_upload(content, request, client) + content = await self._map_media_upload(content, request, client) except ValueError: return await self.forward_to_web(request, token=client.access_token) @@ -1104,28 +1103,28 @@ class ProxyDaemon: body = await request.read() try: - response = await client.upload( + response, maybe_keys = await client.upload( data_provider=BufferedReader(BytesIO(body)), content_type=content_type, filename=file_name, encrypt=True, ) - if not isinstance(response[0], UploadResponse): + if not isinstance(response, UploadResponse): return web.Response( - status=response[0].transport_response.status, - content_type=response[0].transport_response.content_type, + status=response.transport_response.status, + content_type=response.transport_response.content_type, headers=CORS_HEADERS, - body=await response[0].transport_response.read(), + body=await response.transport_response.read(), ) - self.store.save_upload(response[0].content_uri, response[1]) + self.store.save_upload(response.content_uri, maybe_keys) return web.Response( - status=response[0].transport_response.status, - content_type=response[0].transport_response.content_type, + status=response.transport_response.status, + content_type=response.transport_response.content_type, headers=CORS_HEADERS, - body=await response[0].transport_response.read(), + body=await response.transport_response.read(), ) except ClientConnectionError as e: From a533c25efd6f271788b4e9c662778586ac7d020f Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 17:45:34 +0100 Subject: [PATCH 050/146] little cleaning --- pantalaimon/daemon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 52d5628..04ea925 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -846,7 +846,7 @@ class ProxyDaemon: self.store.save_media(self.name, media) try: - response, decrypted_file, error = await self._load_media(mxc_server, mxc_path, file_name, request) + response, decrypted_file = await self._load_media(mxc_server, mxc_path, file_name, request) if response is None and decrypted_file is None: return await self.forward_to_web(request, token=client.access_token) @@ -865,10 +865,10 @@ class ProxyDaemon: encrypt=False, ) - if not isinstance(decrypted_upload[0], UploadResponse): + if not isinstance(decrypted_upload, UploadResponse): raise ValueError - content["url"] = decrypted_upload[0].content_uri + content["url"] = decrypted_upload.content_uri return content @@ -1182,7 +1182,7 @@ class ProxyDaemon: file_name = request.match_info.get("file_name") try: - response, decrypted_file, error = await self._load_media(server_name, media_id, file_name, request) + response, decrypted_file = await self._load_media(server_name, media_id, file_name, request) if response is None and decrypted_file is None: return await self.forward_to_web(request) From 404dfe568ca07d42d1024d16adaf9b7f83727e2d Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 20:52:04 +0100 Subject: [PATCH 051/146] overwrite content --- pantalaimon/daemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 04ea925..03f152e 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -911,6 +911,7 @@ class ProxyDaemon: if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"]: try: content = await self._map_media_upload(content, request, client) + return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) except ValueError: return await self.forward_to_web(request, token=client.access_token) From 3a380cc91efc464005a8f42dfc7d77c9ec0ebd93 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 21:00:09 +0100 Subject: [PATCH 052/146] fix test --- pantalaimon/store.py | 8 ++++---- tests/store_test.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 4e93f71..123f80f 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -182,12 +182,12 @@ class PanStore: return None @use_database - def save_upload(self, content_uri, media): + def save_upload(self, content_uri, upload): PanUploadInfo.insert( content_uri=content_uri, - key=media["key"], - iv=media["iv"], - hashes=media["hashes"], + key=upload["key"], + iv=upload["iv"], + hashes=upload["hashes"], ).on_conflict_ignore().execute() @use_database diff --git a/tests/store_test.py b/tests/store_test.py index 5ea5f96..218575a 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -185,7 +185,7 @@ class TestClass(object): upload = UploadInfo(event.url, event.key, event.iv, event.hashes) - panstore.save_upload(event.url) + panstore.save_upload(event.url, {"key": event.key, "iv": event.iv, "hashes": event.hashes}) upload_cache = panstore.load_upload(event.url) From aa5123bef38b2d12793f052bc4e09d00eb2d60f7 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 22 Dec 2020 22:16:06 +0100 Subject: [PATCH 053/146] fix test --- tests/store_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/store_test.py b/tests/store_test.py index 218575a..afd8725 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -189,7 +189,7 @@ class TestClass(object): upload_cache = panstore.load_upload(event.url) - assert (event.url) in upload_cache + assert (event.url, event.key, event.iv, event.hashes) in upload_cache upload_info = upload_cache[(event.url)] assert upload_info == upload assert upload_info == panstore.load_upload(event.url) From 453fd935d2b35da6345f47d4bccce085f75c8b61 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 23 Dec 2020 00:47:23 +0100 Subject: [PATCH 054/146] fix test --- tests/store_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/store_test.py b/tests/store_test.py index afd8725..ada5737 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -187,9 +187,6 @@ class TestClass(object): panstore.save_upload(event.url, {"key": event.key, "iv": event.iv, "hashes": event.hashes}) - upload_cache = panstore.load_upload(event.url) + upload_info = panstore.load_upload(event.url) - assert (event.url, event.key, event.iv, event.hashes) in upload_cache - upload_info = upload_cache[(event.url)] assert upload_info == upload - assert upload_info == panstore.load_upload(event.url) From f747c83af4d56e0af129eef702ab1b2587eb6f04 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 23 Dec 2020 12:36:29 +0100 Subject: [PATCH 055/146] cache in upload --- pantalaimon/daemon.py | 22 ++++++++++++++++------ pantalaimon/store.py | 28 +++++++++++++++++++++++++--- tests/store_test.py | 13 ++++++++++--- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 03f152e..788dfed 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -105,6 +105,7 @@ class ProxyDaemon: client_info = attr.ib(init=False, default=attr.Factory(dict), type=dict) default_session = attr.ib(init=False, default=None) media_info = attr.ib(init=False, default=None) + upload_info = attr.ib(init=False, default=None) database_name = "pan.db" def __attrs_post_init__(self): @@ -115,6 +116,7 @@ class ProxyDaemon: self.store = PanStore(self.data_dir) accounts = self.store.load_users(self.name) self.media_info = self.store.load_media(self.name) + self.upload_info = self.store.load_upload(self.name) for user_id, device_id in accounts: if self.conf.keyring: @@ -830,9 +832,17 @@ class ProxyDaemon: async def _map_media_upload(self, content, request, client): content_uri = content["url"] - upload = self.store.load_upload(content_uri) - if upload is None: - return await self.forward_to_web(request, token=client.access_token) + try: + upload_info = self.upload_info[content_uri] + except KeyError: + upload_info = self.store.load_upload(self.name, content_uri) + if not upload_info: + logger.info(f"No upload info found for {self.name}/{content_uri}") + + return await self.forward_to_web(request, token=client.access_token) + + self.upload_info[content_uri] = upload_info + mxc = urlparse(content_uri) mxc_server = mxc.netloc.strip("/") @@ -841,7 +851,7 @@ class ProxyDaemon: logger.info(f"Adding media info for {mxc_server}/{mxc_path} to the store") - media = MediaInfo(mxc_server, mxc_path, upload.key, upload.iv, upload.hashes) + media = MediaInfo(mxc_server, mxc_path, upload_info.key, upload_info.iv, upload_info.hashes) self.media_info[(mxc_server, mxc_path)] = media self.store.save_media(self.name, media) @@ -1119,7 +1129,7 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - self.store.save_upload(response.content_uri, maybe_keys) + self.store.save_upload(self.name, response.content_uri, maybe_keys) return web.Response( status=response.transport_response.status, @@ -1133,7 +1143,7 @@ class ProxyDaemon: except SendRetryError as e: return web.Response(status=503, text=str(e)) - async def _load_media(self, server_name, media_id, file_name, request): + async def _load_media(self, server_name, media_id, file_name): try: media_info = self.media_info[(server_name, media_id)] except KeyError: diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 123f80f..fbde863 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -31,6 +31,7 @@ from cachetools import LRUCache MAX_LOADED_MEDIA = 10000 +MAX_LOADED_UPLOAD = 10000 @attr.s @@ -122,13 +123,16 @@ class PanMediaInfo(Model): class PanUploadInfo(Model): + server = ForeignKeyField( + model=Servers, column_name="server_id", backref="upload", on_delete="CASCADE" + ) content_uri = TextField() key = DictField() hashes = DictField() iv = TextField() class Meta: - constraints = [SQL("UNIQUE(content_uri)")] + constraints = [SQL("UNIQUE(server_id, content_uri)")] @attr.s @@ -182,8 +186,11 @@ class PanStore: return None @use_database - def save_upload(self, content_uri, upload): + def save_upload(self, server, content_uri, upload): + server = Servers.get(name=server) + PanUploadInfo.insert( + server=server, content_uri=content_uri, key=upload["key"], iv=upload["iv"], @@ -191,8 +198,23 @@ class PanStore: ).on_conflict_ignore().execute() @use_database - def load_upload(self, content_uri): + def load_upload(self, server, content_uri=None): + server, _ = Servers.get_or_create(name=server) + + if not content_uri: + upload_cache = LRUCache(maxsize=MAX_LOADED_UPLOAD) + + for i, u in enumerate(server.upload): + if i > MAX_LOADED_UPLOAD: + break + + upload = UploadInfo(u.content_uri, u.key, u.iv, u.hashes) + upload_cache[u.content_uri] = upload + + return upload_cache + else: u = PanUploadInfo.get_or_none( + PanUploadInfo.server == server, PanUploadInfo.content_uri == content_uri, ) diff --git a/tests/store_test.py b/tests/store_test.py index ada5737..981170f 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -179,14 +179,21 @@ class TestClass(object): assert media_info == panstore.load_media(server_name, mxc_server, mxc_path) def test_upload_storage(self, panstore): + server_name = "test" + upload_cache = panstore.load_upload(server_name) + assert not upload_cache + event = self.encrypted_media_event - assert not panstore.load_upload(event.url) + assert not panstore.load_upload(server_name, event.url) upload = UploadInfo(event.url, event.key, event.iv, event.hashes) - panstore.save_upload(event.url, {"key": event.key, "iv": event.iv, "hashes": event.hashes}) + panstore.save_upload(server_name, event.url, {"key": event.key, "iv": event.iv, "hashes": event.hashes}) - upload_info = panstore.load_upload(event.url) + upload_cache = panstore.load_upload(server_name) + assert (event.url) in upload_cache + upload_info = upload_cache[event.url] assert upload_info == upload + assert upload_info == panstore.load_upload(server_name, event.url) From fadcf2934d1945645f7cbff4facf31296d156905 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 23 Dec 2020 19:50:29 +0100 Subject: [PATCH 056/146] fix bug --- pantalaimon/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 788dfed..06d9943 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -856,7 +856,7 @@ class ProxyDaemon: self.store.save_media(self.name, media) try: - response, decrypted_file = await self._load_media(mxc_server, mxc_path, file_name, request) + response, decrypted_file = await self._load_media(mxc_server, mxc_path, file_name) if response is None and decrypted_file is None: return await self.forward_to_web(request, token=client.access_token) From 8aaa82abf369e33c6290cc1c801e17ef40de8353 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 24 Dec 2020 17:58:50 +0100 Subject: [PATCH 057/146] profile and m.room.avatar --- pantalaimon/daemon.py | 44 ++++++++++++++++++++++++++++++++++++++----- pantalaimon/main.py | 5 +++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 06d9943..b0c9cef 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -829,8 +829,8 @@ class ProxyDaemon: body=await response.read(), ) - async def _map_media_upload(self, content, request, client): - content_uri = content["url"] + async def _map_media_upload(self, content_key, content, request, client): + content_uri = content[content_key] try: upload_info = self.upload_info[content_uri] @@ -878,7 +878,7 @@ class ProxyDaemon: if not isinstance(decrypted_upload, UploadResponse): raise ValueError - content["url"] = decrypted_upload.content_uri + content[content_key] = decrypted_upload.content_uri return content @@ -918,9 +918,9 @@ class ProxyDaemon: return self._not_json content_msgtype = content["msgtype"] - if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"]: + if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": try: - content = await self._map_media_upload(content, request, client) + content = await self._map_media_upload("url", content, request, client) return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) except ValueError: return await self.forward_to_web(request, token=client.access_token) @@ -1187,6 +1187,40 @@ class ProxyDaemon: return response, decrypted_file + async def profile(self, request): + access_token = self.get_access_token(request) + + if not access_token: + return self._missing_token + + client = await self._find_client(access_token) + if not client: + return self._unknown_token + + room_id = request.match_info["room_id"] + + # The room is not in the joined rooms list, just forward it. + try: + room = client.rooms[room_id] + encrypt = room.encrypted + except KeyError: + return await self.forward_to_web(request, token=client.access_token) + + # The room isn't encrypted just forward the message. + if not encrypt: + try: + content = await request.json() + except (JSONDecodeError, ContentTypeError): + return self._not_json + + try: + content = await self._map_media_upload("avatar_url", content, request, client) + return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) + except ValueError: + return await self.forward_to_web(request, token=client.access_token) + + return await self.forward_to_web(request, token=client.access_token) + async def download(self, request): server_name = request.match_info["server_name"] media_id = request.match_info["media_id"] diff --git a/pantalaimon/main.py b/pantalaimon/main.py index b75ae21..1f0bb7b 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -97,6 +97,11 @@ async def init(data_dir, server_conf, send_queue, recv_queue): r"/_matrix/media/r0/upload", proxy.upload, ), + web.post( + r"/_matrix/client/r0/profile/{userId}/avatar_url", + proxy.profile, + ), + ] ) app.router.add_route("*", "/" + "{proxyPath:.*}", proxy.router) From 3d319b1247b16b63835bf56c07a3fd77faac6c6a Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Sun, 27 Dec 2020 01:51:35 +0100 Subject: [PATCH 058/146] fix bug --- pantalaimon/daemon.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index b0c9cef..2ed122d 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -909,14 +909,14 @@ class ProxyDaemon: msgtype = request.match_info["event_type"] + content = "" + try: + content = await request.json() + except (JSONDecodeError, ContentTypeError): + return self._not_json + # The room isn't encrypted just forward the message. if not encrypt: - content = "" - try: - content = await request.json() - except (JSONDecodeError, ContentTypeError): - return self._not_json - content_msgtype = content["msgtype"] if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": try: From 3d1a807f7e441a2061b6b560d8cda3118b98306b Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 5 Jan 2021 14:28:07 +0100 Subject: [PATCH 059/146] CR fixes --- pantalaimon/daemon.py | 29 ++++++++--------------------- pantalaimon/main.py | 2 +- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 2ed122d..5e2010a 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -1197,30 +1197,17 @@ class ProxyDaemon: if not client: return self._unknown_token - room_id = request.match_info["room_id"] - - # The room is not in the joined rooms list, just forward it. try: - room = client.rooms[room_id] - encrypt = room.encrypted - except KeyError: + content = await request.json() + except (JSONDecodeError, ContentTypeError): + return self._not_json + + try: + content = await self._map_media_upload("avatar_url", content, request, client) + return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) + except ValueError: return await self.forward_to_web(request, token=client.access_token) - # The room isn't encrypted just forward the message. - if not encrypt: - try: - content = await request.json() - except (JSONDecodeError, ContentTypeError): - return self._not_json - - try: - content = await self._map_media_upload("avatar_url", content, request, client) - return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) - except ValueError: - return await self.forward_to_web(request, token=client.access_token) - - return await self.forward_to_web(request, token=client.access_token) - async def download(self, request): server_name = request.match_info["server_name"] media_id = request.match_info["media_id"] diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 1f0bb7b..adde3ea 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -97,7 +97,7 @@ async def init(data_dir, server_conf, send_queue, recv_queue): r"/_matrix/media/r0/upload", proxy.upload, ), - web.post( + web.put( r"/_matrix/client/r0/profile/{userId}/avatar_url", proxy.profile, ), From f0ea2ebd3d862fea30a6c3f6057918254b998fd7 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 6 Jan 2021 18:08:03 +0100 Subject: [PATCH 060/146] CR fixes --- pantalaimon/daemon.py | 51 +++++++++++++++++++++++++++++++------------ pantalaimon/store.py | 27 +++++++++++++++++++---- tests/store_test.py | 4 ++-- 3 files changed, 62 insertions(+), 20 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 5e2010a..3a35a77 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -829,7 +829,7 @@ class ProxyDaemon: body=await response.read(), ) - async def _map_media_upload(self, content_key, content, request, client): + def _get_upload_and_media_info(self, content_key, content, request): content_uri = content[content_key] try: @@ -837,13 +837,11 @@ class ProxyDaemon: except KeyError: upload_info = self.store.load_upload(self.name, content_uri) if not upload_info: - logger.info(f"No upload info found for {self.name}/{content_uri}") - - return await self.forward_to_web(request, token=client.access_token) - - self.upload_info[content_uri] = upload_info + return None, None + self.upload_info[content_uri] = upload_info + content_uri = content[content_key] mxc = urlparse(content_uri) mxc_server = mxc.netloc.strip("/") mxc_path = mxc.path.strip("/") @@ -851,12 +849,19 @@ class ProxyDaemon: logger.info(f"Adding media info for {mxc_server}/{mxc_path} to the store") - media = MediaInfo(mxc_server, mxc_path, upload_info.key, upload_info.iv, upload_info.hashes) - self.media_info[(mxc_server, mxc_path)] = media - self.store.save_media(self.name, media) + media_info = MediaInfo(mxc_server, mxc_path, upload_info.key, upload_info.iv, upload_info.hashes) + self.media_info[(mxc_server, mxc_path)] = media_info + self.store.save_media(self.name, media_info) + return upload_info, media_info, file_name + + async def _map_media_upload(self, content_key, content, request, client): try: - response, decrypted_file = await self._load_media(mxc_server, mxc_path, file_name) + upload_info, media_info, file_name = self._get_upload_and_media_info(content_key, content, request) + if not upload_info: + return await self.forward_to_web(request, token=client.access_token) + + response, decrypted_file = await self._load_media(media_info.mcx_server, media_info.mxc_path, file_name) if response is None and decrypted_file is None: return await self.forward_to_web(request, token=client.access_token) @@ -868,11 +873,12 @@ class ProxyDaemon: if not isinstance(response, DownloadResponse): return await self.forward_to_web(request, token=client.access_token) - decrypted_upload, maybe_keys = await client.upload( + decrypted_upload, _ = await client.upload( data_provider=BufferedReader(BytesIO(decrypted_file)), content_type=response.content_type, filename=file_name, encrypt=False, + filesize=len(decrypted_file), ) if not isinstance(decrypted_upload, UploadResponse): @@ -909,7 +915,6 @@ class ProxyDaemon: msgtype = request.match_info["event_type"] - content = "" try: content = await request.json() except (JSONDecodeError, ContentTypeError): @@ -931,6 +936,23 @@ class ProxyDaemon: async def _send(ignore_unverified=False): try: + content_msgtype = content["msgtype"] + if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": + upload_info, media_info, file_name = self._get_upload_and_media_info("url", content, request) + if not upload_info: + response = await client.room_send( + room_id, msgtype, content, txnid, ignore_unverified + ) + + return web.Response( + status=response.transport_response.status, + content_type=response.transport_response.content_type, + headers=CORS_HEADERS, + body=await response.transport_response.read(), + ) + + content = media_info.to_content(file_name, content_msgtype, upload_info.mimetype), + response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified ) @@ -1119,6 +1141,7 @@ class ProxyDaemon: content_type=content_type, filename=file_name, encrypt=True, + filesize=len(body), ) if not isinstance(response, UploadResponse): @@ -1129,7 +1152,7 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - self.store.save_upload(self.name, response.content_uri, maybe_keys) + self.store.save_upload(self.name, response.content_uri, maybe_keys, content_type) return web.Response( status=response.transport_response.status, @@ -1214,7 +1237,7 @@ class ProxyDaemon: file_name = request.match_info.get("file_name") try: - response, decrypted_file = await self._load_media(server_name, media_id, file_name, request) + response, decrypted_file = await self._load_media(server_name, media_id, file_name) if response is None and decrypted_file is None: return await self.forward_to_web(request) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index fbde863..79e05d7 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -15,7 +15,7 @@ import json import os from collections import defaultdict -from typing import List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import attr from nio.crypto import TrustState @@ -48,6 +48,23 @@ class MediaInfo: iv = attr.ib(type=str) hashes = attr.ib(type=dict) + def to_content(self, file_name: str, msgtype: str, mime_type: str) -> Dict[Any, Any]: + content = { + "body": file_name, + "file": { + "v": "v2", + "key": self.key, + "iv": self.iv, + "hashes": self.hashes, + "url": self.url, + "mimetype": mime_type, + } + } + + if msgtype: + content["msgtype"] = msgtype + + return content @attr.s class UploadInfo: @@ -55,6 +72,7 @@ class UploadInfo: key = attr.ib(type=dict) iv = attr.ib(type=str) hashes = attr.ib(type=dict) + mimetype = attr.ib(type=str) class DictField(TextField): @@ -186,7 +204,7 @@ class PanStore: return None @use_database - def save_upload(self, server, content_uri, upload): + def save_upload(self, server, content_uri, upload, mimetype): server = Servers.get(name=server) PanUploadInfo.insert( @@ -195,6 +213,7 @@ class PanStore: key=upload["key"], iv=upload["iv"], hashes=upload["hashes"], + mimetype=mimetype, ).on_conflict_ignore().execute() @use_database @@ -208,7 +227,7 @@ class PanStore: if i > MAX_LOADED_UPLOAD: break - upload = UploadInfo(u.content_uri, u.key, u.iv, u.hashes) + upload = UploadInfo(u.content_uri, u.key, u.iv, u.hashes, u.mimetype) upload_cache[u.content_uri] = upload return upload_cache @@ -221,7 +240,7 @@ class PanStore: if not u: return None - return UploadInfo(u.content_uri, u.key, u.iv, u.hashes) + return UploadInfo(u.content_uri, u.key, u.iv, u.hashes, u.mimetype) @use_database def save_media(self, server, media): diff --git a/tests/store_test.py b/tests/store_test.py index 981170f..b31d41a 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -187,9 +187,9 @@ class TestClass(object): assert not panstore.load_upload(server_name, event.url) - upload = UploadInfo(event.url, event.key, event.iv, event.hashes) + upload = UploadInfo(event.url, event.key, event.iv, event.hashes, event.mimetype) - panstore.save_upload(server_name, event.url, {"key": event.key, "iv": event.iv, "hashes": event.hashes}) + panstore.save_upload(server_name, event.url, {"key": event.key, "iv": event.iv, "hashes": event.hashes}, event.mimetype) upload_cache = panstore.load_upload(server_name) From 0bf49f7fdd840a195ccd9216ded7f067ed8fedf1 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 11:22:11 +0100 Subject: [PATCH 061/146] remove keys from uploadinfo, save media and upload at upload time --- pantalaimon/daemon.py | 43 ++++++++++++++++++++++++++++--------------- pantalaimon/store.py | 20 ++++++-------------- tests/store_test.py | 5 +++-- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 3a35a77..31bbd00 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -837,7 +837,7 @@ class ProxyDaemon: except KeyError: upload_info = self.store.load_upload(self.name, content_uri) if not upload_info: - return None, None + return None, None, None self.upload_info[content_uri] = upload_info @@ -845,23 +845,24 @@ class ProxyDaemon: mxc = urlparse(content_uri) mxc_server = mxc.netloc.strip("/") mxc_path = mxc.path.strip("/") - file_name = request.match_info.get("file_name") - logger.info(f"Adding media info for {mxc_server}/{mxc_path} to the store") + media_info = self.store.load_media(self.name, mxc_server, mxc_path) + if not media_info: + return None, None, None - media_info = MediaInfo(mxc_server, mxc_path, upload_info.key, upload_info.iv, upload_info.hashes) self.media_info[(mxc_server, mxc_path)] = media_info - self.store.save_media(self.name, media_info) + + file_name = request.match_info.get("file_name") return upload_info, media_info, file_name - async def _map_media_upload(self, content_key, content, request, client): + async def _map_decrypted_uri(self, content_key, content, request, client): try: upload_info, media_info, file_name = self._get_upload_and_media_info(content_key, content, request) - if not upload_info: + if not upload_info or not media_info: return await self.forward_to_web(request, token=client.access_token) - response, decrypted_file = await self._load_media(media_info.mcx_server, media_info.mxc_path, file_name) + response, decrypted_file = await self._load_decrypted_file(media_info.mcx_server, media_info.mxc_path, file_name) if response is None and decrypted_file is None: return await self.forward_to_web(request, token=client.access_token) @@ -925,7 +926,7 @@ class ProxyDaemon: content_msgtype = content["msgtype"] if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": try: - content = await self._map_media_upload("url", content, request, client) + content = await self._map_decrypted_uri("url", content, request, client) return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) except ValueError: return await self.forward_to_web(request, token=client.access_token) @@ -939,7 +940,7 @@ class ProxyDaemon: content_msgtype = content["msgtype"] if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": upload_info, media_info, file_name = self._get_upload_and_media_info("url", content, request) - if not upload_info: + if not upload_info or not media_info: response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified ) @@ -951,7 +952,11 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - content = media_info.to_content(file_name, content_msgtype, upload_info.mimetype), + content = media_info.to_content(content["url"], + file_name, + content_msgtype, + upload_info.mimetype + ), response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified @@ -1152,7 +1157,15 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - self.store.save_upload(self.name, response.content_uri, maybe_keys, content_type) + self.store.save_upload(self.name, response.content_uri, content_type) + + mxc = urlparse(response.content_uri) + mxc_server = mxc.netloc.strip("/") + mxc_path = mxc.path.strip("/") + + logger.info(f"Adding media info for {mxc_server}/{mxc_path} to the store") + media_info = MediaInfo(mxc_server, mxc_path, maybe_keys["key"], maybe_keys["iv"], maybe_keys["hashes"]) + self.store.save_media(self.name, media_info) return web.Response( status=response.transport_response.status, @@ -1166,7 +1179,7 @@ class ProxyDaemon: except SendRetryError as e: return web.Response(status=503, text=str(e)) - async def _load_media(self, server_name, media_id, file_name): + async def _load_decrypted_file(self, server_name, media_id, file_name): try: media_info = self.media_info[(server_name, media_id)] except KeyError: @@ -1226,7 +1239,7 @@ class ProxyDaemon: return self._not_json try: - content = await self._map_media_upload("avatar_url", content, request, client) + content = await self._map_decrypted_uri("avatar_url", content, request, client) return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) except ValueError: return await self.forward_to_web(request, token=client.access_token) @@ -1237,7 +1250,7 @@ class ProxyDaemon: file_name = request.match_info.get("file_name") try: - response, decrypted_file = await self._load_media(server_name, media_id, file_name) + response, decrypted_file = await self._load_decrypted_file(server_name, media_id, file_name) if response is None and decrypted_file is None: return await self.forward_to_web(request) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 79e05d7..b7e3978 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -48,7 +48,7 @@ class MediaInfo: iv = attr.ib(type=str) hashes = attr.ib(type=dict) - def to_content(self, file_name: str, msgtype: str, mime_type: str) -> Dict[Any, Any]: + def to_content(self, url: str, file_name: str, msgtype: str, mime_type: str) -> Dict[Any, Any]: content = { "body": file_name, "file": { @@ -56,7 +56,7 @@ class MediaInfo: "key": self.key, "iv": self.iv, "hashes": self.hashes, - "url": self.url, + "url": url, "mimetype": mime_type, } } @@ -69,9 +69,6 @@ class MediaInfo: @attr.s class UploadInfo: content_uri = attr.ib(type=str) - key = attr.ib(type=dict) - iv = attr.ib(type=str) - hashes = attr.ib(type=dict) mimetype = attr.ib(type=str) @@ -145,9 +142,7 @@ class PanUploadInfo(Model): model=Servers, column_name="server_id", backref="upload", on_delete="CASCADE" ) content_uri = TextField() - key = DictField() - hashes = DictField() - iv = TextField() + mimetype = TextField() class Meta: constraints = [SQL("UNIQUE(server_id, content_uri)")] @@ -204,15 +199,12 @@ class PanStore: return None @use_database - def save_upload(self, server, content_uri, upload, mimetype): + def save_upload(self, server, content_uri, mimetype): server = Servers.get(name=server) PanUploadInfo.insert( server=server, content_uri=content_uri, - key=upload["key"], - iv=upload["iv"], - hashes=upload["hashes"], mimetype=mimetype, ).on_conflict_ignore().execute() @@ -227,7 +219,7 @@ class PanStore: if i > MAX_LOADED_UPLOAD: break - upload = UploadInfo(u.content_uri, u.key, u.iv, u.hashes, u.mimetype) + upload = UploadInfo(u.content_uri, u.mimetype) upload_cache[u.content_uri] = upload return upload_cache @@ -240,7 +232,7 @@ class PanStore: if not u: return None - return UploadInfo(u.content_uri, u.key, u.iv, u.hashes, u.mimetype) + return UploadInfo(u.content_uri, u.mimetype) @use_database def save_media(self, server, media): diff --git a/tests/store_test.py b/tests/store_test.py index b31d41a..cb72c5a 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -183,13 +183,14 @@ class TestClass(object): upload_cache = panstore.load_upload(server_name) assert not upload_cache + mimetype = "image/jpeg" event = self.encrypted_media_event assert not panstore.load_upload(server_name, event.url) - upload = UploadInfo(event.url, event.key, event.iv, event.hashes, event.mimetype) + upload = UploadInfo(event.url, mimetype) - panstore.save_upload(server_name, event.url, {"key": event.key, "iv": event.iv, "hashes": event.hashes}, event.mimetype) + panstore.save_upload(server_name, event.url, mimetype) upload_cache = panstore.load_upload(server_name) From 33e938b095795c1d33cea1ca4fe9902a64fd4bcf Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 11:57:36 +0100 Subject: [PATCH 062/146] bugfix --- pantalaimon/daemon.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 31bbd00..887eee3 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -952,15 +952,19 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - content = media_info.to_content(content["url"], + media_content = media_info.to_content(content["url"], file_name, content_msgtype, upload_info.mimetype ), - response = await client.room_send( - room_id, msgtype, content, txnid, ignore_unverified - ) + response = await client.room_send( + room_id, msgtype, media_content, txnid, ignore_unverified + ) + else: + response = await client.room_send( + room_id, msgtype, content, txnid, ignore_unverified + ) return web.Response( status=response.transport_response.status, From 5dd4c50c2a923d22fdf0f93e4d121ede4a50c984 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 16:57:07 +0100 Subject: [PATCH 063/146] store filename in upload info --- pantalaimon/daemon.py | 16 +++++++--------- pantalaimon/store.py | 6 ++++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 887eee3..cb5ff6b 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -829,7 +829,7 @@ class ProxyDaemon: body=await response.read(), ) - def _get_upload_and_media_info(self, content_key, content, request): + def _get_upload_and_media_info(self, content_key, content): content_uri = content[content_key] try: @@ -852,17 +852,15 @@ class ProxyDaemon: self.media_info[(mxc_server, mxc_path)] = media_info - file_name = request.match_info.get("file_name") - - return upload_info, media_info, file_name + return upload_info, media_info async def _map_decrypted_uri(self, content_key, content, request, client): try: - upload_info, media_info, file_name = self._get_upload_and_media_info(content_key, content, request) + upload_info, media_info = self._get_upload_and_media_info(content_key, content) if not upload_info or not media_info: return await self.forward_to_web(request, token=client.access_token) - response, decrypted_file = await self._load_decrypted_file(media_info.mcx_server, media_info.mxc_path, file_name) + response, decrypted_file = await self._load_decrypted_file(media_info.mcx_server, media_info.mxc_path, upload_info.filename) if response is None and decrypted_file is None: return await self.forward_to_web(request, token=client.access_token) @@ -877,7 +875,7 @@ class ProxyDaemon: decrypted_upload, _ = await client.upload( data_provider=BufferedReader(BytesIO(decrypted_file)), content_type=response.content_type, - filename=file_name, + filename=upload_info.filename, encrypt=False, filesize=len(decrypted_file), ) @@ -939,7 +937,7 @@ class ProxyDaemon: try: content_msgtype = content["msgtype"] if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": - upload_info, media_info, file_name = self._get_upload_and_media_info("url", content, request) + upload_info, media_info = self._get_upload_and_media_info("url", content) if not upload_info or not media_info: response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified @@ -953,7 +951,7 @@ class ProxyDaemon: ) media_content = media_info.to_content(content["url"], - file_name, + upload_info.filename, content_msgtype, upload_info.mimetype ), diff --git a/pantalaimon/store.py b/pantalaimon/store.py index b7e3978..6513ff0 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -69,6 +69,7 @@ class MediaInfo: @attr.s class UploadInfo: content_uri = attr.ib(type=str) + filename = attr.ib(type=str) mimetype = attr.ib(type=str) @@ -142,6 +143,7 @@ class PanUploadInfo(Model): model=Servers, column_name="server_id", backref="upload", on_delete="CASCADE" ) content_uri = TextField() + filename = TextField() mimetype = TextField() class Meta: @@ -219,7 +221,7 @@ class PanStore: if i > MAX_LOADED_UPLOAD: break - upload = UploadInfo(u.content_uri, u.mimetype) + upload = UploadInfo(u.content_uri, u.filename, u.mimetype) upload_cache[u.content_uri] = upload return upload_cache @@ -232,7 +234,7 @@ class PanStore: if not u: return None - return UploadInfo(u.content_uri, u.mimetype) + return UploadInfo(u.content_uri, u.filename, u.mimetype) @use_database def save_media(self, server, media): From b6b4362adebba5cf773b20e45088d4bdb5efb456 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 16:58:24 +0100 Subject: [PATCH 064/146] fix test --- tests/store_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/store_test.py b/tests/store_test.py index cb72c5a..5363288 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -183,12 +183,13 @@ class TestClass(object): upload_cache = panstore.load_upload(server_name) assert not upload_cache + filename = "orange_cat.jpg" mimetype = "image/jpeg" event = self.encrypted_media_event assert not panstore.load_upload(server_name, event.url) - upload = UploadInfo(event.url, mimetype) + upload = UploadInfo(event.url, filename, mimetype) panstore.save_upload(server_name, event.url, mimetype) From 6cedf846bdc9e0303954dd72b19c2275a864d260 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 17:08:34 +0100 Subject: [PATCH 065/146] store filename in upload info --- pantalaimon/daemon.py | 6 +++--- pantalaimon/store.py | 3 ++- tests/store_test.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index cb5ff6b..2b54f0e 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -837,7 +837,7 @@ class ProxyDaemon: except KeyError: upload_info = self.store.load_upload(self.name, content_uri) if not upload_info: - return None, None, None + return None, None self.upload_info[content_uri] = upload_info @@ -848,7 +848,7 @@ class ProxyDaemon: media_info = self.store.load_media(self.name, mxc_server, mxc_path) if not media_info: - return None, None, None + return None, None self.media_info[(mxc_server, mxc_path)] = media_info @@ -1159,7 +1159,7 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - self.store.save_upload(self.name, response.content_uri, content_type) + self.store.save_upload(self.name, response.content_uri, file_name, content_type) mxc = urlparse(response.content_uri) mxc_server = mxc.netloc.strip("/") diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 6513ff0..4fdadf8 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -201,12 +201,13 @@ class PanStore: return None @use_database - def save_upload(self, server, content_uri, mimetype): + def save_upload(self, server, content_uri, filename, mimetype): server = Servers.get(name=server) PanUploadInfo.insert( server=server, content_uri=content_uri, + filename=filename, mimetype=mimetype, ).on_conflict_ignore().execute() diff --git a/tests/store_test.py b/tests/store_test.py index 5363288..fe7411e 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -191,7 +191,7 @@ class TestClass(object): upload = UploadInfo(event.url, filename, mimetype) - panstore.save_upload(server_name, event.url, mimetype) + panstore.save_upload(server_name, event.url, filename, mimetype) upload_cache = panstore.load_upload(server_name) From e66a3bc2d19c8454585fef87f6333978a15e691d Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 17:48:19 +0100 Subject: [PATCH 066/146] fix bug --- pantalaimon/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 2b54f0e..92bae48 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -860,7 +860,7 @@ class ProxyDaemon: if not upload_info or not media_info: return await self.forward_to_web(request, token=client.access_token) - response, decrypted_file = await self._load_decrypted_file(media_info.mcx_server, media_info.mxc_path, upload_info.filename) + response, decrypted_file = await self._load_decrypted_file(media_info.mxc_server, media_info.mxc_path, upload_info.filename) if response is None and decrypted_file is None: return await self.forward_to_web(request, token=client.access_token) From 8fc906a9e6f4e33f5532c418662f37a98b656da7 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 17:52:50 +0100 Subject: [PATCH 067/146] fix bug --- pantalaimon/store.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 4fdadf8..4353a86 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -61,6 +61,9 @@ class MediaInfo: } } + if not file_name: + content["body"] = url + if msgtype: content["msgtype"] = msgtype From 83d1e1287072538f89b50808fd23e7536216cd10 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 17:54:46 +0100 Subject: [PATCH 068/146] fix bug --- pantalaimon/store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 4353a86..7f83ff1 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -61,7 +61,7 @@ class MediaInfo: } } - if not file_name: + if len(file_name) == 0: content["body"] = url if msgtype: From a24f80c4d0935e8f9231a10c6dd9d9742921affa Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 17:57:54 +0100 Subject: [PATCH 069/146] fix bug --- pantalaimon/store.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 7f83ff1..3d8f681 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -50,7 +50,6 @@ class MediaInfo: def to_content(self, url: str, file_name: str, msgtype: str, mime_type: str) -> Dict[Any, Any]: content = { - "body": file_name, "file": { "v": "v2", "key": self.key, @@ -61,7 +60,9 @@ class MediaInfo: } } - if len(file_name) == 0: + if len(file_name) > 0: + content["body"] = file_name + else: content["body"] = url if msgtype: From 266b049cfcb26523a7978768134f6293b9c56ee0 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 7 Jan 2021 18:11:02 +0100 Subject: [PATCH 070/146] override existing content --- pantalaimon/daemon.py | 6 +----- pantalaimon/store.py | 16 +++------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 92bae48..815fd78 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -950,11 +950,7 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - media_content = media_info.to_content(content["url"], - upload_info.filename, - content_msgtype, - upload_info.mimetype - ), + media_content = media_info.to_content(content, upload_info.mimetype) response = await client.room_send( room_id, msgtype, media_content, txnid, ignore_unverified diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 3d8f681..911850d 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -48,26 +48,16 @@ class MediaInfo: iv = attr.ib(type=str) hashes = attr.ib(type=dict) - def to_content(self, url: str, file_name: str, msgtype: str, mime_type: str) -> Dict[Any, Any]: - content = { - "file": { + def to_content(self, content: Dict, mime_type: str) -> Dict[Any, Any]: + content["file"] = { "v": "v2", "key": self.key, "iv": self.iv, "hashes": self.hashes, - "url": url, + "url": content["url"], "mimetype": mime_type, - } } - if len(file_name) > 0: - content["body"] = file_name - else: - content["body"] = url - - if msgtype: - content["msgtype"] = msgtype - return content @attr.s From 6ece7020de3d6fabb7dda8b56651722e634c0c8d Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Mon, 11 Jan 2021 16:37:46 +0100 Subject: [PATCH 071/146] remove hidden forward_to_web in _map_decrypted_uri and rely on except --- pantalaimon/daemon.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 815fd78..f4053d7 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -855,22 +855,18 @@ class ProxyDaemon: return upload_info, media_info async def _map_decrypted_uri(self, content_key, content, request, client): - try: - upload_info, media_info = self._get_upload_and_media_info(content_key, content) - if not upload_info or not media_info: - return await self.forward_to_web(request, token=client.access_token) + upload_info, media_info = self._get_upload_and_media_info(content_key, content) + if not upload_info or not media_info: + raise ValueError - response, decrypted_file = await self._load_decrypted_file(media_info.mxc_server, media_info.mxc_path, upload_info.filename) + response, decrypted_file = await self._load_decrypted_file(media_info.mxc_server, media_info.mxc_path, + upload_info.filename) - if response is None and decrypted_file is None: - return await self.forward_to_web(request, token=client.access_token) - except ClientConnectionError as e: - return web.Response(status=500, text=str(e)) - except KeyError: - return await self.forward_to_web(request, token=client.access_token) + if response is None and decrypted_file is None: + raise ValueError if not isinstance(response, DownloadResponse): - return await self.forward_to_web(request, token=client.access_token) + raise ValueError decrypted_upload, _ = await client.upload( data_provider=BufferedReader(BytesIO(decrypted_file)), @@ -926,6 +922,10 @@ class ProxyDaemon: try: content = await self._map_decrypted_uri("url", content, request, client) return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except KeyError: + return await self.forward_to_web(request, token=client.access_token) except ValueError: return await self.forward_to_web(request, token=client.access_token) @@ -1239,6 +1239,10 @@ class ProxyDaemon: try: content = await self._map_decrypted_uri("avatar_url", content, request, client) return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except KeyError: + return await self.forward_to_web(request, token=client.access_token) except ValueError: return await self.forward_to_web(request, token=client.access_token) From 787540f329df861278fa8ff2e35c2c3bb9d7d4e2 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Mon, 11 Jan 2021 16:58:03 +0100 Subject: [PATCH 072/146] replace ValueError with custom NotDecryptedAvailableError --- pantalaimon/daemon.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index f4053d7..268e99a 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -83,6 +83,16 @@ CORS_HEADERS = { } +class NotDecryptedAvailableError(BaseException): + """ Inappropriate argument value (of correct type). """ + def __init__(self, *args, **kwargs): # real signature unknown + pass + + @staticmethod # known case of __new__ + def __new__(*args, **kwargs): # real signature unknown + """ Create and return a new object. See help(type) for accurate signature. """ + pass + @attr.s class ProxyDaemon: name = attr.ib() @@ -857,16 +867,16 @@ class ProxyDaemon: async def _map_decrypted_uri(self, content_key, content, request, client): upload_info, media_info = self._get_upload_and_media_info(content_key, content) if not upload_info or not media_info: - raise ValueError + raise NotDecryptedAvailableError response, decrypted_file = await self._load_decrypted_file(media_info.mxc_server, media_info.mxc_path, upload_info.filename) if response is None and decrypted_file is None: - raise ValueError + raise NotDecryptedAvailableError if not isinstance(response, DownloadResponse): - raise ValueError + raise NotDecryptedAvailableError decrypted_upload, _ = await client.upload( data_provider=BufferedReader(BytesIO(decrypted_file)), @@ -877,7 +887,7 @@ class ProxyDaemon: ) if not isinstance(decrypted_upload, UploadResponse): - raise ValueError + raise NotDecryptedAvailableError content[content_key] = decrypted_upload.content_uri @@ -924,9 +934,7 @@ class ProxyDaemon: return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) - except KeyError: - return await self.forward_to_web(request, token=client.access_token) - except ValueError: + except (KeyError, NotDecryptedAvailableError): return await self.forward_to_web(request, token=client.access_token) return await self.forward_to_web(request, token=client.access_token) @@ -1241,9 +1249,7 @@ class ProxyDaemon: return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) - except KeyError: - return await self.forward_to_web(request, token=client.access_token) - except ValueError: + except (KeyError, NotDecryptedAvailableError): return await self.forward_to_web(request, token=client.access_token) async def download(self, request): From 1c317ed2944a5b322a399006e3c6840176a66056 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Mon, 11 Jan 2021 17:43:56 +0100 Subject: [PATCH 073/146] cr fix --- pantalaimon/daemon.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 268e99a..35b7f65 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -83,14 +83,8 @@ CORS_HEADERS = { } -class NotDecryptedAvailableError(BaseException): - """ Inappropriate argument value (of correct type). """ - def __init__(self, *args, **kwargs): # real signature unknown - pass - - @staticmethod # known case of __new__ - def __new__(*args, **kwargs): # real signature unknown - """ Create and return a new object. See help(type) for accurate signature. """ +class NotDecryptedAvailableError(Exception): + """Exception that signals that no decrypted upload is available""" pass @attr.s From 12ae367626f939b505f3f44f3599007e01846da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 Jan 2021 10:00:54 +0100 Subject: [PATCH 074/146] daemon: Fix a couple of styel issues --- pantalaimon/daemon.py | 6 +++--- pantalaimon/store.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 35b7f65..a038490 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -84,8 +84,9 @@ CORS_HEADERS = { class NotDecryptedAvailableError(Exception): - """Exception that signals that no decrypted upload is available""" - pass + """Exception that signals that no decrypted upload is available""" + pass + @attr.s class ProxyDaemon: @@ -1222,7 +1223,6 @@ class ProxyDaemon: return response, decrypted_file - async def profile(self, request): access_token = self.get_access_token(request) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 911850d..c1b4691 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -60,6 +60,7 @@ class MediaInfo: return content + @attr.s class UploadInfo: content_uri = attr.ib(type=str) From 59051c530a343a6887ea0f9ccddd6f6964f6d923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 Jan 2021 11:52:52 +0100 Subject: [PATCH 075/146] ui: Fix the notification initialization Closes: #81. --- pantalaimon/ui.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pantalaimon/ui.py b/pantalaimon/ui.py index 25eb2f8..cedca65 100644 --- a/pantalaimon/ui.py +++ b/pantalaimon/ui.py @@ -30,6 +30,7 @@ if UI_ENABLED: from gi.repository import GLib from pydbus import SessionBus from pydbus.generic import signal + from dbus.mainloop.glib import DBusGMainLoop from nio import RoomKeyRequest, RoomKeyRequestCancellation @@ -447,6 +448,7 @@ if UI_ENABLED: config = attr.ib() loop = attr.ib(init=False) + dbus_loop = attr.ib(init=False) store = attr.ib(init=False) users = attr.ib(init=False) devices = attr.ib(init=False) @@ -457,6 +459,7 @@ if UI_ENABLED: def __attrs_post_init__(self): self.loop = None + self.dbus_loop = None id_counter = IdCounter() @@ -632,11 +635,12 @@ if UI_ENABLED: return True def run(self): + self.dbus_loop = DBusGMainLoop() self.loop = GLib.MainLoop() if self.config.notifications: try: - notify2.init("pantalaimon", mainloop=self.loop) + notify2.init("pantalaimon", mainloop=self.dbus_loop) self.notifications = True except dbus.DBusException: logger.error( @@ -646,6 +650,7 @@ if UI_ENABLED: self.notifications = False GLib.timeout_add(100, self.message_callback) + if not self.loop: return From bd7af084f7f2b9d5e8b32c6c7414c80ea0713a6e Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 15 Jan 2021 14:23:37 +0000 Subject: [PATCH 076/146] Do not fail if msgtype is not defined --- pantalaimon/daemon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index a038490..10f1799 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -922,7 +922,7 @@ class ProxyDaemon: # The room isn't encrypted just forward the message. if not encrypt: - content_msgtype = content["msgtype"] + content_msgtype = content.get("msgtype") if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": try: content = await self._map_decrypted_uri("url", content, request, client) @@ -938,7 +938,7 @@ class ProxyDaemon: async def _send(ignore_unverified=False): try: - content_msgtype = content["msgtype"] + content_msgtype = content.get("msgtype") if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": upload_info, media_info = self._get_upload_and_media_info("url", content) if not upload_info or not media_info: From 3cb1d735cf2efcf56f5958ea652be20aaa7895a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Jan 2021 10:53:04 +0100 Subject: [PATCH 077/146] pantalaimon: Bump our version --- pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index adde3ea..0ed6e35 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -259,7 +259,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.8.0", prog_name="pantalaimon") +@click.version_option(version="0.9.0", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index e551b89..2de2b22 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.8.0", prog_name="panctl") +@click.version_option(version="0.9.0", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 5cbad28..a0a377f 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.8.0", + version="0.9.0", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 30bf1969564d575808c2917236a177993e03b679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Jan 2021 10:53:20 +0100 Subject: [PATCH 078/146] pantalaimon: Update the changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a05ffd..335e27a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.9.0 2021-01-19 + +### Added + +- [[59051c5]] Fix the notification initialization allowing the DBUS thread to + start again +- [[#79]] Support media uploads, thanks to @aspacca + +[59051c5]: https://github.com/matrix-org/pantalaimon/commit/59051c530a343a6887ea0f9ccddd6f6964f6d923 +[#79]: https://github.com/matrix-org/pantalaimon/pull/79 + ## 0.8.0 2020-09-30 ### Changed From 3baae08ac36e258632e224b655e177a765a939f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Jan 2021 11:00:31 +0100 Subject: [PATCH 079/146] setup.py: Bump the allowed nio version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a0a377f..fb064e0 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.14, < 0.16" + "matrix-nio[e2e] >= 0.14, < 0.17" ], extras_require={ "ui": [ From bb65f0fac5e58ba110c389ba917ca05cfe98a6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Jan 2021 11:01:53 +0100 Subject: [PATCH 080/146] pantalaimon: Bump our version --- pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 0ed6e35..a2e1b54 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -259,7 +259,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.9.0", prog_name="pantalaimon") +@click.version_option(version="0.9.1", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 2de2b22..c81560f 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.9.0", prog_name="panctl") +@click.version_option(version="0.9.1", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index fb064e0..b9d3320 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.9.0", + version="0.9.1", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From e426d0fca6b9c7adb01095acdfe78ef4345c9088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Jan 2021 11:02:11 +0100 Subject: [PATCH 081/146] pantalaimon: Update the changelog --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 335e27a..4c823df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.9.1 2021-01-19 + +### Changed + +- [[3baae08]] Bump the allowed nio version + ## 0.9.0 2021-01-19 -### Added +### Fixed - [[59051c5]] Fix the notification initialization allowing the DBUS thread to start again + +### Added + - [[#79]] Support media uploads, thanks to @aspacca +[3baae08]: https://github.com/matrix-org/pantalaimon/commit/3baae08ac36e258632e224b655e177a765a939f3 [59051c5]: https://github.com/matrix-org/pantalaimon/commit/59051c530a343a6887ea0f9ccddd6f6964f6d923 [#79]: https://github.com/matrix-org/pantalaimon/pull/79 From 376ca72014a4b1f8a63b2547b3856eccbf688ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vojt=C4=9Bch=20K=C3=A1n=C4=9B?= Date: Tue, 9 Mar 2021 11:43:28 +0100 Subject: [PATCH 082/146] setup.py: Bump the allowed nio version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9d3320..e0587f2 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.14, < 0.17" + "matrix-nio[e2e] >= 0.14, < 0.18" ], extras_require={ "ui": [ From edab476a90b1e682424d6866d169ac24e9f87d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Mar 2021 09:36:43 +0100 Subject: [PATCH 083/146] pantalaimon: Bump the version --- pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index a2e1b54..6c69c82 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -259,7 +259,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.9.1", prog_name="pantalaimon") +@click.version_option(version="0.9.2", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index c81560f..e67e429 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.9.1", prog_name="panctl") +@click.version_option(version="0.9.2", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index e0587f2..307f84b 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.9.1", + version="0.9.2", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From db9cf778912a7c5212cf7dfe5150a158f58b8cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 10 Mar 2021 09:42:47 +0100 Subject: [PATCH 084/146] pantalaimon: Update the changelog --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c823df..46d7c45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.9.2 2021-03-10 + +### Changed + +- [[#89]] Bump the allowed nio version + +[#89]: https://github.com/matrix-org/pantalaimon/pull/89 + ## 0.9.1 2021-01-19 ### Changed - [[3baae08]] Bump the allowed nio version +[3baae08]: https://github.com/matrix-org/pantalaimon/commit/3baae08ac36e258632e224b655e177a765a939f3 + ## 0.9.0 2021-01-19 ### Fixed @@ -21,7 +31,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#79]] Support media uploads, thanks to @aspacca -[3baae08]: https://github.com/matrix-org/pantalaimon/commit/3baae08ac36e258632e224b655e177a765a939f3 [59051c5]: https://github.com/matrix-org/pantalaimon/commit/59051c530a343a6887ea0f9ccddd6f6964f6d923 [#79]: https://github.com/matrix-org/pantalaimon/pull/79 From 73f68c76fb05037bd7fe71688ce39eb1f526a385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 6 May 2021 15:10:43 +0200 Subject: [PATCH 085/146] pantalaimon: Bump the allowed nio version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 307f84b..0d18eac 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.14, < 0.18" + "matrix-nio[e2e] >= 0.14, < 0.19" ], extras_require={ "ui": [ From 87169df413b00f0491b483fd1a3cc296797741fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 14 May 2021 13:53:51 +0200 Subject: [PATCH 086/146] pantalaimon: Bump our version --- CHANGELOG.md | 8 ++++++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46d7c45..c1dba35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.9.3 2021-05-14 + +### Changed + +- [[#73f68c7]] Bump the allowed nio version + +[73f68c7]: https://github.com/matrix-org/pantalaimon/commit/73f68c76fb05037bd7fe71688ce39eb1f526a385 + ## 0.9.2 2021-03-10 ### Changed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 6c69c82..0108cdb 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -259,7 +259,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.9.2", prog_name="pantalaimon") +@click.version_option(version="0.9.3", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index e67e429..1b486cc 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.9.2", prog_name="panctl") +@click.version_option(version="0.9.3", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 0d18eac..011d3e4 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.9.2", + version="0.9.3", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 7b4cc0527d7fc17c0995700d52f582668577e768 Mon Sep 17 00:00:00 2001 From: Alexander Manning Date: Fri, 14 May 2021 13:25:29 +0100 Subject: [PATCH 087/146] Encrypt Thumbnails When a client sends a thumbnail with their media message, encrypt the thumbnail too. --- pantalaimon/daemon.py | 11 +++++++++++ pantalaimon/store.py | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 10f1799..1d0afe4 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -954,6 +954,17 @@ class ProxyDaemon: ) media_content = media_info.to_content(content, upload_info.mimetype) + if media_content["info"].get("thumbnail_url", False): + ( + thumb_upload_info, + thumb_media_info, + ) = self._get_upload_and_media_info( + "thumbnail_url", media_content["info"] + ) + if thumb_upload_info and thumb_media_info: + media_content = thumb_media_info.to_thumbnail( + media_content, thumb_upload_info.mimetype + ) response = await client.room_send( room_id, msgtype, media_content, txnid, ignore_unverified diff --git a/pantalaimon/store.py b/pantalaimon/store.py index c1b4691..135936f 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -60,6 +60,19 @@ class MediaInfo: return content + def to_thumbnail(self, content: Dict, mime_type: str) -> Dict[Any, Any]: + content["info"]["thumbnail_file"] = { + "v": "v2", + "key": self.key, + "iv": self.iv, + "hashes": self.hashes, + "url": content["info"]["thumbnail_url"], + "mimetype": mime_type, + } + + return content + + @attr.s class UploadInfo: From 5e81131ecf4e0b67fef09589d9610743da6e7319 Mon Sep 17 00:00:00 2001 From: Alexander Manning Date: Fri, 14 May 2021 16:35:20 +0100 Subject: [PATCH 088/146] Remove return from MediaInfo.to_thumbnail() and from MediaInfo.to_content() --- pantalaimon/daemon.py | 12 ++++++------ pantalaimon/store.py | 4 ---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 1d0afe4..9693c9c 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -953,21 +953,21 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - media_content = media_info.to_content(content, upload_info.mimetype) - if media_content["info"].get("thumbnail_url", False): + media_info.to_content(content, upload_info.mimetype) + if content["info"].get("thumbnail_url", False): ( thumb_upload_info, thumb_media_info, ) = self._get_upload_and_media_info( - "thumbnail_url", media_content["info"] + "thumbnail_url", content["info"] ) if thumb_upload_info and thumb_media_info: - media_content = thumb_media_info.to_thumbnail( - media_content, thumb_upload_info.mimetype + thumb_media_info.to_thumbnail( + content, thumb_upload_info.mimetype ) response = await client.room_send( - room_id, msgtype, media_content, txnid, ignore_unverified + room_id, msgtype, content, txnid, ignore_unverified ) else: response = await client.room_send( diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 135936f..3e647f6 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -58,7 +58,6 @@ class MediaInfo: "mimetype": mime_type, } - return content def to_thumbnail(self, content: Dict, mime_type: str) -> Dict[Any, Any]: content["info"]["thumbnail_file"] = { @@ -70,9 +69,6 @@ class MediaInfo: "mimetype": mime_type, } - return content - - @attr.s class UploadInfo: From eabd5f5b51e4221b162ab3c8e8a6de926fa2c7af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 28 May 2021 14:20:18 +0200 Subject: [PATCH 089/146] pantalaimon: Add a config option to drop old room keys This patch adds a config option that enables a mode wher a proxy will only keep the latest room key from a sender in a certain room. This is useful for bots that use pantalaimon since they are mostly only interested in the latest messages and don't care about room history. --- contrib/pantalaimon.conf | 1 + docs/man/pantalaimon.5 | 5 ++++ docs/man/pantalaimon.5.md | 9 ++++++- pantalaimon/config.py | 6 +++++ pantalaimon/main.py | 5 +++- pantalaimon/store.py | 49 +++++++++++++++++++++++++++++++++++++-- 6 files changed, 71 insertions(+), 4 deletions(-) diff --git a/contrib/pantalaimon.conf b/contrib/pantalaimon.conf index 69a7465..dc4cedc 100644 --- a/contrib/pantalaimon.conf +++ b/contrib/pantalaimon.conf @@ -12,3 +12,4 @@ Proxy = http://localhost:8080 SSL = False IgnoreVerification = False UseKeyring = True +DropOldKeys = False diff --git a/docs/man/pantalaimon.5 b/docs/man/pantalaimon.5 index 448ef3a..62b281e 100644 --- a/docs/man/pantalaimon.5 +++ b/docs/man/pantalaimon.5 @@ -51,6 +51,11 @@ This option configures if a proxy instance should use the OS keyring to store its own access tokens. The access tokens are required for the daemon to resume operation. If this is set to "No", access tokens are stored in the pantalaimon database in plaintext. Defaults to "Yes". +.It Cm DropOldKeys +This option configures if a proxy instance should only keep the latest version +of a room key from a certain user around. This effectively means that only newly +incoming messages will be decryptable, the proxy will be unable to decrypt the +room history. Defaults to "No". .It Cm SearchRequests This option configures if the proxy should make additional HTTP requests to the server when clients use the search API endpoint. Some data that is required to diff --git a/docs/man/pantalaimon.5.md b/docs/man/pantalaimon.5.md index fc197ec..4a78e80 100644 --- a/docs/man/pantalaimon.5.md +++ b/docs/man/pantalaimon.5.md @@ -62,6 +62,13 @@ The following keys are optional in the proxy instance sections: > operation. If this is set to "No", access tokens are stored in the pantalaimon > database in plaintext. Defaults to "Yes". +**DropOldKeys** + +> This option configures if a proxy instance should only keep the latest version +> of a room key from a certain user around. This effectively means that only newly +> incoming messages will be decryptable, the proxy will be unable to decrypt the +> room history. Defaults to "No". + Aditional to the homeserver section a special section with the name **Default** can be used to configure the following values for all homeservers: @@ -150,4 +157,4 @@ pantalaimon(8) was written by Damir Jelić <[poljar@termina.org.uk](mailto:poljar@termina.org.uk)>. -Linux 5.1.3-arch2-1-ARCH - May 8, 2019 +Linux 5.11.16-arch1-1 - May 8, 2019 diff --git a/pantalaimon/config.py b/pantalaimon/config.py index 2a01714..5a5ca4f 100644 --- a/pantalaimon/config.py +++ b/pantalaimon/config.py @@ -39,6 +39,7 @@ class PanConfigParser(configparser.ConfigParser): "IndexingBatchSize": "100", "HistoryFetchDelay": "3000", "DebugEncryption": "False", + "DropOldKeys": "False", }, converters={ "address": parse_address, @@ -121,6 +122,8 @@ class ServerConfig: the room history. history_fetch_delay (int): The delay between room history fetching requests in seconds. + drop_old_keys (bool): Should Pantalaimon only keep the most recent + decryption key around. """ name = attr.ib(type=str) @@ -137,6 +140,7 @@ class ServerConfig: index_encrypted_only = attr.ib(type=bool, default=True) indexing_batch_size = attr.ib(type=int, default=100) history_fetch_delay = attr.ib(type=int, default=3) + drop_old_keys = attr.ib(type=bool, default=False) @attr.s @@ -229,6 +233,7 @@ class PanConfig: f"already defined before." ) listen_set.add(listen_tuple) + drop_old_keys = section.getboolean("DropOldKeys") server_conf = ServerConfig( section_name, @@ -243,6 +248,7 @@ class PanConfig: index_encrypted_only, indexing_batch_size, history_fetch_delay / 1000, + drop_old_keys, ) self.servers[section_name] = server_conf diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 0108cdb..3a5d067 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -29,6 +29,7 @@ from logbook import StderrHandler from pantalaimon.config import PanConfig, PanConfigError, parse_log_level from pantalaimon.daemon import ProxyDaemon from pantalaimon.log import logger +from pantalaimon.store import KeyDroppingSqliteStore from pantalaimon.thread_messages import DaemonResponse from pantalaimon.ui import UI_ENABLED @@ -47,6 +48,8 @@ def create_dirs(data_dir, conf_dir): async def init(data_dir, server_conf, send_queue, recv_queue): """Initialize the proxy and the http server.""" + store_class = KeyDroppingSqliteStore if server_conf.drop_old_keys else None + proxy = ProxyDaemon( server_conf.name, server_conf.homeserver, @@ -56,6 +59,7 @@ async def init(data_dir, server_conf, send_queue, recv_queue): recv_queue=recv_queue.async_q if recv_queue else None, proxy=server_conf.proxy.geturl() if server_conf.proxy else None, ssl=None if server_conf.ssl is True else False, + client_store_class=store_class, ) # 100 MB max POST size @@ -101,7 +105,6 @@ async def init(data_dir, server_conf, send_queue, recv_queue): r"/_matrix/client/r0/profile/{userId}/avatar_url", proxy.profile, ), - ] ) app.router.add_route("*", "/" + "{proxyPath:.*}", proxy.router) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 3e647f6..0d1afd5 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -18,10 +18,12 @@ from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple import attr -from nio.crypto import TrustState +from nio.crypto import TrustState, GroupSessionStore from nio.store import ( Accounts, + MegolmInboundSessions, DeviceKeys, + SqliteStore, DeviceTrustState, use_database, use_database_atomic, @@ -29,7 +31,6 @@ from nio.store import ( from peewee import SQL, DoesNotExist, ForeignKeyField, Model, SqliteDatabase, TextField from cachetools import LRUCache - MAX_LOADED_MEDIA = 10000 MAX_LOADED_UPLOAD = 10000 @@ -452,3 +453,47 @@ class PanStore: store[account.user_id] = device_store return store + + +class KeyDroppingSqliteStore(SqliteStore): + @use_database + def save_inbound_group_session(self, session): + """Save the provided Megolm inbound group session to the database. + + Args: + session (InboundGroupSession): The session to save. + """ + account = self._get_account() + assert account + + MegolmInboundSessions.delete().where( + MegolmInboundSessions.sender_key == session.sender_key, + MegolmInboundSessions.account == account, + MegolmInboundSessions.room_id == session.room_id, + ).execute() + + super().save_inbound_group_session(session) + + @use_database + def load_inbound_group_sessions(self): + store = super().load_inbound_group_sessions() + + return KeyDroppingGroupSessionStore.from_group_session_store(store) + + +class KeyDroppingGroupSessionStore(GroupSessionStore): + def from_group_session_store(store): + new_store = KeyDroppingGroupSessionStore() + new_store._entries = store._entries + + return new_store + + def add(self, session) -> bool: + room_id = session.room_id + sender_key = session.sender_key + if session in self._entries[room_id][sender_key].values(): + return False + + self._entries[room_id][sender_key].clear() + self._entries[room_id][sender_key][session.id] = session + return True From 90cdc5545162afa28b70c5f5f2f7e8e6636b8f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 28 May 2021 14:30:04 +0200 Subject: [PATCH 090/146] pantalaimon: Reformat our source files using black --- pantalaimon/daemon.py | 52 +++++++++++++++++++++++++++++++++---------- pantalaimon/store.py | 13 +++++------ 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 9693c9c..a807724 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -85,6 +85,7 @@ CORS_HEADERS = { class NotDecryptedAvailableError(Exception): """Exception that signals that no decrypted upload is available""" + pass @@ -864,8 +865,9 @@ class ProxyDaemon: if not upload_info or not media_info: raise NotDecryptedAvailableError - response, decrypted_file = await self._load_decrypted_file(media_info.mxc_server, media_info.mxc_path, - upload_info.filename) + response, decrypted_file = await self._load_decrypted_file( + media_info.mxc_server, media_info.mxc_path, upload_info.filename + ) if response is None and decrypted_file is None: raise NotDecryptedAvailableError @@ -923,10 +925,17 @@ class ProxyDaemon: # The room isn't encrypted just forward the message. if not encrypt: content_msgtype = content.get("msgtype") - if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": + if ( + content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] + or msgtype == "m.room.avatar" + ): try: - content = await self._map_decrypted_uri("url", content, request, client) - return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) + content = await self._map_decrypted_uri( + "url", content, request, client + ) + return await self.forward_to_web( + request, data=json.dumps(content), token=client.access_token + ) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) except (KeyError, NotDecryptedAvailableError): @@ -939,8 +948,13 @@ class ProxyDaemon: async def _send(ignore_unverified=False): try: content_msgtype = content.get("msgtype") - if content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar": - upload_info, media_info = self._get_upload_and_media_info("url", content) + if ( + content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] + or msgtype == "m.room.avatar" + ): + upload_info, media_info = self._get_upload_and_media_info( + "url", content + ) if not upload_info or not media_info: response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified @@ -1169,14 +1183,22 @@ class ProxyDaemon: body=await response.transport_response.read(), ) - self.store.save_upload(self.name, response.content_uri, file_name, content_type) + self.store.save_upload( + self.name, response.content_uri, file_name, content_type + ) mxc = urlparse(response.content_uri) mxc_server = mxc.netloc.strip("/") mxc_path = mxc.path.strip("/") logger.info(f"Adding media info for {mxc_server}/{mxc_path} to the store") - media_info = MediaInfo(mxc_server, mxc_path, maybe_keys["key"], maybe_keys["iv"], maybe_keys["hashes"]) + media_info = MediaInfo( + mxc_server, + mxc_path, + maybe_keys["key"], + maybe_keys["iv"], + maybe_keys["hashes"], + ) self.store.save_media(self.name, media_info) return web.Response( @@ -1250,8 +1272,12 @@ class ProxyDaemon: return self._not_json try: - content = await self._map_decrypted_uri("avatar_url", content, request, client) - return await self.forward_to_web(request, data=json.dumps(content), token=client.access_token) + content = await self._map_decrypted_uri( + "avatar_url", content, request, client + ) + return await self.forward_to_web( + request, data=json.dumps(content), token=client.access_token + ) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) except (KeyError, NotDecryptedAvailableError): @@ -1263,7 +1289,9 @@ class ProxyDaemon: file_name = request.match_info.get("file_name") try: - response, decrypted_file = await self._load_decrypted_file(server_name, media_id, file_name) + response, decrypted_file = await self._load_decrypted_file( + server_name, media_id, file_name + ) if response is None and decrypted_file is None: return await self.forward_to_web(request) diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 0d1afd5..f1c35e9 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -51,15 +51,14 @@ class MediaInfo: def to_content(self, content: Dict, mime_type: str) -> Dict[Any, Any]: content["file"] = { - "v": "v2", - "key": self.key, - "iv": self.iv, - "hashes": self.hashes, - "url": content["url"], - "mimetype": mime_type, + "v": "v2", + "key": self.key, + "iv": self.iv, + "hashes": self.hashes, + "url": content["url"], + "mimetype": mime_type, } - def to_thumbnail(self, content: Dict, mime_type: str) -> Dict[Any, Any]: content["info"]["thumbnail_file"] = { "v": "v2", From 2b359c22d2bd3138de17d952f685c687daca9563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 17 May 2021 10:36:52 +0200 Subject: [PATCH 091/146] store: Split out the media cache loading logic from the load media method --- pantalaimon/daemon.py | 2 +- pantalaimon/store.py | 42 ++++++++++++++++++++++-------------------- tests/store_test.py | 4 ++-- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index a807724..b238ba7 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -121,7 +121,7 @@ class ProxyDaemon: self.hostname = self.homeserver.hostname self.store = PanStore(self.data_dir) accounts = self.store.load_users(self.name) - self.media_info = self.store.load_media(self.name) + self.media_info = self.store.load_media_cache(self.name) self.upload_info = self.store.load_upload(self.name) for user_id, device_id in accounts: diff --git a/pantalaimon/store.py b/pantalaimon/store.py index f1c35e9..60a36a4 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -254,32 +254,34 @@ class PanStore: hashes=media.hashes, ).on_conflict_ignore().execute() + @use_database + def load_media_cache(self, server): + server, _ = Servers.get_or_create(name=server) + media_cache = LRUCache(maxsize=MAX_LOADED_MEDIA) + + for i, m in enumerate(server.media): + if i > MAX_LOADED_MEDIA: + break + + media = MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) + media_cache[(m.mxc_server, m.mxc_path)] = media + + return media_cache + @use_database def load_media(self, server, mxc_server=None, mxc_path=None): server, _ = Servers.get_or_create(name=server) - if not mxc_path: - media_cache = LRUCache(maxsize=MAX_LOADED_MEDIA) + m = PanMediaInfo.get_or_none( + PanMediaInfo.server == server, + PanMediaInfo.mxc_server == mxc_server, + PanMediaInfo.mxc_path == mxc_path, + ) - for i, m in enumerate(server.media): - if i > MAX_LOADED_MEDIA: - break + if not m: + return None - media = MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) - media_cache[(m.mxc_server, m.mxc_path)] = media - - return media_cache - else: - m = PanMediaInfo.get_or_none( - PanMediaInfo.server == server, - PanMediaInfo.mxc_server == mxc_server, - PanMediaInfo.mxc_path == mxc_path, - ) - - if not m: - return None - - return MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) + return MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) @use_database_atomic def replace_fetcher_task(self, server, pan_user, old_task, new_task): diff --git a/tests/store_test.py b/tests/store_test.py index fe7411e..1d16e10 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -153,7 +153,7 @@ class TestClass(object): def test_media_storage(self, panstore): server_name = "test" - media_cache = panstore.load_media(server_name) + media_cache = panstore.load_media_cache(server_name) assert not media_cache event = self.encrypted_media_event @@ -171,7 +171,7 @@ class TestClass(object): panstore.save_media(server_name, media) - media_cache = panstore.load_media(server_name) + media_cache = panstore.load_media_cache(server_name) assert (mxc_server, mxc_path) in media_cache media_info = media_cache[(mxc_server, mxc_path)] From a7286dfc9fd889eb58da33e899561494bea30f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 3 Jun 2021 10:39:37 +0200 Subject: [PATCH 092/146] pantalaimon: Bumb the version --- CHANGELOG.md | 16 ++++++++++++++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1dba35..7403103 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## 0.9.3 2021-05-14 +### Added + +- [[#98]] Add the ability to remove old room keys +- [[#95]] Encrypt thumbnails uploaded by a client + +### Fixed + +- [[#96]] Split out the media cache loading logic to avoid returning the + whole LRU cache when it shouldn't + +[#98]: https://github.com/matrix-org/pantalaimon/pull/98 +[#96]: https://github.com/matrix-org/pantalaimon/pull/96 +[#95]: https://github.com/matrix-org/pantalaimon/pull/95 + +## 0.9.3 2021-05-14 + ### Changed - [[#73f68c7]] Bump the allowed nio version diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 3a5d067..271a706 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -262,7 +262,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.9.3", prog_name="pantalaimon") +@click.version_option(version="0.10.0", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 1b486cc..26344a1 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.9.3", prog_name="panctl") +@click.version_option(version="0.10.0", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 011d3e4..a604a44 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.9.3", + version="0.10.0", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 48fd7037727696f379205e6e60f7d7cb078da3e1 Mon Sep 17 00:00:00 2001 From: CortexPE Date: Mon, 21 Jun 2021 17:24:02 +0800 Subject: [PATCH 093/146] Patch #90 --- pantalaimon/daemon.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index b238ba7..71f474a 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -835,9 +835,7 @@ class ProxyDaemon: body=await response.read(), ) - def _get_upload_and_media_info(self, content_key, content): - content_uri = content[content_key] - + def _get_upload_and_media_info(self, content_uri: str): try: upload_info = self.upload_info[content_uri] except KeyError: @@ -847,7 +845,6 @@ class ProxyDaemon: self.upload_info[content_uri] = upload_info - content_uri = content[content_key] mxc = urlparse(content_uri) mxc_server = mxc.netloc.strip("/") mxc_path = mxc.path.strip("/") @@ -860,8 +857,8 @@ class ProxyDaemon: return upload_info, media_info - async def _map_decrypted_uri(self, content_key, content, request, client): - upload_info, media_info = self._get_upload_and_media_info(content_key, content) + async def _decrypt_uri(self, content_uri, client): + upload_info, media_info = self._get_upload_and_media_info(content_uri) if not upload_info or not media_info: raise NotDecryptedAvailableError @@ -877,7 +874,7 @@ class ProxyDaemon: decrypted_upload, _ = await client.upload( data_provider=BufferedReader(BytesIO(decrypted_file)), - content_type=response.content_type, + content_type=upload_info.mimetype, filename=upload_info.filename, encrypt=False, filesize=len(decrypted_file), @@ -886,9 +883,7 @@ class ProxyDaemon: if not isinstance(decrypted_upload, UploadResponse): raise NotDecryptedAvailableError - content[content_key] = decrypted_upload.content_uri - - return content + return decrypted_upload.content_uri async def send_message(self, request): access_token = self.get_access_token(request) @@ -930,9 +925,8 @@ class ProxyDaemon: or msgtype == "m.room.avatar" ): try: - content = await self._map_decrypted_uri( - "url", content, request, client - ) + content["url"] = await self._decrypt_uri(content["url"], client) + content["info"]["thumbnail_url"] = await self._decrypt_uri(content["info"]["thumbnail_url"], client) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) @@ -952,9 +946,7 @@ class ProxyDaemon: content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar" ): - upload_info, media_info = self._get_upload_and_media_info( - "url", content - ) + upload_info, media_info = self._get_upload_and_media_info(content["url"]) if not upload_info or not media_info: response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified @@ -973,7 +965,7 @@ class ProxyDaemon: thumb_upload_info, thumb_media_info, ) = self._get_upload_and_media_info( - "thumbnail_url", content["info"] + content["info"]["thumbnail_url"] ) if thumb_upload_info and thumb_media_info: thumb_media_info.to_thumbnail( @@ -1272,9 +1264,7 @@ class ProxyDaemon: return self._not_json try: - content = await self._map_decrypted_uri( - "avatar_url", content, request, client - ) + content["avatar_url"] = await self._decrypt_uri(content["avatar_url"], client) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) From c455b37b67573531fd0f96a292d67fac32173b09 Mon Sep 17 00:00:00 2001 From: CortexPE Date: Mon, 21 Jun 2021 17:32:51 +0800 Subject: [PATCH 094/146] prevent touching thumbnail info when not needed --- pantalaimon/daemon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 71f474a..a2f5a37 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -926,7 +926,8 @@ class ProxyDaemon: ): try: content["url"] = await self._decrypt_uri(content["url"], client) - content["info"]["thumbnail_url"] = await self._decrypt_uri(content["info"]["thumbnail_url"], client) + if "info" in content and "thumbnail_url" in content["info"]: + content["info"]["thumbnail_url"] = await self._decrypt_uri(content["info"]["thumbnail_url"], client) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) From 95bf21765708d4c85257f882cfbfde2487f1e55b Mon Sep 17 00:00:00 2001 From: CortexPE Date: Fri, 2 Jul 2021 19:45:40 +0800 Subject: [PATCH 095/146] patch KeyError on sync --- pantalaimon/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index b2d2a72..76d31a2 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -905,7 +905,7 @@ class PanClient(AsyncClient): self.handle_to_device_from_sync_body(body) - for room_id, room_dict in body["rooms"]["join"].items(): + for room_id, room_dict in body.get("rooms", {}).get("join", {}).items(): try: if not self.rooms[room_id].encrypted: logger.info( @@ -920,7 +920,7 @@ class PanClient(AsyncClient): # pan sync stream did. Let's assume that the room is encrypted. pass - for event in room_dict["timeline"]["events"]: + for event in room_dict.get("timeline", {}).get("events", []): if "type" not in event: continue From c36aca183b1cbac7eadeac028744151196c34ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 6 Jul 2021 10:41:23 +0200 Subject: [PATCH 096/146] pantalaimon: Bump our version --- CHANGELOG.md | 14 +++++++++++++- pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7403103..f3a6066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## 0.9.3 2021-05-14 +## 0.10.1 2021-07-06 + +### Fixed + +- [[#100]] Don't require the rooms dicts in the sync response +- [[#99]] Thumbnails not generating for media uploaded in unencrypted rooms + whole LRU cache when it shouldn't + +[#100]: https://github.com/matrix-org/pantalaimon/pull/100 +[#99]: https://github.com/matrix-org/pantalaimon/pull/99 + + +## 0.10.0 2021-05-14 ### Added diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 271a706..84e9429 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -262,7 +262,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.10.0", prog_name="pantalaimon") +@click.version_option(version="0.10.1", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 26344a1..7f76bf9 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.10.0", prog_name="panctl") +@click.version_option(version="0.10.1", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index a604a44..7f7d78f 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.10.0", + version="0.10.1", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From 340dbf2eb6d2c3f3a20f985bdd25cca0934bec78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Jul 2021 10:44:03 +0200 Subject: [PATCH 097/146] pantalaimon: Prevent the sending of messages if the client didn't sync Pantalaimon depends on the room state to be present to decide correctly if a message should be encrypted or now. The room state is fetched from the server every time we restart, this means that failure to fetch room states from the server would hinder us from doing the correct decision. This patch prevents a downgrade to sending unencrypted messages if we're unable to sync at least once with the server. --- pantalaimon/client.py | 5 ++++- pantalaimon/daemon.py | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index 76d31a2..e0188f2 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -410,6 +410,10 @@ class PanClient(AsyncClient): except (asyncio.CancelledError, KeyboardInterrupt): return + @property + def has_been_synced(self) -> bool: + self.last_sync_token is not None + async def sync_tasks(self, response): if self.index: await self.index.commit_events() @@ -540,7 +544,6 @@ class PanClient(AsyncClient): timeout = 30000 sync_filter = {"room": {"state": {"lazy_load_members": True}}} next_batch = self.pan_store.load_token(self.server_name, self.user_id) - self.last_sync_token = next_batch # We don't store any room state so initial sync needs to be with the # full_state parameter. Subsequent ones are normal. diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index a2f5a37..bb9b8e5 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -902,7 +902,22 @@ class ProxyDaemon: room = client.rooms[room_id] encrypt = room.encrypted except KeyError: - return await self.forward_to_web(request, token=client.access_token) + if client.has_been_synced: + return await self.forward_to_web(request, token=client.access_token) + else: + logger.error( + "The internal Pantalaimon client did not manage " + "to sync with the server." + ) + return web.json_response( + { + "errcode": "M_UNKNOWN", + "error": "The pantalaimon client did not manage to sync with " + "the server", + }, + headers=CORS_HEADERS, + status=500, + ) # Don't encrypt reactions for now - they are weird and clients # need to support them like this. From f875499d6b0a9d968270872a4a81527517dd703e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Jul 2021 11:05:02 +0200 Subject: [PATCH 098/146] pantalaimon: Fix some formatting issues --- pantalaimon/daemon.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index bb9b8e5..bdba6a8 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -942,7 +942,9 @@ class ProxyDaemon: try: content["url"] = await self._decrypt_uri(content["url"], client) if "info" in content and "thumbnail_url" in content["info"]: - content["info"]["thumbnail_url"] = await self._decrypt_uri(content["info"]["thumbnail_url"], client) + content["info"]["thumbnail_url"] = await self._decrypt_uri( + content["info"]["thumbnail_url"], client + ) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) @@ -962,7 +964,9 @@ class ProxyDaemon: content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] or msgtype == "m.room.avatar" ): - upload_info, media_info = self._get_upload_and_media_info(content["url"]) + upload_info, media_info = self._get_upload_and_media_info( + content["url"] + ) if not upload_info or not media_info: response = await client.room_send( room_id, msgtype, content, txnid, ignore_unverified @@ -1280,7 +1284,9 @@ class ProxyDaemon: return self._not_json try: - content["avatar_url"] = await self._decrypt_uri(content["avatar_url"], client) + content["avatar_url"] = await self._decrypt_uri( + content["avatar_url"], client + ) return await self.forward_to_web( request, data=json.dumps(content), token=client.access_token ) From a9f6ad2c7e08c72e8274a6bdf2954edf1c2d6b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Jul 2021 12:25:39 +0200 Subject: [PATCH 099/146] pantalaimon: Don't forward messages if we think the room isn't joined --- pantalaimon/daemon.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index bdba6a8..bbec91a 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -897,13 +897,22 @@ class ProxyDaemon: room_id = request.match_info["room_id"] - # The room is not in the joined rooms list, just forward it. try: room = client.rooms[room_id] encrypt = room.encrypted except KeyError: + # The room is not in the joined rooms list, either the pan client + # didn't manage to sync the state or we're not joined, in either + # case send an error response. if client.has_been_synced: - return await self.forward_to_web(request, token=client.access_token) + return web.json_response( + { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event." + }, + headers=CORS_HEADERS, + status=403, + ) else: logger.error( "The internal Pantalaimon client did not manage " From e3be7bee3be0ceeae07b067ec4c90f5849b9763f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 14 Jul 2021 13:48:16 +0200 Subject: [PATCH 100/146] pantalaimon: Bump our version --- CHANGELOG.md | 9 +++++++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 4 ++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a6066..c0dda50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.10.2 2021-07-14 + +### Fixed + +- [[#103]] Prevent E2EE downgrade on failed syncs + +[#103]: https://github.com/matrix-org/pantalaimon/pull/103 + + ## 0.10.1 2021-07-06 ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 84e9429..fe5e167 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -262,7 +262,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.10.1", prog_name="pantalaimon") +@click.version_option(version="0.10.2", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 7f76bf9..fdb9921 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.10.1", prog_name="panctl") +@click.version_option(version="0.10.2", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 7f7d78f..b13317a 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.10.1", + version="0.10.2", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.14, < 0.19" + "matrix-nio[e2e] >= 0.18, < 0.19" ], extras_require={ "ui": [ From 054a3bcb0b39a33553a9898153443056dc1832fd Mon Sep 17 00:00:00 2001 From: "Jan Alexander Steffens (heftig)" Date: Thu, 26 Aug 2021 20:29:53 +0200 Subject: [PATCH 101/146] pantalaimon: Forward raw path instead of quoting the decoded path If the original path had quoted slash characters (`%2F`), the code decoded them and then left them unquoted, causing the forwarded request to fail. An example is trying to ban a user with a slash in their name using Mjolnir, which PUTs a state event with `rule:` in the URL. Use the undecoded path to avoid this. --- pantalaimon/daemon.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index bbec91a..ffe0f57 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -482,9 +482,7 @@ class ProxyDaemon: assert session - path = urllib.parse.quote( - request.path - ) # re-encode path stuff like room aliases + path = request.raw_path method = request.method headers = CIMultiDict(request.headers) From ed7aa55ef0eeac9112c6902a86823c218ef40ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 2 Sep 2021 11:42:50 +0200 Subject: [PATCH 102/146] pantalaimon: Bump our version --- CHANGELOG.md | 10 ++++++++++ pantalaimon/panctl.py | 2 +- setup.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0dda50..1c4dfcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.10.3 2021-09-02 + +### Fixed + +- [[#105]] Use the raw_path when forwarding requests, avoiding URL + decoding/encoding issues. + +[#105]: https://github.com/matrix-org/pantalaimon/pull/105 + + ## 0.10.2 2021-07-14 ### Fixed diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index fdb9921..f418099 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -690,7 +690,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.10.2", prog_name="panctl") +@click.version_option(version="0.10.3", prog_name="panctl") def main(): loop = asyncio.get_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index b13317a..0eda577 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.10.2", + version="0.10.3", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", From cd36ca68d5edb2d33f5a015ab04d8be4bf037912 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Sat, 4 Sep 2021 13:13:50 +0200 Subject: [PATCH 103/146] thumbnail_url can be None --- pantalaimon/daemon.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index ffe0f57..d080bfb 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -948,7 +948,10 @@ class ProxyDaemon: ): try: content["url"] = await self._decrypt_uri(content["url"], client) - if "info" in content and "thumbnail_url" in content["info"]: + if ( + "info" in content and "thumbnail_url" in content["info"] + and not content["info"]["thumbnail_url"] == None + ): content["info"]["thumbnail_url"] = await self._decrypt_uri( content["info"]["thumbnail_url"], client ) From e5fb0b7f17c16ede41b87a5cb3feaf2fd7c95935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 2 Feb 2022 14:59:40 +0100 Subject: [PATCH 104/146] ci: Enable github CI --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++++ tox.ini | 13 +++--------- 2 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4872347 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: Build Status + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10'] + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Tox and any other packages + run: | + wget https://gitlab.matrix.org/matrix-org/olm/-/archive/master/olm-master.tar.bz2 + tar -xvf olm-master.tar.bz2 + pushd olm-master && make && sudo make PREFIX="/usr" install && popd + rm -r olm-master + pip install tox + - name: Run Tox + run: tox -e py + + coverage: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install Tox and any other packages + run: | + wget https://gitlab.matrix.org/matrix-org/olm/-/archive/master/olm-master.tar.bz2 + tar -xvf olm-master.tar.bz2 + pushd olm-master && make && sudo make PREFIX="/usr" install && popd + rm -r olm-master + pip install tox + - name: Run Tox + run: tox -e coverage diff --git a/tox.ini b/tox.ini index 2fb9dd8..c3a2630 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,14 @@ -# content of: tox.ini , put in same dir as setup.py [tox] -envlist = py38,py39,coverage -[testenv] -basepython = - py38: python3.8 - py39: python3.9 - py3: python3.9 +envlist = coverage +[testenv] deps = -rtest-requirements.txt install_command = pip install {opts} {packages} -passenv = TOXENV CI TRAVIS TRAVIS_* +passenv = TOXENV CI commands = pytest -usedevelop = True [testenv:coverage] -basepython = python3.9 commands = pytest --cov=pantalaimon --cov-report term-missing coverage xml From c89e87c22af991d07327d710d9862c5521c739b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 2 Feb 2022 14:41:18 +0100 Subject: [PATCH 105/146] fix(daemon): Don't use the raw path if we need to sanitize filters Using the raw path sabotages the filter sanitization. --- pantalaimon/daemon.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index d080bfb..68cbb97 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -460,6 +460,7 @@ class ProxyDaemon: data=None, # type: bytes session=None, # type: aiohttp.ClientSession token=None, # type: str + use_raw_path=True, # type: bool ): # type: (...) -> aiohttp.ClientResponse """Forward the given request to our configured homeserver. @@ -474,6 +475,10 @@ class ProxyDaemon: should be used to forward the request. token (str, optional): The access token that should be used for the request. + use_raw_path (str, optional): Should the raw path be used from the + request or should we use the path and re-encode it. Some may need + their filters to be sanitized, this requires the parsed version of + the path, otherise we leave the path as is. """ if not session: if not self.default_session: @@ -482,7 +487,7 @@ class ProxyDaemon: assert session - path = request.raw_path + path = request.raw_path if use_raw_path else urllib.parse.quote(request.path) method = request.method headers = CIMultiDict(request.headers) @@ -761,7 +766,7 @@ class ProxyDaemon: try: response = await self.forward_request( - request, params=query, token=client.access_token + request, params=query, token=client.access_token, use_raw_path=False ) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) @@ -809,7 +814,9 @@ class ProxyDaemon: query["filter"] = request_filter try: - response = await self.forward_request(request, params=query) + response = await self.forward_request( + request, params=query, use_raw_path=False + ) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) From e62cfe068aa32408a7eb78a95aa1764b6f432011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 2 Feb 2022 14:44:49 +0100 Subject: [PATCH 106/146] fix(daemon): Don't use strip to filter Bearer from auth header Using strip might remove more than what is intended here. --- pantalaimon/daemon.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 68cbb97..f8fa159 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -418,7 +418,9 @@ class ProxyDaemon: access_token = request.query.get("access_token", "") if not access_token: - access_token = request.headers.get("Authorization", "").strip("Bearer ") + access_token = request.headers.get("Authorization", "").replace( + "Bearer ", "", 1 + ) return access_token From 3dd80517070b1817c0e60553e177f2ab0096f319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 2 Feb 2022 14:43:30 +0100 Subject: [PATCH 107/146] chore: Style fixes --- pantalaimon/daemon.py | 7 ++++--- pantalaimon/index.py | 1 - pantalaimon/main.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index f8fa159..aec62ea 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -915,7 +915,7 @@ class ProxyDaemon: return web.json_response( { "errcode": "M_FORBIDDEN", - "error": "You do not have permission to send the event." + "error": "You do not have permission to send the event.", }, headers=CORS_HEADERS, status=403, @@ -958,8 +958,9 @@ class ProxyDaemon: try: content["url"] = await self._decrypt_uri(content["url"], client) if ( - "info" in content and "thumbnail_url" in content["info"] - and not content["info"]["thumbnail_url"] == None + "info" in content + and "thumbnail_url" in content["info"] + and content["info"]["thumbnail_url"] is not None ): content["info"]["thumbnail_url"] = await self._decrypt_uri( content["info"]["thumbnail_url"], client diff --git a/pantalaimon/index.py b/pantalaimon/index.py index 5c8e02b..7350e58 100644 --- a/pantalaimon/index.py +++ b/pantalaimon/index.py @@ -501,6 +501,5 @@ if False: return search_result - else: INDEXING_ENABLED = False diff --git a/pantalaimon/main.py b/pantalaimon/main.py index fe5e167..2f25f64 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -63,7 +63,7 @@ async def init(data_dir, server_conf, send_queue, recv_queue): ) # 100 MB max POST size - app = web.Application(client_max_size=1024 ** 2 * 100) + app = web.Application(client_max_size=1024**2 * 100) app.add_routes( [ From 86060a2f7561d40bd142c8bc1847e151c3d40dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 3 Feb 2022 11:42:37 +0100 Subject: [PATCH 108/146] fix(panctl): Allow GLib to be imported from pgi Some distributions seem to have renamed the gi module to pgi, try to import GLib from pgi if importing from gi fails. --- pantalaimon/panctl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index f418099..db7869f 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -23,7 +23,12 @@ from typing import List import attr import click -from gi.repository import GLib + +try: + from gi.repository import GLib +except ModuleNotFoundError: + from pgi.repository import GLib + from prompt_toolkit import __version__ as ptk_version from prompt_toolkit import HTML, PromptSession, print_formatted_text from prompt_toolkit.completion import Completer, Completion, PathCompleter From 82fcbf8e855fbe99cda05ff5128a5fbb9e58a2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 3 Feb 2022 11:45:07 +0100 Subject: [PATCH 109/146] fix(pancl): Ensure that we create a event loop at the start --- pantalaimon/panctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index db7869f..7ae82a6 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -697,7 +697,7 @@ class PanCtl: ) @click.version_option(version="0.10.3", prog_name="panctl") def main(): - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() glib_loop = GLib.MainLoop() try: From 85c7b685e5c48a43ad74876edd6d7973fa7883ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 4 Feb 2022 15:15:45 +0100 Subject: [PATCH 110/146] Bump our version --- CHANGELOG.md | 13 +++++++++++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 4 ++-- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c4dfcb..168f740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.10.4 2022-02-04 + +### Fixed + +- [[#122]] Fix the GLib import for panctl on some distributions +- [[#120]] Don't use strip to filter Bearer from the auth header +- [[#118]] Don't use the raw path if we need to sanitize filters, fixing room + history fetching for Fractal + +[#122]: https://github.com/matrix-org/pantalaimon/pull/122 +[#120]: https://github.com/matrix-org/pantalaimon/pull/120 +[#118]: https://github.com/matrix-org/pantalaimon/pull/118 + ## 0.10.3 2021-09-02 ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 2f25f64..8ffb6ed 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -262,7 +262,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.10.2", prog_name="pantalaimon") +@click.version_option(version="0.10.4", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 7ae82a6..8652a4c 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -695,7 +695,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.10.3", prog_name="panctl") +@click.version_option(version="0.10.4", prog_name="panctl") def main(): loop = asyncio.new_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 0eda577..5495f67 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.10.3", + version="0.10.4", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.18, < 0.19" + "matrix-nio[e2e] >= 0.18, < 0.20" ], extras_require={ "ui": [ From eb4f3b54b4a4df807ea069aba2085362e05a05a6 Mon Sep 17 00:00:00 2001 From: Kim Brose Date: Tue, 5 Apr 2022 13:27:00 +0200 Subject: [PATCH 111/146] clarify wording in README#Usage --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index af1563b..39056c0 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ IgnoreVerification = True Usage ===== -While pantalaimon is a daemon, it is meant to be run as your own user. It won't +While pantalaimon is a daemon, it is meant to be run as the same user as the app it is proxying for. It won't verify devices for you automatically, unless configured to do so, and requires user interaction to verify, ignore or blacklist devices. A more complete description of Pantalaimon can be found in the [man page](docs/man/pantalaimon.8.md). From 109ceed0bb53b07c9c1b4ff94e806b044f397b8d Mon Sep 17 00:00:00 2001 From: Alfonso Montero Date: Fri, 29 Apr 2022 12:20:37 +0000 Subject: [PATCH 112/146] Readme: add alternate Docker build method --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index af1563b..3ddb32b 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,10 @@ docker build -t pantalaimon . # volume below is for where Pantalaimon should dump some data. docker run -it --rm -v /path/to/pantalaimon/dir:/data -p 8008:8008 pantalaimon ``` +The Docker image in the above example can alternatively be built straight from any branch or tag without the need to clone the repo, just by using this syntax: +```bash +docker build -t pantalaimon github.com/matrix-org/pantalaimon#master +``` An example `pantalaimon.conf` for Docker is: ```conf From 2883df45c061ca2830818f0a2c153c0a1f405564 Mon Sep 17 00:00:00 2001 From: Christian Paul Date: Mon, 5 Sep 2022 11:09:19 +0200 Subject: [PATCH 113/146] Update pantalaimon.8.md: it's own -> its own --- docs/man/pantalaimon.8.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/man/pantalaimon.8.md b/docs/man/pantalaimon.8.md index 431d5a4..004e5f2 100644 --- a/docs/man/pantalaimon.8.md +++ b/docs/man/pantalaimon.8.md @@ -24,7 +24,7 @@ behalf of the client. is supposed to run as your own user and listen to connections on a non-privileged port. A client needs to log in using the standard Matrix HTTP calls to register itself to the daemon, such a registered user is called a pan -user and will have it's own sync loop to keep up with the server. Multiple matrix +user and will have its own sync loop to keep up with the server. Multiple matrix clients can connect and use the same pan user. If user interaction is required From 6b7b87bb6a0f4cd341e2b9ae0d5d94c7aa317dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 24 May 2022 15:05:58 +0200 Subject: [PATCH 114/146] fix(daemon): Make sure the token variable is declared --- pantalaimon/daemon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index aec62ea..e9704d1 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -125,6 +125,8 @@ class ProxyDaemon: self.upload_info = self.store.load_upload(self.name) for user_id, device_id in accounts: + token = None + if self.conf.keyring: try: token = keyring.get_password( From 4254a5d9d50183a37df28e93d0a2d3fd63ceb33e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 12 Sep 2022 13:44:23 +0200 Subject: [PATCH 115/146] feat(daemon): Proxy the v3 endpoints as well --- pantalaimon/main.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 8ffb6ed..3002ee1 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -68,28 +68,45 @@ async def init(data_dir, server_conf, send_queue, recv_queue): app.add_routes( [ web.post("/_matrix/client/r0/login", proxy.login), + web.post("/_matrix/client/v3/login", proxy.login), web.get("/_matrix/client/r0/sync", proxy.sync), + web.get("/_matrix/client/v3/sync", proxy.sync), web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages), + web.get("/_matrix/client/v3/rooms/{room_id}/messages", proxy.messages), web.put( r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}", proxy.send_message, ), + web.put( + r"/_matrix/client/v3/rooms/{room_id}/send/{event_type}/{txnid}", + proxy.send_message, + ), web.post( r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}", proxy.send_message, ), web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter), + web.post("/_matrix/client/v3/user/{user_id}/filter", proxy.filter), web.post("/.well-known/matrix/client", proxy.well_known), web.get("/.well-known/matrix/client", proxy.well_known), web.post("/_matrix/client/r0/search", proxy.search), + web.post("/_matrix/client/v3/search", proxy.search), web.options("/_matrix/client/r0/search", proxy.search_opts), + web.options("/_matrix/client/v3/search", proxy.search_opts), web.get( "/_matrix/media/v1/download/{server_name}/{media_id}", proxy.download ), + web.get( + "/_matrix/media/v3/download/{server_name}/{media_id}", proxy.download + ), web.get( "/_matrix/media/v1/download/{server_name}/{media_id}/{file_name}", proxy.download, ), + web.get( + "/_matrix/media/v3/download/{server_name}/{media_id}/{file_name}", + proxy.download, + ), web.get( "/_matrix/media/r0/download/{server_name}/{media_id}", proxy.download ), @@ -101,10 +118,18 @@ async def init(data_dir, server_conf, send_queue, recv_queue): r"/_matrix/media/r0/upload", proxy.upload, ), + web.post( + r"/_matrix/media/v3/upload", + proxy.upload, + ), web.put( r"/_matrix/client/r0/profile/{userId}/avatar_url", proxy.profile, ), + web.put( + r"/_matrix/client/v3/profile/{userId}/avatar_url", + proxy.profile, + ), ] ) app.router.add_route("*", "/" + "{proxyPath:.*}", proxy.router) From 64a3fb0a48ad6659a28dc9773f95f5e80abd8989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 28 Sep 2022 16:32:39 +0200 Subject: [PATCH 116/146] chore: Fix a bunch of typos --- README.md | 2 +- docs/man/panctl.1 | 2 +- docs/man/panctl.md | 2 +- docs/man/pantalaimon.5 | 2 +- docs/man/pantalaimon.5.md | 2 +- pantalaimon/client.py | 4 ++-- pantalaimon/config.py | 4 ++-- pantalaimon/daemon.py | 20 ++++++++++---------- pantalaimon/panctl.py | 2 +- pantalaimon/ui.py | 36 ++++++++++++++++++------------------ tests/pan_client_test.py | 8 ++++---- 11 files changed, 42 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 9f6b174..47916d8 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Pantalaimon can also be found on pypi: pip install pantalaimon -Pantalaimon contains a dbus based UI that can be used to controll the daemon. +Pantalaimon contains a dbus based UI that can be used to control the daemon. The dbus based UI is completely optional and needs to be installed with the daemon: diff --git a/docs/man/panctl.1 b/docs/man/panctl.1 index e1ebb8e..705dd97 100644 --- a/docs/man/panctl.1 +++ b/docs/man/panctl.1 @@ -51,7 +51,7 @@ The message will be sent away after all devices are marked as ignored. In contrast to the .Cm send-anyways command this command cancels the sending of a message to an encrypted room with -unverified devices and gives the user the oportunity to verify or blacklist +unverified devices and gives the user the opportunity to verify or blacklist devices as they see fit. .It Cm import-keys Ar pan-user Ar file Ar passphrase Import end-to-end encryption keys from the given file for the given pan-user. diff --git a/docs/man/panctl.md b/docs/man/panctl.md index ed733f5..30fa8f4 100644 --- a/docs/man/panctl.md +++ b/docs/man/panctl.md @@ -74,7 +74,7 @@ are as follows: > In contrast to the > **send-anyways** > command this command cancels the sending of a message to an encrypted room with -> unverified devices and gives the user the oportunity to verify or blacklist +> unverified devices and gives the user the opportunity to verify or blacklist > devices as they see fit. **import-keys** *pan-user* *file* *passphrase* diff --git a/docs/man/pantalaimon.5 b/docs/man/pantalaimon.5 index 62b281e..01fd2d1 100644 --- a/docs/man/pantalaimon.5 +++ b/docs/man/pantalaimon.5 @@ -86,7 +86,7 @@ The amount of time to wait between room message history requests to the Homeserver in ms. Defaults to 3000. .El .Pp -Aditional to the homeserver section a special section with the name +Additional to the homeserver section a special section with the name .Cm Default can be used to configure the following values for all homeservers: .Cm ListenAddress , diff --git a/docs/man/pantalaimon.5.md b/docs/man/pantalaimon.5.md index 4a78e80..60be6f8 100644 --- a/docs/man/pantalaimon.5.md +++ b/docs/man/pantalaimon.5.md @@ -69,7 +69,7 @@ The following keys are optional in the proxy instance sections: > incoming messages will be decryptable, the proxy will be unable to decrypt the > room history. Defaults to "No". -Aditional to the homeserver section a special section with the name +Additional to the homeserver section a special section with the name **Default** can be used to configure the following values for all homeservers: **ListenAddress**, diff --git a/pantalaimon/client.py b/pantalaimon/client.py index e0188f2..5b4ce05 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -735,7 +735,7 @@ class PanClient(AsyncClient): pass response = ( - f"Succesfully continued the key requests from " + f"Successfully continued the key requests from " f"{message.user_id} via {message.device_id}" ) ret = "m.ok" @@ -760,7 +760,7 @@ class PanClient(AsyncClient): if cancelled: response = ( - f"Succesfully cancelled key requests from " + f"Successfully cancelled key requests from " f"{message.user_id} via {message.device_id}" ) ret = "m.ok" diff --git a/pantalaimon/config.py b/pantalaimon/config.py index 5a5ca4f..8ee51d2 100644 --- a/pantalaimon/config.py +++ b/pantalaimon/config.py @@ -31,7 +31,7 @@ class PanConfigParser(configparser.ConfigParser): "IgnoreVerification": "False", "ListenAddress": "localhost", "ListenPort": "8009", - "LogLevel": "warnig", + "LogLevel": "warning", "Notifications": "on", "UseKeyring": "yes", "SearchRequests": "off", @@ -113,7 +113,7 @@ class ServerConfig: E2E encrypted messages. keyring (bool): Enable or disable the OS keyring for the storage of access tokens. - search_requests (bool): Enable or disable aditional Homeserver requests + search_requests (bool): Enable or disable additional Homeserver requests for the search API endpoint. index_encrypted_only (bool): Enable or disable message indexing fro non-encrypted rooms. diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index e9704d1..3264cab 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -227,7 +227,7 @@ class ProxyDaemon: if ret: msg = ( - f"Device {device.id} of user " f"{device.user_id} succesfully verified." + f"Device {device.id} of user " f"{device.user_id} successfully verified." ) await client.send_update_device(device) else: @@ -242,7 +242,7 @@ class ProxyDaemon: if ret: msg = ( f"Device {device.id} of user " - f"{device.user_id} succesfully unverified." + f"{device.user_id} successfully unverified." ) await client.send_update_device(device) else: @@ -257,7 +257,7 @@ class ProxyDaemon: if ret: msg = ( f"Device {device.id} of user " - f"{device.user_id} succesfully blacklisted." + f"{device.user_id} successfully blacklisted." ) await client.send_update_device(device) else: @@ -274,7 +274,7 @@ class ProxyDaemon: if ret: msg = ( f"Device {device.id} of user " - f"{device.user_id} succesfully unblacklisted." + f"{device.user_id} successfully unblacklisted." ) await client.send_update_device(device) else: @@ -358,7 +358,7 @@ class ProxyDaemon: else: info_msg = ( - f"Succesfully exported keys for {client.user_id} " f"to {path}" + f"Successfully exported keys for {client.user_id} " f"to {path}" ) logger.info(info_msg) await self.send_response( @@ -381,7 +381,7 @@ class ProxyDaemon: ) else: info_msg = ( - f"Succesfully imported keys for {client.user_id} " f"from {path}" + f"Successfully imported keys for {client.user_id} " f"from {path}" ) logger.info(info_msg) await self.send_response( @@ -482,7 +482,7 @@ class ProxyDaemon: use_raw_path (str, optional): Should the raw path be used from the request or should we use the path and re-encode it. Some may need their filters to be sanitized, this requires the parsed version of - the path, otherise we leave the path as is. + the path, otherwise we leave the path as is. """ if not session: if not self.default_session: @@ -616,7 +616,7 @@ class ProxyDaemon: await pan_client.close() return - logger.info(f"Succesfully started new background sync client for " f"{user_id}") + logger.info(f"Successfully started new background sync client for " f"{user_id}") await self.send_ui_message( UpdateUsersMessage(self.name, user_id, pan_client.device_id) @@ -682,7 +682,7 @@ class ProxyDaemon: if user_id and access_token: logger.info( - f"User: {user} succesfully logged in, starting " + f"User: {user} successfully logged in, starting " f"a background sync client." ) await self.start_pan_client( @@ -1033,7 +1033,7 @@ class ProxyDaemon: except SendRetryError as e: return web.Response(status=503, text=str(e)) - # Aquire a semaphore here so we only send out one + # Acquire a semaphore here so we only send out one # UnverifiedDevicesSignal sem = client.send_semaphores[room_id] diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 8652a4c..088bcaa 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -464,7 +464,7 @@ class PanCtl: def sas_done(self, pan_user, user_id, device_id, _): print( f"Device {device_id} of user {user_id}" - f" succesfully verified for pan user {pan_user}." + f" successfully verified for pan user {pan_user}." ) def show_sas_invite(self, pan_user, user_id, device_id, _): diff --git a/pantalaimon/ui.py b/pantalaimon/ui.py index cedca65..813b67e 100644 --- a/pantalaimon/ui.py +++ b/pantalaimon/ui.py @@ -470,14 +470,14 @@ if UI_ENABLED: self.bus.publish("org.pantalaimon1", self.control_if, self.device_if) def unverified_notification(self, message): - notificaton = notify2.Notification( + notification = notify2.Notification( "Unverified devices.", message=( f"There are unverified devices in the room " f"{message.room_display_name}." ), ) - notificaton.set_category("im") + notification.set_category("im") def send_cb(notification, action_key, user_data): message = user_data @@ -488,20 +488,20 @@ if UI_ENABLED: self.control_if.CancelSending(message.pan_user, message.room_id) if "actions" in notify2.get_server_caps(): - notificaton.add_action("send", "Send anyways", send_cb, message) - notificaton.add_action("cancel", "Cancel sending", cancel_cb, message) + notification.add_action("send", "Send anyways", send_cb, message) + notification.add_action("cancel", "Cancel sending", cancel_cb, message) - notificaton.show() + notification.show() def sas_invite_notification(self, message): - notificaton = notify2.Notification( + notification = notify2.Notification( "Key verification invite", message=( f"{message.user_id} via {message.device_id} has started " f"a key verification process." ), ) - notificaton.set_category("im") + notification.set_category("im") def accept_cb(notification, action_key, user_data): message = user_data @@ -516,17 +516,17 @@ if UI_ENABLED: ) if "actions" in notify2.get_server_caps(): - notificaton.add_action("accept", "Accept", accept_cb, message) - notificaton.add_action("cancel", "Cancel", cancel_cb, message) + notification.add_action("accept", "Accept", accept_cb, message) + notification.add_action("cancel", "Cancel", cancel_cb, message) - notificaton.show() + notification.show() def sas_show_notification(self, message): emojis = [x[0] for x in message.emoji] emoji_str = " ".join(emojis) - notificaton = notify2.Notification( + notification = notify2.Notification( "Short authentication string", message=( f"Short authentication string for the key verification of" @@ -534,7 +534,7 @@ if UI_ENABLED: f"{emoji_str}" ), ) - notificaton.set_category("im") + notification.set_category("im") def confirm_cb(notification, action_key, user_data): message = user_data @@ -549,21 +549,21 @@ if UI_ENABLED: ) if "actions" in notify2.get_server_caps(): - notificaton.add_action("confirm", "Confirm", confirm_cb, message) - notificaton.add_action("cancel", "Cancel", cancel_cb, message) + notification.add_action("confirm", "Confirm", confirm_cb, message) + notification.add_action("cancel", "Cancel", cancel_cb, message) - notificaton.show() + notification.show() def sas_done_notification(self, message): - notificaton = notify2.Notification( + notification = notify2.Notification( "Device successfully verified.", message=( f"Device {message.device_id} of user {message.user_id} " f"successfully verified." ), ) - notificaton.set_category("im") - notificaton.show() + notification.set_category("im") + notification.show() def message_callback(self): try: diff --git a/tests/pan_client_test.py b/tests/pan_client_test.py index 7202698..5932432 100644 --- a/tests/pan_client_test.py +++ b/tests/pan_client_test.py @@ -28,7 +28,7 @@ ALICE_ID = "@alice:example.org" async def client(tmpdir, loop): store = PanStore(tmpdir) queue = janus.Queue() - conf = ServerConfig("example", "https://exapmle.org") + conf = ServerConfig("example", "https://example.org") conf.history_fetch_delay = 0.1 store.save_server_user("example", "@example:example.org") @@ -421,7 +421,7 @@ class TestClass(object): tasks = client.pan_store.load_fetcher_tasks(client.server_name, client.user_id) assert len(tasks) == 1 - # Check that the task is our prev_batch from the sync resposne + # Check that the task is our prev_batch from the sync response assert tasks[0].room_id == TEST_ROOM_ID assert tasks[0].token == "t392-516_47314_0_7_1_1_1_11444_1" @@ -431,7 +431,7 @@ class TestClass(object): tasks = client.pan_store.load_fetcher_tasks(client.server_name, client.user_id) assert len(tasks) == 1 - # Check that the task is our end token from the messages resposne + # Check that the task is our end token from the messages response assert tasks[0].room_id == TEST_ROOM_ID assert tasks[0].token == "t47409-4357353_219380_26003_2265" @@ -519,7 +519,7 @@ class TestClass(object): ) assert len(tasks) == 1 - # Check that the task is our end token from the messages resposne + # Check that the task is our end token from the messages response assert tasks[0].room_id == TEST_ROOM_ID assert tasks[0].token == "t47409-4357353_219380_26003_2265" From b5a419e488fe985b0d2ef9a8212e71c27ea6a7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 28 Sep 2022 16:46:36 +0200 Subject: [PATCH 117/146] chore: Bump our version --- CHANGELOG.md | 13 +++++++++++++ pantalaimon/main.py | 2 +- pantalaimon/panctl.py | 2 +- setup.py | 4 ++-- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 168f740..e654661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.10.5 2022-09-28 + +### Added + +- [[#137]] Proxy the v3 endpoints as well + +### Fixed + +- [[#130]] Make sure the token variable is declared + +[#137]: https://github.com/matrix-org/pantalaimon/pull/137 +[#130]: https://github.com/matrix-org/pantalaimon/pull/130 + ## 0.10.4 2022-02-04 ### Fixed diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 3002ee1..0ab42d6 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -287,7 +287,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.10.4", prog_name="pantalaimon") +@click.version_option(version="0.10.5", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 088bcaa..1f97fe7 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -695,7 +695,7 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.10.4", prog_name="panctl") +@click.version_option(version="0.10.5", prog_name="panctl") def main(): loop = asyncio.new_event_loop() glib_loop = GLib.MainLoop() diff --git a/setup.py b/setup.py index 5495f67..46798ba 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with open("README.md", encoding="utf-8") as f: setup( name="pantalaimon", - version="0.10.4", + version="0.10.5", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", @@ -29,7 +29,7 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.18, < 0.20" + "matrix-nio[e2e] >= 0.20, < 0.21" ], extras_require={ "ui": [ From 127373fdcc91c5a4403e25444526822e59083a0d Mon Sep 17 00:00:00 2001 From: Igor Artemenko Date: Tue, 6 Dec 2022 12:38:38 +0000 Subject: [PATCH 118/146] Fix typo --- pantalaimon/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index 5b4ce05..be32859 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -135,7 +135,7 @@ class InvalidLimit(Exception): class SqliteQStore(SqliteStore): def _create_database(self): return SqliteQueueDatabase( - self.database_path, pragmas=(("foregign_keys", 1), ("secure_delete", 1)) + self.database_path, pragmas=(("foreign_keys", 1), ("secure_delete", 1)) ) def close(self): From 313a5d528c0eb6c6c0246c6325199d0605b7c344 Mon Sep 17 00:00:00 2001 From: Igor Artemenko Date: Thu, 12 Jan 2023 16:26:24 +0000 Subject: [PATCH 119/146] Decrypt initial message after joining new DM Suppose that Alice logs in using Element. Before this change, when Bob would send a DM to Alice through Pantalaimon, Alice would not be able to decrypt Bob's initial message. Instead, she would see "Unable to decrypt: The sender's device has not sent us the keys for this message." and Pantalaimon's olmsessions table would have no associated records. Any future messages would be visible however. On the other hand, when Alice (using Element) is the one to send the first DM to Bob, he can decrypt the initial message. For Pantalaimon to execute /keys/claim, get_missing_sessions must return the invitee's device (and log "Missing session for device"). If Pantalaimon calls this method too soon, then self.device_store will not have the device. To populate self.device_store before Pantalaimon calls get_missing_sessions, it must execute /keys/query (and invoke _handle_key_query) earlier, during the /createRoom request. Pantalaimon does execute the /keys/query request during a sync after the server finishes creating the DM (and logs "Adding new device to the device store for user"), but only after checking unsuccessfully for the device in self.device_store. After this change, Pantalaimon executes /keys/claim, there is one record in olmsessions, and Alice can decrypt Bob's initial message. --- pantalaimon/daemon.py | 21 +++++++++++++++++++++ pantalaimon/main.py | 2 ++ 2 files changed, 23 insertions(+) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 3264cab..d241aff 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -793,6 +793,27 @@ class ProxyDaemon: body=await response.read(), ) + async def createRoom(self, request): + try: + content = await request.json() + except (JSONDecodeError, ContentTypeError): + return self._not_json + + invite = content.get("invite", ()) + if invite: + access_token = self.get_access_token(request) + + if not access_token: + return self._missing_token + + client = await self._find_client(access_token) + if not client: + return self._unknown_token + + client.users_for_key_query.update(invite) + + return await self.forward_to_web(request) + async def messages(self, request): access_token = self.get_access_token(request) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 0ab42d6..cd27170 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -71,6 +71,8 @@ async def init(data_dir, server_conf, send_queue, recv_queue): web.post("/_matrix/client/v3/login", proxy.login), web.get("/_matrix/client/r0/sync", proxy.sync), web.get("/_matrix/client/v3/sync", proxy.sync), + web.post("/_matrix/client/r0/createRoom", proxy.createRoom), + web.post("/_matrix/client/v3/createRoom", proxy.createRoom), web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages), web.get("/_matrix/client/v3/rooms/{room_id}/messages", proxy.messages), web.put( From 807deb94ee9e39bb570a904960d750e8d0dd5086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerber=20L=C3=B3r=C3=A1nt=20Viktor?= Date: Fri, 3 Feb 2023 05:45:50 +0100 Subject: [PATCH 120/146] fix media --- pantalaimon/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 3264cab..c5e4ac4 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -1273,7 +1273,7 @@ class ProxyDaemon: client = next(iter(self.pan_clients.values())) try: - response = await client.download(server_name, media_id, file_name) + response = await client.download(server_name=server_name, media_id=media_id, filename=file_name) except ClientConnectionError as e: raise e From 6638393042654b3f497ac1ea4b5f550b7024b490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 25 May 2023 19:57:25 +0200 Subject: [PATCH 121/146] Fix tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c3a2630..d90feda 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = coverage deps = -rtest-requirements.txt install_command = pip install {opts} {packages} -passenv = TOXENV CI +passenv = TOXENV,CI commands = pytest [testenv:coverage] From 3968c69aa846889970df1372ba9aa54c1c5e4290 Mon Sep 17 00:00:00 2001 From: Igor Artemenko Date: Wed, 5 Apr 2023 16:17:04 +0000 Subject: [PATCH 122/146] Set Pantalaimon presence to offline Before this change, Pantalaimon users would always appear online because that is the default state when the /sync endpoint's set_presence parameter is not set. By explicitly setting the parameter to "offline", only the user-facing client (which executes its own /sync request) affects the presence state. --- pantalaimon/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index be32859..ececee3 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -554,6 +554,7 @@ class PanClient(AsyncClient): full_state=True, since=next_batch, loop_sleep_time=loop_sleep_time, + set_presence="offline", ) ) self.task = task From 8b2a1173fd228e536024ecda49d36671ff16de70 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 1 Oct 2024 18:50:43 -0700 Subject: [PATCH 123/146] update README with new setup instructions --- README.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 47916d8..0573108 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ specifies one or more homeservers for pantalaimon to connect to. A minimal pantalaimon configuration looks like this: ```dosini [local-matrix] -Homeserver = https://localhost:8448 +Homeserver = https://localhost:443 ListenAddress = localhost ListenPort = 8009 ``` @@ -140,3 +140,40 @@ To control the daemon an interactive utility is provided in the form of `panctl` can be used to verify, blacklist or ignore devices, import or export session keys, or to introspect devices of users that we share encrypted rooms with. + +### Setup +This is all coming from an excellent comment that you can find [here](https://github.com/matrix-org/pantalaimon/issues/154#issuecomment-1951591191). + + + +1) Ensure you have an OS keyring installed. In my case I installed gnome-keyring. You may also want a GUI like seahorse to inspect the keyring. (pantalaimon will work without a keyring but your client will have to log in with the password every time pantalaimon is restarted, instead of being able to reuse the access token from the previous successful login.) + +2) In case you have prior attempts, clean the slate by deleting the ~/.local/share/pantalaimon directory. + +3) Start pantalaimon. + +4) Connect a client to the ListenAddress:ListenPort you specified in pantalaimon.conf, eg to 127.0.0.1:8009, using the same username and password you would've used to login to your homeserver directly. + +5) The login should succeed, but at this point all encrypted messages will fail to decrypt. This is fine. + +6) Start another client that you were already using for your encrypted chats previously. In my case this was app.element.io, so the rest of the steps here assume that. + +7) Run panctl. At the prompt, run start-verification . here is the full user ID like @arnavion:arnavion.dev. If you only have the one Element session, panctl will show you the device ID as an autocomplete hint so you don't have to look it up. If you do need to look it up, go to Element -> profile icon -> All Settings -> Sessions, expand the "Current session" item, and the "Session ID" is the device ID. + +8) In Element you will see a popup "Incoming Verification Request". Click "Continue". It will change to a popup containing some emojis, and panctl will print the same emojis. Click the "They match" button. It will now change to a popup like "Waiting for other client to confirm..." + +9) In panctl, run confirm-verification , ie the same command as before but with confirm-verification instead of start-verification. + +10) At this point, if you look at all your sessions in Element (profile icon -> All Settings -> Sessions), you should see "pantalaimon" in the "Other sessions" list as a "Verified" session. + +11) Export the E2E room keys that Element was using via profile icon -> Security & Privacy -> Export E2E room keys. Pick any password and then save the file to some path. + +12) Back in panctl, run import-keys . After a few seconds, in the output of pantalaimon, you should see a log like INFO: pantalaimon: Successfully imported keys for from + +13) Close and restart the client you had used in step 5, ie the one you want to connect to pantalaimon. Now, finally, you should be able to see the encrypted chats be decrypted. + +14) Delete the E2E room keys backup file from step 12. You don't need it any more. + +15) If in step 11 you had other unverified sessions from pantalaimon from your prior attempts, you can sign out of them too. + +You will probably have to repeat steps 12-15 any time you start a new encrypted chat in Element. From 634ac7ed68580ef8afafc02a82bf33e7c1c31259 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 1 Oct 2024 19:14:39 -0700 Subject: [PATCH 124/146] chore: remove depreciated dependency pydbus for dasbus --- pantalaimon/panctl.py | 6 +++--- pantalaimon/ui.py | 26 +++++++++++++------------- setup.py | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 1f97fe7..6519d8b 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -34,7 +34,7 @@ from prompt_toolkit import HTML, PromptSession, print_formatted_text from prompt_toolkit.completion import Completer, Completion, PathCompleter from prompt_toolkit.document import Document from prompt_toolkit.patch_stdout import patch_stdout -from pydbus import SessionBus +from dasbus.connection import SessionMessageBus PTK2 = ptk_version.startswith("2.") @@ -404,8 +404,8 @@ class PanCtl: commands = list(command_help.keys()) def __attrs_post_init__(self): - self.bus = SessionBus() - self.pan_bus = self.bus.get("org.pantalaimon1") + self.bus = SessionMessageBus() + self.pan_bus = self.bus.get_connection("org.pantalaimon1") self.ctl = self.pan_bus["org.pantalaimon1.control"] self.devices = self.pan_bus["org.pantalaimon1.devices"] diff --git a/pantalaimon/ui.py b/pantalaimon/ui.py index 813b67e..08e7e50 100644 --- a/pantalaimon/ui.py +++ b/pantalaimon/ui.py @@ -17,7 +17,7 @@ from importlib import util UI_ENABLED = ( util.find_spec("gi") is not None and util.find_spec("gi.repository") is not None - and util.find_spec("pydbus") is not None + and util.find_spec("dasbus") is not None ) if UI_ENABLED: @@ -28,8 +28,8 @@ if UI_ENABLED: import dbus import notify2 from gi.repository import GLib - from pydbus import SessionBus - from pydbus.generic import signal + from dasbus import SessionMessageBus + from dasbus.signal import Signal from dbus.mainloop.glib import DBusGMainLoop from nio import RoomKeyRequest, RoomKeyRequestCancellation @@ -123,8 +123,8 @@ if UI_ENABLED: """ - Response = signal() - UnverifiedDevices = signal() + Response = Signal() + UnverifiedDevices = Signal() def __init__(self, queue, server_list, id_counter): self.queue = queue @@ -297,13 +297,13 @@ if UI_ENABLED: """ - VerificationInvite = signal() - VerificationCancel = signal() - VerificationString = signal() - VerificationDone = signal() + VerificationInvite = Signal() + VerificationCancel = Signal() + VerificationString = Signal() + VerificationDone = Signal() - KeyRequest = signal() - KeyRequestCancel = signal() + KeyRequest = Signal() + KeyRequestCancel = Signal() def __init__(self, queue, id_counter): self.device_list = dict() @@ -466,8 +466,8 @@ if UI_ENABLED: self.control_if = Control(self.send_queue, self.server_list, id_counter) self.device_if = Devices(self.send_queue, id_counter) - self.bus = SessionBus() - self.bus.publish("org.pantalaimon1", self.control_if, self.device_if) + self.bus = SessionMessageBus() + self.bus.publish_object("org.pantalaimon1", self.control_if, self.device_if) def unverified_notification(self, message): notification = notify2.Notification( diff --git a/setup.py b/setup.py index 46798ba..75e0184 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ setup( "ui": [ "dbus-python >= 1.2, < 1.3", "PyGObject >= 3.36, < 3.39", - "pydbus >= 0.6, < 0.7", + "dasbus == 1.71", "notify2 >= 0.3, < 0.4", ] }, From 76dc74d250ee4d626da01c31725b46ae3ace5222 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 1 Oct 2024 19:20:29 -0700 Subject: [PATCH 125/146] chore: Format code with ruff --- pantalaimon/client.py | 12 ++++---- pantalaimon/config.py | 1 - pantalaimon/daemon.py | 14 +++++---- pantalaimon/index.py | 1 - pantalaimon/store.py | 1 - setup.py | 13 +++++---- tests/conftest.py | 47 +++++++++++++----------------- tests/pan_client_test.py | 8 +++-- tests/proxy_test.py | 34 +++++++++------------- tests/store_test.py | 63 ++++++++++++++++++++++------------------ 10 files changed, 96 insertions(+), 98 deletions(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index ececee3..882f6be 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -709,7 +709,6 @@ class PanClient(AsyncClient): for share in self.get_active_key_requests( message.user_id, message.device_id ): - continued = True if not self.continue_key_share(share): @@ -811,8 +810,9 @@ class PanClient(AsyncClient): if not isinstance(event, MegolmEvent): logger.warn( - "Encrypted event is not a megolm event:" - "\n{}".format(pformat(event_dict)) + "Encrypted event is not a megolm event:" "\n{}".format( + pformat(event_dict) + ) ) return False @@ -836,9 +836,9 @@ class PanClient(AsyncClient): decrypted_event.source["content"]["url"] = decrypted_event.url if decrypted_event.thumbnail_url: - decrypted_event.source["content"]["info"][ - "thumbnail_url" - ] = decrypted_event.thumbnail_url + decrypted_event.source["content"]["info"]["thumbnail_url"] = ( + decrypted_event.thumbnail_url + ) event_dict.update(decrypted_event.source) event_dict["decrypted"] = True diff --git a/pantalaimon/config.py b/pantalaimon/config.py index 8ee51d2..a5e59d1 100644 --- a/pantalaimon/config.py +++ b/pantalaimon/config.py @@ -186,7 +186,6 @@ class PanConfig: try: for section_name, section in config.items(): - if section_name == "Default": continue diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index c5e4ac4..2e48ac6 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -227,7 +227,8 @@ class ProxyDaemon: if ret: msg = ( - f"Device {device.id} of user " f"{device.user_id} successfully verified." + f"Device {device.id} of user " + f"{device.user_id} successfully verified." ) await client.send_update_device(device) else: @@ -309,7 +310,6 @@ class ProxyDaemon: DeviceUnblacklistMessage, ), ): - device = client.device_store[message.user_id].get(message.device_id, None) if not device: @@ -616,7 +616,9 @@ class ProxyDaemon: await pan_client.close() return - logger.info(f"Successfully started new background sync client for " f"{user_id}") + logger.info( + f"Successfully started new background sync client for " f"{user_id}" + ) await self.send_ui_message( UpdateUsersMessage(self.name, user_id, pan_client.device_id) @@ -733,7 +735,7 @@ class ProxyDaemon: return decryption_method(body, ignore_failures=False) except EncryptionError: logger.info("Error decrypting sync, waiting for next pan " "sync") - await client.synced.wait(), + (await client.synced.wait(),) logger.info("Pan synced, retrying decryption.") try: @@ -1273,7 +1275,9 @@ class ProxyDaemon: client = next(iter(self.pan_clients.values())) try: - response = await client.download(server_name=server_name, media_id=media_id, filename=file_name) + response = await client.download( + server_name=server_name, media_id=media_id, filename=file_name + ) except ClientConnectionError as e: raise e diff --git a/pantalaimon/index.py b/pantalaimon/index.py index 7350e58..e4f4aa0 100644 --- a/pantalaimon/index.py +++ b/pantalaimon/index.py @@ -230,7 +230,6 @@ if False: ) for message in query: - event = message.event event_dict = { diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 60a36a4..ee6f377 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -431,7 +431,6 @@ class PanStore: device_store = defaultdict(dict) for d in account.device_keys: - if d.deleted: continue diff --git a/setup.py b/setup.py index 46798ba..5b5fe45 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,7 @@ setup( url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", - description=("A Matrix proxy daemon that adds E2E encryption " - "capabilities."), + description=("A Matrix proxy daemon that adds E2E encryption " "capabilities."), long_description=long_description, long_description_content_type="text/markdown", license="Apache License, Version 2.0", @@ -29,7 +28,7 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.20, < 0.21" + "matrix-nio[e2e] >= 0.20, < 0.21", ], extras_require={ "ui": [ @@ -40,8 +39,10 @@ setup( ] }, entry_points={ - "console_scripts": ["pantalaimon=pantalaimon.main:main", - "panctl=pantalaimon.panctl:main"], + "console_scripts": [ + "pantalaimon=pantalaimon.main:main", + "panctl=pantalaimon.panctl:main", + ], }, - zip_safe=False + zip_safe=False, ) diff --git a/tests/conftest.py b/tests/conftest.py index 6103f77..ceb8902 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,11 +34,9 @@ class Provider(BaseProvider): def client(self): return ClientInfo(faker.mx_id(), faker.access_token()) - def avatar_url(self): return "mxc://{}/{}#auto".format( - faker.hostname(), - "".join(choices(ascii_letters) for i in range(24)) + faker.hostname(), "".join(choices(ascii_letters) for i in range(24)) ) def olm_key_pair(self): @@ -56,7 +54,6 @@ class Provider(BaseProvider): ) - faker.add_provider(Provider) @@ -80,13 +77,7 @@ def tempdir(): @pytest.fixture def panstore(tempdir): for _ in range(10): - store = SqliteStore( - faker.mx_id(), - faker.device_id(), - tempdir, - "", - "pan.db" - ) + store = SqliteStore(faker.mx_id(), faker.device_id(), tempdir, "", "pan.db") account = OlmAccount() store.save_account(account) @@ -130,21 +121,23 @@ async def pan_proxy_server(tempdir, aiohttp_server): recv_queue=ui_queue.async_q, proxy=None, ssl=False, - client_store_class=SqliteStore + client_store_class=SqliteStore, ) - app.add_routes([ - web.post("/_matrix/client/r0/login", proxy.login), - web.get("/_matrix/client/r0/sync", proxy.sync), - web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages), - web.put( - r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}", - proxy.send_message - ), - web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter), - web.post("/_matrix/client/r0/search", proxy.search), - web.options("/_matrix/client/r0/search", proxy.search_opts), - ]) + app.add_routes( + [ + web.post("/_matrix/client/r0/login", proxy.login), + web.get("/_matrix/client/r0/sync", proxy.sync), + web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages), + web.put( + r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}", + proxy.send_message, + ), + web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter), + web.post("/_matrix/client/r0/search", proxy.search), + web.options("/_matrix/client/r0/search", proxy.search_opts), + ] + ) server = await aiohttp_server(app) @@ -161,7 +154,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client): "access_token": "abc123", "device_id": "GHTYAJCE", "home_server": "example.org", - "user_id": "@example:example.org" + "user_id": "@example:example.org", } aioclient = await aiohttp_client(server) @@ -170,7 +163,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client): "https://example.org/_matrix/client/r0/login", status=200, payload=login_response, - repeat=True + repeat=True, ) await aioclient.post( @@ -179,7 +172,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client): "type": "m.login.password", "user": "example", "password": "wordpass", - } + }, ) yield server, aioclient, proxy, queues diff --git a/tests/pan_client_test.py b/tests/pan_client_test.py index 5932432..318d1b3 100644 --- a/tests/pan_client_test.py +++ b/tests/pan_client_test.py @@ -380,7 +380,9 @@ class TestClass(object): ) aioresponse.get( - sync_url, status=200, payload=self.initial_sync_response, + sync_url, + status=200, + payload=self.initial_sync_response, ) aioresponse.get(sync_url, status=200, payload=self.empty_sync, repeat=True) @@ -454,7 +456,9 @@ class TestClass(object): ) aioresponse.get( - sync_url, status=200, payload=self.initial_sync_response, + sync_url, + status=200, + payload=self.initial_sync_response, ) aioresponse.get(sync_url, status=200, payload=self.empty_sync, repeat=True) diff --git a/tests/proxy_test.py b/tests/proxy_test.py index b50379e..f0856d8 100644 --- a/tests/proxy_test.py +++ b/tests/proxy_test.py @@ -27,7 +27,7 @@ class TestClass(object): "access_token": "abc123", "device_id": "GHTYAJCE", "home_server": "example.org", - "user_id": "@example:example.org" + "user_id": "@example:example.org", } @property @@ -36,12 +36,7 @@ class TestClass(object): @property def keys_upload_response(self): - return { - "one_time_key_counts": { - "curve25519": 10, - "signed_curve25519": 20 - } - } + return {"one_time_key_counts": {"curve25519": 10, "signed_curve25519": 20}} @property def example_devices(self): @@ -52,10 +47,7 @@ class TestClass(object): devices[device.user_id][device.id] = device bob_device = OlmDevice( - BOB_ID, - BOB_DEVICE, - {"ed25519": BOB_ONETIME, - "curve25519": BOB_CURVE} + BOB_ID, BOB_DEVICE, {"ed25519": BOB_ONETIME, "curve25519": BOB_CURVE} ) devices[BOB_ID][BOB_DEVICE] = bob_device @@ -71,7 +63,7 @@ class TestClass(object): "https://example.org/_matrix/client/r0/login", status=200, payload=self.login_response, - repeat=True + repeat=True, ) assert not daemon.pan_clients @@ -82,7 +74,7 @@ class TestClass(object): "type": "m.login.password", "user": "example", "password": "wordpass", - } + }, ) assert resp.status == 200 @@ -105,11 +97,11 @@ class TestClass(object): "https://example.org/_matrix/client/r0/login", status=200, payload=self.login_response, - repeat=True + repeat=True, ) sync_url = re.compile( - r'^https://example\.org/_matrix/client/r0/sync\?access_token=.*' + r"^https://example\.org/_matrix/client/r0/sync\?access_token=.*" ) aioresponse.get( @@ -124,14 +116,16 @@ class TestClass(object): "type": "m.login.password", "user": "example", "password": "wordpass", - } + }, ) # Check that the pan client started to sync after logging in. pan_client = list(daemon.pan_clients.values())[0] assert len(pan_client.rooms) == 1 - async def test_pan_client_keys_upload(self, pan_proxy_server, aiohttp_client, aioresponse): + async def test_pan_client_keys_upload( + self, pan_proxy_server, aiohttp_client, aioresponse + ): server, daemon, _ = pan_proxy_server client = await aiohttp_client(server) @@ -140,11 +134,11 @@ class TestClass(object): "https://example.org/_matrix/client/r0/login", status=200, payload=self.login_response, - repeat=True + repeat=True, ) sync_url = re.compile( - r'^https://example\.org/_matrix/client/r0/sync\?access_token=.*' + r"^https://example\.org/_matrix/client/r0/sync\?access_token=.*" ) aioresponse.get( @@ -169,7 +163,7 @@ class TestClass(object): "type": "m.login.password", "user": "example", "password": "wordpass", - } + }, ) pan_client = list(daemon.pan_clients.values())[0] diff --git a/tests/store_test.py b/tests/store_test.py index 1d16e10..13acfb0 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -27,7 +27,7 @@ class TestClass(object): "type": "m.room.message", "unsigned": {"age": 43289803095}, "user_id": "@example2:localhost", - "age": 43289803095 + "age": 43289803095, } ) @@ -43,39 +43,41 @@ class TestClass(object): "type": "m.room.message", "unsigned": {"age": 43289803095}, "user_id": "@example2:localhost", - "age": 43289803095 + "age": 43289803095, } ) @property def encrypted_media_event(self): - return RoomEncryptedMedia.from_dict({ - "room_id": "!testroom:localhost", - "event_id": "$15163622445EBvZK:localhost", - "origin_server_ts": 1516362244030, - "sender": "@example2:localhost", - "type": "m.room.message", - "content": { - "body": "orange_cat.jpg", - "msgtype": "m.image", - "file": { - "v": "v2", - "key": { - "alg": "A256CTR", - "ext": True, - "k": "yx0QvkgYlasdWEsdalkejaHBzCkKEBAp3tB7dGtWgrs", - "key_ops": ["encrypt", "decrypt"], - "kty": "oct" + return RoomEncryptedMedia.from_dict( + { + "room_id": "!testroom:localhost", + "event_id": "$15163622445EBvZK:localhost", + "origin_server_ts": 1516362244030, + "sender": "@example2:localhost", + "type": "m.room.message", + "content": { + "body": "orange_cat.jpg", + "msgtype": "m.image", + "file": { + "v": "v2", + "key": { + "alg": "A256CTR", + "ext": True, + "k": "yx0QvkgYlasdWEsdalkejaHBzCkKEBAp3tB7dGtWgrs", + "key_ops": ["encrypt", "decrypt"], + "kty": "oct", + }, + "iv": "0pglXX7fspIBBBBAEERLFd", + "hashes": { + "sha256": "eXRDFvh+aXsQRj8a+5ZVVWUQ9Y6u9DYiz4tq1NvbLu8" + }, + "url": "mxc://localhost/maDtasSiPFjROFMnlwxIhhyW", + "mimetype": "image/jpeg", }, - "iv": "0pglXX7fspIBBBBAEERLFd", - "hashes": { - "sha256": "eXRDFvh+aXsQRj8a+5ZVVWUQ9Y6u9DYiz4tq1NvbLu8" - }, - "url": "mxc://localhost/maDtasSiPFjROFMnlwxIhhyW", - "mimetype": "image/jpeg" - } + }, } - }) + ) def test_account_loading(self, panstore): accounts = panstore.load_all_users() @@ -131,6 +133,7 @@ class TestClass(object): pytest.skip("Indexing needs to be enabled to test this") from pantalaimon.index import Index, IndexStore + loop = asyncio.get_event_loop() store = IndexStore("example", tempdir) @@ -148,8 +151,10 @@ class TestClass(object): assert len(result["results"]) == 1 assert result["count"] == 1 assert result["results"][0]["result"] == self.test_event.source - assert (result["results"][0]["context"]["events_after"][0] - == self.another_event.source) + assert ( + result["results"][0]["context"]["events_after"][0] + == self.another_event.source + ) def test_media_storage(self, panstore): server_name = "test" From 369f73f3fb53975fd1f448ede53e021fd2afe85a Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 1 Oct 2024 19:27:30 -0700 Subject: [PATCH 126/146] chore: removing unused imports --- pantalaimon/client.py | 1 - pantalaimon/index.py | 1 - pantalaimon/main.py | 1 - pantalaimon/store.py | 2 +- tests/proxy_test.py | 2 -- tests/store_test.py | 5 +---- 6 files changed, 2 insertions(+), 10 deletions(-) diff --git a/pantalaimon/client.py b/pantalaimon/client.py index 882f6be..f2f6895 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -16,7 +16,6 @@ import asyncio import os from collections import defaultdict from pprint import pformat -from typing import Any, Dict, Optional from urllib.parse import urlparse from aiohttp.client_exceptions import ClientConnectionError diff --git a/pantalaimon/index.py b/pantalaimon/index.py index e4f4aa0..3d49614 100644 --- a/pantalaimon/index.py +++ b/pantalaimon/index.py @@ -23,7 +23,6 @@ if False: import json import os from functools import partial - from typing import Any, Dict, List, Optional, Tuple import attr import tantivy diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 0ab42d6..abc411c 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -15,7 +15,6 @@ import asyncio import os import signal -from typing import Optional import click import janus diff --git a/pantalaimon/store.py b/pantalaimon/store.py index ee6f377..0dfe045 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -15,7 +15,7 @@ import json import os from collections import defaultdict -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict import attr from nio.crypto import TrustState, GroupSessionStore diff --git a/tests/proxy_test.py b/tests/proxy_test.py index f0856d8..ecf515e 100644 --- a/tests/proxy_test.py +++ b/tests/proxy_test.py @@ -1,9 +1,7 @@ -import asyncio import json import re from collections import defaultdict -from aiohttp import web from nio.crypto import OlmDevice from conftest import faker diff --git a/tests/store_test.py b/tests/store_test.py index 13acfb0..2ecef85 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -1,12 +1,10 @@ import asyncio -import pdb import pprint import pytest from nio import RoomMessage, RoomEncryptedMedia from urllib.parse import urlparse -from conftest import faker from pantalaimon.index import INDEXING_ENABLED from pantalaimon.store import FetchTask, MediaInfo, UploadInfo @@ -81,7 +79,6 @@ class TestClass(object): def test_account_loading(self, panstore): accounts = panstore.load_all_users() - # pdb.set_trace() assert len(accounts) == 10 def test_token_saving(self, panstore, access_token): @@ -132,7 +129,7 @@ class TestClass(object): if not INDEXING_ENABLED: pytest.skip("Indexing needs to be enabled to test this") - from pantalaimon.index import Index, IndexStore + from pantalaimon.index import IndexStore loop = asyncio.get_event_loop() From 5caaaf565176144adb35d12ca5e0c5a918e19444 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Wed, 2 Oct 2024 18:12:51 -0700 Subject: [PATCH 127/146] chore: Add code fences to the README --- README.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0573108..412db2a 100644 --- a/README.md +++ b/README.md @@ -146,34 +146,35 @@ This is all coming from an excellent comment that you can find [here](https://gi -1) Ensure you have an OS keyring installed. In my case I installed gnome-keyring. You may also want a GUI like seahorse to inspect the keyring. (pantalaimon will work without a keyring but your client will have to log in with the password every time pantalaimon is restarted, instead of being able to reuse the access token from the previous successful login.) +1) Ensure you have an OS keyring installed. In my case I installed `gnome-keyring`. You may also want a GUI like `seahorse` to inspect the keyring. (pantalaimon will work without a keyring but your client will have to log in with the password every time `pantalaimon` is restarted, instead of being able to reuse the access token from the previous successful login.) -2) In case you have prior attempts, clean the slate by deleting the ~/.local/share/pantalaimon directory. +2) In case you have prior attempts, clean the slate by deleting the `~/.local/share/pantalaimon` directory. -3) Start pantalaimon. +3) Start `pantalaimon`. -4) Connect a client to the ListenAddress:ListenPort you specified in pantalaimon.conf, eg to 127.0.0.1:8009, using the same username and password you would've used to login to your homeserver directly. +4) Connect a client to the `ListenAddress:ListenPort` you specified in `pantalaimon.conf`, eg to `127.0.0.1:8009`, using the same username and password you would've used to login to your homeserver directly. 5) The login should succeed, but at this point all encrypted messages will fail to decrypt. This is fine. -6) Start another client that you were already using for your encrypted chats previously. In my case this was app.element.io, so the rest of the steps here assume that. +6) Start another client that you were already using for your encrypted chats previously. In my case this was `app.element.io`, so the rest of the steps here assume that. -7) Run panctl. At the prompt, run start-verification . here is the full user ID like @arnavion:arnavion.dev. If you only have the one Element session, panctl will show you the device ID as an autocomplete hint so you don't have to look it up. If you do need to look it up, go to Element -> profile icon -> All Settings -> Sessions, expand the "Current session" item, and the "Session ID" is the device ID. +7) Run `panctl`. At the prompt, run `start-verification `. `` here is the full user ID like `@arnavion:arnavion.dev`. If you only have the one Element session, `panctl` will show you the device ID as an autocomplete hint so you don't have to look it up. If you do need to look it up, go to Element -> profile icon -> All Settings -> Sessions, expand the "Current session" item, and the "Session ID" is the device ID. -8) In Element you will see a popup "Incoming Verification Request". Click "Continue". It will change to a popup containing some emojis, and panctl will print the same emojis. Click the "They match" button. It will now change to a popup like "Waiting for other client to confirm..." +8) In Element you will see a popup "Incoming Verification Request". Click "Continue". It will change to a popup containing some emojis, and `panctl` will print the same emojis. Click the "They match" button. It will now change to a popup like "Waiting for other client to confirm..." -9) In panctl, run confirm-verification , ie the same command as before but with confirm-verification instead of start-verification. +9) In `panctl`, run `confirm-verification `, ie the same command as before but with `confirm-verification` instead of `start-verification`. 10) At this point, if you look at all your sessions in Element (profile icon -> All Settings -> Sessions), you should see "pantalaimon" in the "Other sessions" list as a "Verified" session. 11) Export the E2E room keys that Element was using via profile icon -> Security & Privacy -> Export E2E room keys. Pick any password and then save the file to some path. -12) Back in panctl, run import-keys . After a few seconds, in the output of pantalaimon, you should see a log like INFO: pantalaimon: Successfully imported keys for from +12) Back in `panctl`, run `import-keys `. After a few seconds, in the output of `pantalaimon`, you should see a log like `INFO: pantalaimon: Successfully imported keys for from `. -13) Close and restart the client you had used in step 5, ie the one you want to connect to pantalaimon. Now, finally, you should be able to see the encrypted chats be decrypted. +13) Close and restart the client you had used in step 5, ie the one you want to connect to `pantalaimon`. Now, finally, you should be able to see the encrypted chats be decrypted. 14) Delete the E2E room keys backup file from step 12. You don't need it any more. + 15) If in step 11 you had other unverified sessions from pantalaimon from your prior attempts, you can sign out of them too. -You will probably have to repeat steps 12-15 any time you start a new encrypted chat in Element. +You will probably have to repeat steps 11-14 any time you start a new encrypted chat in Element. From 5426d5bf9d5109f113bdf10a6133eefdcacad2ff Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Mon, 28 Oct 2024 20:07:12 -0700 Subject: [PATCH 128/146] chore: upgrade matrix-nio and fix dasbus version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 379f76f..5b9abec 100644 --- a/setup.py +++ b/setup.py @@ -28,13 +28,13 @@ setup( "cachetools >= 3.0.0", "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] >= 0.20, < 0.21", + "matrix-nio[e2e] >= 0.24, < 0.25.2", ], extras_require={ "ui": [ "dbus-python >= 1.2, < 1.3", "PyGObject >= 3.36, < 3.39", - "dasbus == 1.71", + "dasbus == 1.7", "notify2 >= 0.3, < 0.4", ] }, From e2abab1ecc8a5cfec171f164456d10a514d4216e Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Mon, 28 Oct 2024 20:11:44 -0700 Subject: [PATCH 129/146] chore: update pip install instructions in README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 412db2a..0e45f28 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,11 @@ Installing pantalaimon works like usually with python packages: python setup.py install +or you can use `pip` and install it with: +``` +pip install .[ui] +``` + Pantalaimon can also be found on pypi: pip install pantalaimon From bfb3b0615342c7625ceae7f95bc91fe9d2e3d54c Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Mon, 28 Oct 2024 20:13:16 -0700 Subject: [PATCH 130/146] docs: Update how to install packages with package manager --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 0e45f28..7f2bd2d 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,9 @@ or you can use `pip` and install it with: pip install .[ui] ``` +It is recommended that you create a virtual environment first or install dependencies +via your package manager. They are usually found with `python-`. + Pantalaimon can also be found on pypi: pip install pantalaimon From 93a1fcb36fcf0d11a69e982f4a977d94159d7ad0 Mon Sep 17 00:00:00 2001 From: Icy-Thought Date: Fri, 1 Nov 2024 18:36:20 +0100 Subject: [PATCH 131/146] chore: update PyGObject dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5b9abec..fde2d4e 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( extras_require={ "ui": [ "dbus-python >= 1.2, < 1.3", - "PyGObject >= 3.36, < 3.39", + "PyGObject >= 3.46, < 3.50", "dasbus == 1.7", "notify2 >= 0.3, < 0.4", ] From e9fb8a4a57cd2179f6ef9645e910b864a55413f1 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Wed, 20 Nov 2024 20:20:36 -0800 Subject: [PATCH 132/146] chore: change from appdirs to platformdirs --- pantalaimon/main.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pantalaimon/main.py b/pantalaimon/main.py index 3468365..741937e 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -22,7 +22,7 @@ import keyring import logbook import nio from aiohttp import web -from appdirs import user_config_dir, user_data_dir +from platformdirs import user_config_dir, user_data_dir from logbook import StderrHandler from pantalaimon.config import PanConfig, PanConfigError, parse_log_level diff --git a/setup.py b/setup.py index fde2d4e..bba80e6 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( install_requires=[ "attrs >= 19.3.0", "aiohttp >= 3.6, < 4.0", - "appdirs >= 1.4.4", + "platformdirs >= 4.3.6", "click >= 7.1.2", "keyring >= 21.2.1", "logbook >= 1.5.3", From 29d18653dcd4c7cd5ebeea2521b7f17607d67c38 Mon Sep 17 00:00:00 2001 From: Arka Dash Date: Thu, 5 Dec 2024 00:07:03 +0530 Subject: [PATCH 133/146] Using shlex.split instead str.split --- pantalaimon/panctl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 6519d8b..8e087d0 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -20,6 +20,7 @@ import sys from collections import defaultdict from itertools import zip_longest from typing import List +from shlex import split import attr import click @@ -589,7 +590,7 @@ class PanCtl: parser = PanctlParser(self.commands) try: - args = parser.parse_args(result.split()) + args = parser.parse_args(split(result)) except ParseError: continue From 42cdcc251924a7c99528fa56b1c77824865c0ffc Mon Sep 17 00:00:00 2001 From: Arka Dash Date: Thu, 5 Dec 2024 11:07:05 +0530 Subject: [PATCH 134/146] set posix to False --- pantalaimon/panctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 8e087d0..a6e6c89 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -590,7 +590,7 @@ class PanCtl: parser = PanctlParser(self.commands) try: - args = parser.parse_args(split(result)) + args = parser.parse_args(split(result, posix=False)) except ParseError: continue From 26d9a55ce88dfaeb4507c1a7f85cada6debba9f1 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Sun, 29 Dec 2024 20:09:14 -0800 Subject: [PATCH 135/146] fix: revert ui.py to pydbus --- pantalaimon/ui.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pantalaimon/ui.py b/pantalaimon/ui.py index 08e7e50..813b67e 100644 --- a/pantalaimon/ui.py +++ b/pantalaimon/ui.py @@ -17,7 +17,7 @@ from importlib import util UI_ENABLED = ( util.find_spec("gi") is not None and util.find_spec("gi.repository") is not None - and util.find_spec("dasbus") is not None + and util.find_spec("pydbus") is not None ) if UI_ENABLED: @@ -28,8 +28,8 @@ if UI_ENABLED: import dbus import notify2 from gi.repository import GLib - from dasbus import SessionMessageBus - from dasbus.signal import Signal + from pydbus import SessionBus + from pydbus.generic import signal from dbus.mainloop.glib import DBusGMainLoop from nio import RoomKeyRequest, RoomKeyRequestCancellation @@ -123,8 +123,8 @@ if UI_ENABLED: """ - Response = Signal() - UnverifiedDevices = Signal() + Response = signal() + UnverifiedDevices = signal() def __init__(self, queue, server_list, id_counter): self.queue = queue @@ -297,13 +297,13 @@ if UI_ENABLED: """ - VerificationInvite = Signal() - VerificationCancel = Signal() - VerificationString = Signal() - VerificationDone = Signal() + VerificationInvite = signal() + VerificationCancel = signal() + VerificationString = signal() + VerificationDone = signal() - KeyRequest = Signal() - KeyRequestCancel = Signal() + KeyRequest = signal() + KeyRequestCancel = signal() def __init__(self, queue, id_counter): self.device_list = dict() @@ -466,8 +466,8 @@ if UI_ENABLED: self.control_if = Control(self.send_queue, self.server_list, id_counter) self.device_if = Devices(self.send_queue, id_counter) - self.bus = SessionMessageBus() - self.bus.publish_object("org.pantalaimon1", self.control_if, self.device_if) + self.bus = SessionBus() + self.bus.publish("org.pantalaimon1", self.control_if, self.device_if) def unverified_notification(self, message): notification = notify2.Notification( From c3ee162802cc08d219860fef8912ea4cef436d9f Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Sun, 29 Dec 2024 20:11:25 -0800 Subject: [PATCH 136/146] fix: revert steup.py to use pydbus --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bba80e6..aeb1a6b 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( "ui": [ "dbus-python >= 1.2, < 1.3", "PyGObject >= 3.46, < 3.50", - "dasbus == 1.7", + "pydbus >= 0.6, < 0.7", "notify2 >= 0.3, < 0.4", ] }, From 0f52d303d48301171a9c39dbd6f86765e2b45aff Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Sun, 29 Dec 2024 20:12:44 -0800 Subject: [PATCH 137/146] fix: revert panctl.py to pydbus --- pantalaimon/panctl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index a6e6c89..f37ab88 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -35,7 +35,7 @@ from prompt_toolkit import HTML, PromptSession, print_formatted_text from prompt_toolkit.completion import Completer, Completion, PathCompleter from prompt_toolkit.document import Document from prompt_toolkit.patch_stdout import patch_stdout -from dasbus.connection import SessionMessageBus +from pydbus import SessionBus PTK2 = ptk_version.startswith("2.") @@ -405,8 +405,8 @@ class PanCtl: commands = list(command_help.keys()) def __attrs_post_init__(self): - self.bus = SessionMessageBus() - self.pan_bus = self.bus.get_connection("org.pantalaimon1") + self.bus = SessionBus() + self.pan_bus = self.bus.get("org.pantalaimon1") self.ctl = self.pan_bus["org.pantalaimon1.control"] self.devices = self.pan_bus["org.pantalaimon1.devices"] From 659bf093f8fe4accb8ec893c3c983d094d9ce90d Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Sun, 6 Apr 2025 17:51:29 -0700 Subject: [PATCH 138/146] =?UTF-8?q?chore:=20Fix=20failing=20tests=20by=20r?= =?UTF-8?q?emoving=20unused=20fixture=20=F0=9F=A5=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/pan_client_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pan_client_test.py b/tests/pan_client_test.py index 318d1b3..67f130f 100644 --- a/tests/pan_client_test.py +++ b/tests/pan_client_test.py @@ -25,7 +25,7 @@ ALICE_ID = "@alice:example.org" @pytest.fixture -async def client(tmpdir, loop): +async def client(tmpdir): store = PanStore(tmpdir) queue = janus.Queue() conf = ServerConfig("example", "https://example.org") @@ -371,7 +371,7 @@ class TestClass(object): await client.loop_stop() - async def test_history_fetching_tasks(self, client, aioresponse, loop): + async def test_history_fetching_tasks(self, client, aioresponse): if not INDEXING_ENABLED: pytest.skip("Indexing needs to be enabled to test this") @@ -447,7 +447,7 @@ class TestClass(object): await client.loop_stop() - async def test_history_fetching_resume(self, client, aioresponse, loop): + async def test_history_fetching_resume(self, client, aioresponse): if not INDEXING_ENABLED: pytest.skip("Indexing needs to be enabled to test this") From 5c68db3971da41a882016fd228f2782c487b6878 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 3 Jun 2025 10:39:25 -0700 Subject: [PATCH 139/146] =?UTF-8?q?build:=20Add=20manpages=20to=20distribu?= =?UTF-8?q?tion=20=F0=9F=92=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aeb1a6b..9f0584e 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,34 @@ # -*- coding: utf-8 -*- from setuptools import find_packages, setup +import os with open("README.md", encoding="utf-8") as f: long_description = f.read() + +def get_manpages(): + """ + This function goes and gets all the man pages so they can be installed when + the package is installed. + """ + man_pages = [] + for root, _, files in os.walk("docs/man"): + for file in files: + if file.endswith((".1", ".5", ".8")): + man_section = file.split(".")[-1] + dest_dir = os.path.join("share", "man", f"man{man_section}") + man_pages.append((dest_dir, [os.path.join(root, file)])) + return man_pages + + setup( name="pantalaimon", version="0.10.5", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", - description=("A Matrix proxy daemon that adds E2E encryption " "capabilities."), + description=("A Matrix proxy daemon that adds E2E encryption capabilities."), long_description=long_description, long_description_content_type="text/markdown", license="Apache License, Version 2.0", @@ -45,4 +62,5 @@ setup( ], }, zip_safe=False, + data_files=get_manpages(), ) From da7f426c60f907905e18221d39a9ce1948344b61 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 3 Jun 2025 17:26:32 -0700 Subject: [PATCH 140/146] =?UTF-8?q?build:=20Fixing=20tests=20with=20async?= =?UTF-8?q?=20=F0=9F=A5=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pytest.ini | 2 ++ test-requirements.txt | 1 + tox.ini | 1 + 3 files changed, 4 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..d280de0 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode = auto \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt index 36816bf..e16edb3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,4 +5,5 @@ pytest-cov faker aiohttp pytest-aiohttp +pytest-asyncio aioresponses diff --git a/tox.ini b/tox.ini index d90feda..c19ce56 100644 --- a/tox.ini +++ b/tox.ini @@ -18,5 +18,6 @@ deps = -rtest-requirements.txt coverage codecov>=1.4.0 + pytest-asyncio setenv = COVERAGE_FILE=.coverage From a7be6cbfcb95dd9d761d230526f1be1c8013d25e Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 3 Jun 2025 17:41:48 -0700 Subject: [PATCH 141/146] =?UTF-8?q?chore:=20pin=20package=20versions=20for?= =?UTF-8?q?=20testing=20=F0=9F=8F=B5=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test-requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index e16edb3..570aea6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,9 +1,9 @@ -pytest -pytest-flake8 -pytest-isort -pytest-cov -faker -aiohttp -pytest-aiohttp -pytest-asyncio -aioresponses +pytest==8.3.5 +pytest-flake8==1.3.0 +pytest-isort==4.0.0 +pytest-cov==6.1.1 +faker==37.1.0 +aiohttp==3.11.16 +pytest-aiohttp==1.1.0 +pytest-asyncio==0.26.0 +aioresponses==0.7.8 From 50cb9c41a8b9945c9c34a33295ef0c6bdecf88a5 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 3 Jun 2025 17:46:08 -0700 Subject: [PATCH 142/146] chore: Removed unused file --- pytest.ini | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index d280de0..0000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -asyncio_mode = auto \ No newline at end of file From b42b24019a8a407a15c3c1574c00e21095310a30 Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 3 Jun 2025 17:48:50 -0700 Subject: [PATCH 143/146] =?UTF-8?q?chore:=20Pin=20different=20version=20of?= =?UTF-8?q?=20pytest-flake8=20=F0=9F=98=B5=E2=80=8D=F0=9F=92=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 570aea6..fe924a0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ pytest==8.3.5 -pytest-flake8==1.3.0 +pytest-flake8==1.2.2 pytest-isort==4.0.0 pytest-cov==6.1.1 faker==37.1.0 From dcf02179a9cde1506bee43a772ecf4ccd8299f1c Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 3 Jun 2025 17:53:36 -0700 Subject: [PATCH 144/146] UGH, why is the CI not seeing these package versions???????? --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fe924a0..662440f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ pytest==8.3.5 pytest-flake8==1.2.2 pytest-isort==4.0.0 -pytest-cov==6.1.1 +pytest-cov==5.0.0 faker==37.1.0 aiohttp==3.11.16 pytest-aiohttp==1.1.0 From 5f041b25d2810828a5908f4c7fbacf6553ec6c1b Mon Sep 17 00:00:00 2001 From: Hank Greenburg Date: Tue, 3 Jun 2025 17:56:59 -0700 Subject: [PATCH 145/146] Work, I beg of you. --- test-requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 662440f..bdbc671 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,8 +2,8 @@ pytest==8.3.5 pytest-flake8==1.2.2 pytest-isort==4.0.0 pytest-cov==5.0.0 -faker==37.1.0 -aiohttp==3.11.16 -pytest-aiohttp==1.1.0 -pytest-asyncio==0.26.0 -aioresponses==0.7.8 +faker<=37.1.0 +aiohttp<=3.11.16 +pytest-aiohttp<=1.1.0 +pytest-asyncio<=0.26.0 +aioresponses<=0.7.8 From 34432fe059028b081ddf7fbeb008d2e9d7974ec5 Mon Sep 17 00:00:00 2001 From: Aurelia Date: Wed, 4 Jun 2025 00:42:44 +0200 Subject: [PATCH 146/146] ci: add docker build and release --- .github/workflows/docker.yml | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..56531f5 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,53 @@ +name: Create and publish a docker image + +on: + workflow_dispatch: + push: + branches: + - master + tags: + - '\d+.\d+.\d+' + pull_request: + branches: + - master + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag,prefix=v + type=sha + + - name: Build and push docker image + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }}