diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4872347 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: Build Status + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10'] + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Tox and any other packages + run: | + wget https://gitlab.matrix.org/matrix-org/olm/-/archive/master/olm-master.tar.bz2 + tar -xvf olm-master.tar.bz2 + pushd olm-master && make && sudo make PREFIX="/usr" install && popd + rm -r olm-master + pip install tox + - name: Run Tox + run: tox -e py + + coverage: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install Tox and any other packages + run: | + wget https://gitlab.matrix.org/matrix-org/olm/-/archive/master/olm-master.tar.bz2 + tar -xvf olm-master.tar.bz2 + pushd olm-master && make && sudo make PREFIX="/usr" install && popd + rm -r olm-master + pip install tox + - name: Run Tox + run: tox -e coverage diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..56531f5 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,53 @@ +name: Create and publish a docker image + +on: + workflow_dispatch: + push: + branches: + - master + tags: + - '\d+.\d+.\d+' + pull_request: + branches: + - master + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag,prefix=v + type=sha + + - name: Build and push docker image + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.travis.yml b/.travis.yml index ccd725b..84318a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,14 +16,14 @@ before_install: matrix: include: - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - - python: 3.7 + - python: 3.8 + env: TOXENV=py38 + - python: 3.9 + env: TOXENV=py39 + - python: 3.9 env: TOXENV=coverage -install: pip install tox-travis PyGObject dbus-python aioresponses +install: pip install tox-travis aioresponses script: tox after_success: diff --git a/CHANGELOG.md b/CHANGELOG.md index ca3fa00..e654661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,159 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.10.5 2022-09-28 + +### Added + +- [[#137]] Proxy the v3 endpoints as well + +### Fixed + +- [[#130]] Make sure the token variable is declared + +[#137]: https://github.com/matrix-org/pantalaimon/pull/137 +[#130]: https://github.com/matrix-org/pantalaimon/pull/130 + +## 0.10.4 2022-02-04 + +### Fixed + +- [[#122]] Fix the GLib import for panctl on some distributions +- [[#120]] Don't use strip to filter Bearer from the auth header +- [[#118]] Don't use the raw path if we need to sanitize filters, fixing room + history fetching for Fractal + +[#122]: https://github.com/matrix-org/pantalaimon/pull/122 +[#120]: https://github.com/matrix-org/pantalaimon/pull/120 +[#118]: https://github.com/matrix-org/pantalaimon/pull/118 + +## 0.10.3 2021-09-02 + +### Fixed + +- [[#105]] Use the raw_path when forwarding requests, avoiding URL + decoding/encoding issues. + +[#105]: https://github.com/matrix-org/pantalaimon/pull/105 + + +## 0.10.2 2021-07-14 + +### Fixed + +- [[#103]] Prevent E2EE downgrade on failed syncs + +[#103]: https://github.com/matrix-org/pantalaimon/pull/103 + + +## 0.10.1 2021-07-06 + +### Fixed + +- [[#100]] Don't require the rooms dicts in the sync response +- [[#99]] Thumbnails not generating for media uploaded in unencrypted rooms + whole LRU cache when it shouldn't + +[#100]: https://github.com/matrix-org/pantalaimon/pull/100 +[#99]: https://github.com/matrix-org/pantalaimon/pull/99 + + +## 0.10.0 2021-05-14 + +### Added + +- [[#98]] Add the ability to remove old room keys +- [[#95]] Encrypt thumbnails uploaded by a client + +### Fixed + +- [[#96]] Split out the media cache loading logic to avoid returning the + whole LRU cache when it shouldn't + +[#98]: https://github.com/matrix-org/pantalaimon/pull/98 +[#96]: https://github.com/matrix-org/pantalaimon/pull/96 +[#95]: https://github.com/matrix-org/pantalaimon/pull/95 + +## 0.9.3 2021-05-14 + +### Changed + +- [[#73f68c7]] Bump the allowed nio version + +[73f68c7]: https://github.com/matrix-org/pantalaimon/commit/73f68c76fb05037bd7fe71688ce39eb1f526a385 + +## 0.9.2 2021-03-10 + +### Changed + +- [[#89]] Bump the allowed nio version + +[#89]: https://github.com/matrix-org/pantalaimon/pull/89 + +## 0.9.1 2021-01-19 + +### Changed + +- [[3baae08]] Bump the allowed nio version + +[3baae08]: https://github.com/matrix-org/pantalaimon/commit/3baae08ac36e258632e224b655e177a765a939f3 + +## 0.9.0 2021-01-19 + +### Fixed + +- [[59051c5]] Fix the notification initialization allowing the DBUS thread to + start again + +### Added + +- [[#79]] Support media uploads, thanks to @aspacca + +[59051c5]: https://github.com/matrix-org/pantalaimon/commit/59051c530a343a6887ea0f9ccddd6f6964f6d923 +[#79]: https://github.com/matrix-org/pantalaimon/pull/79 + +## 0.8.0 2020-09-30 + +### Changed + +- [[#69]] If no password is provided to /login, the daemon will re-use the original login response. + +[#69]: https://github.com/matrix-org/pantalaimon/pull/69 + +## 0.7.0 2020-09-02 + +### Fixed + +- [[#60]] Sanitize the GET /rooms/{room_id}/messages filters as well. +- [[#62]] Store media info when decrypting instead of using a event callback. + +### Changed + +- [[d425e2d]] Increase the max POST size. + +[#62]: https://github.com/matrix-org/pantalaimon/pull/62 +[#60]: https://github.com/matrix-org/pantalaimon/pull/60 +[d425e2d]: https://github.com/matrix-org/pantalaimon/commit/d425e2d188aed32c3fe87cac210c0943fd51b085 + +## 0.6.5 2020-07-02 + +### Fixed + +- [[a1ce950]] Allow to send messages using a POST request since Synapse seems to + allow it. + +[a1ce950]: https://github.com/matrix-org/pantalaimon/commit/a1ce95076ecd80c880028691feeced8d28cacad9 + +## 0.6.4 2020-06-21 + +### Changed +- Bump the maximal supported nio version. + +## 0.6.3 2020-05-28 + +### Fixed +- Fix our dep requirements to avoid incompatibilities between nio and pantalaimon. + ## 0.6.2 2020-05-27 ### Fixed diff --git a/README.md b/README.md index af1563b..7f2bd2d 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,19 @@ Installing pantalaimon works like usually with python packages: python setup.py install +or you can use `pip` and install it with: +``` +pip install .[ui] +``` + +It is recommended that you create a virtual environment first or install dependencies +via your package manager. They are usually found with `python-`. + Pantalaimon can also be found on pypi: pip install pantalaimon -Pantalaimon contains a dbus based UI that can be used to controll the daemon. +Pantalaimon contains a dbus based UI that can be used to control the daemon. The dbus based UI is completely optional and needs to be installed with the daemon: @@ -77,6 +85,10 @@ docker build -t pantalaimon . # volume below is for where Pantalaimon should dump some data. docker run -it --rm -v /path/to/pantalaimon/dir:/data -p 8008:8008 pantalaimon ``` +The Docker image in the above example can alternatively be built straight from any branch or tag without the need to clone the repo, just by using this syntax: +```bash +docker build -t pantalaimon github.com/matrix-org/pantalaimon#master +``` An example `pantalaimon.conf` for Docker is: ```conf @@ -96,7 +108,7 @@ IgnoreVerification = True Usage ===== -While pantalaimon is a daemon, it is meant to be run as your own user. It won't +While pantalaimon is a daemon, it is meant to be run as the same user as the app it is proxying for. It won't verify devices for you automatically, unless configured to do so, and requires user interaction to verify, ignore or blacklist devices. A more complete description of Pantalaimon can be found in the [man page](docs/man/pantalaimon.8.md). @@ -107,7 +119,7 @@ specifies one or more homeservers for pantalaimon to connect to. A minimal pantalaimon configuration looks like this: ```dosini [local-matrix] -Homeserver = https://localhost:8448 +Homeserver = https://localhost:443 ListenAddress = localhost ListenPort = 8009 ``` @@ -136,3 +148,41 @@ To control the daemon an interactive utility is provided in the form of `panctl` can be used to verify, blacklist or ignore devices, import or export session keys, or to introspect devices of users that we share encrypted rooms with. + +### Setup +This is all coming from an excellent comment that you can find [here](https://github.com/matrix-org/pantalaimon/issues/154#issuecomment-1951591191). + + + +1) Ensure you have an OS keyring installed. In my case I installed `gnome-keyring`. You may also want a GUI like `seahorse` to inspect the keyring. (pantalaimon will work without a keyring but your client will have to log in with the password every time `pantalaimon` is restarted, instead of being able to reuse the access token from the previous successful login.) + +2) In case you have prior attempts, clean the slate by deleting the `~/.local/share/pantalaimon` directory. + +3) Start `pantalaimon`. + +4) Connect a client to the `ListenAddress:ListenPort` you specified in `pantalaimon.conf`, eg to `127.0.0.1:8009`, using the same username and password you would've used to login to your homeserver directly. + +5) The login should succeed, but at this point all encrypted messages will fail to decrypt. This is fine. + +6) Start another client that you were already using for your encrypted chats previously. In my case this was `app.element.io`, so the rest of the steps here assume that. + +7) Run `panctl`. At the prompt, run `start-verification `. `` here is the full user ID like `@arnavion:arnavion.dev`. If you only have the one Element session, `panctl` will show you the device ID as an autocomplete hint so you don't have to look it up. If you do need to look it up, go to Element -> profile icon -> All Settings -> Sessions, expand the "Current session" item, and the "Session ID" is the device ID. + +8) In Element you will see a popup "Incoming Verification Request". Click "Continue". It will change to a popup containing some emojis, and `panctl` will print the same emojis. Click the "They match" button. It will now change to a popup like "Waiting for other client to confirm..." + +9) In `panctl`, run `confirm-verification `, ie the same command as before but with `confirm-verification` instead of `start-verification`. + +10) At this point, if you look at all your sessions in Element (profile icon -> All Settings -> Sessions), you should see "pantalaimon" in the "Other sessions" list as a "Verified" session. + +11) Export the E2E room keys that Element was using via profile icon -> Security & Privacy -> Export E2E room keys. Pick any password and then save the file to some path. + +12) Back in `panctl`, run `import-keys `. After a few seconds, in the output of `pantalaimon`, you should see a log like `INFO: pantalaimon: Successfully imported keys for from `. + +13) Close and restart the client you had used in step 5, ie the one you want to connect to `pantalaimon`. Now, finally, you should be able to see the encrypted chats be decrypted. + +14) Delete the E2E room keys backup file from step 12. You don't need it any more. + + +15) If in step 11 you had other unverified sessions from pantalaimon from your prior attempts, you can sign out of them too. + +You will probably have to repeat steps 11-14 any time you start a new encrypted chat in Element. diff --git a/contrib/pantalaimon.conf b/contrib/pantalaimon.conf index 69a7465..dc4cedc 100644 --- a/contrib/pantalaimon.conf +++ b/contrib/pantalaimon.conf @@ -12,3 +12,4 @@ Proxy = http://localhost:8080 SSL = False IgnoreVerification = False UseKeyring = True +DropOldKeys = False diff --git a/docs/man/panctl.1 b/docs/man/panctl.1 index e1ebb8e..705dd97 100644 --- a/docs/man/panctl.1 +++ b/docs/man/panctl.1 @@ -51,7 +51,7 @@ The message will be sent away after all devices are marked as ignored. In contrast to the .Cm send-anyways command this command cancels the sending of a message to an encrypted room with -unverified devices and gives the user the oportunity to verify or blacklist +unverified devices and gives the user the opportunity to verify or blacklist devices as they see fit. .It Cm import-keys Ar pan-user Ar file Ar passphrase Import end-to-end encryption keys from the given file for the given pan-user. diff --git a/docs/man/panctl.md b/docs/man/panctl.md index ed733f5..30fa8f4 100644 --- a/docs/man/panctl.md +++ b/docs/man/panctl.md @@ -74,7 +74,7 @@ are as follows: > In contrast to the > **send-anyways** > command this command cancels the sending of a message to an encrypted room with -> unverified devices and gives the user the oportunity to verify or blacklist +> unverified devices and gives the user the opportunity to verify or blacklist > devices as they see fit. **import-keys** *pan-user* *file* *passphrase* diff --git a/docs/man/pantalaimon.5 b/docs/man/pantalaimon.5 index 79651ba..01fd2d1 100644 --- a/docs/man/pantalaimon.5 +++ b/docs/man/pantalaimon.5 @@ -51,6 +51,11 @@ This option configures if a proxy instance should use the OS keyring to store its own access tokens. The access tokens are required for the daemon to resume operation. If this is set to "No", access tokens are stored in the pantalaimon database in plaintext. Defaults to "Yes". +.It Cm DropOldKeys +This option configures if a proxy instance should only keep the latest version +of a room key from a certain user around. This effectively means that only newly +incoming messages will be decryptable, the proxy will be unable to decrypt the +room history. Defaults to "No". .It Cm SearchRequests This option configures if the proxy should make additional HTTP requests to the server when clients use the search API endpoint. Some data that is required to @@ -81,7 +86,7 @@ The amount of time to wait between room message history requests to the Homeserver in ms. Defaults to 3000. .El .Pp -Aditional to the homeserver section a special section with the name +Additional to the homeserver section a special section with the name .Cm Default can be used to configure the following values for all homeservers: .Cm ListenAddress , @@ -128,7 +133,7 @@ Default location of the configuration file. The following example shows a configured pantalaimon proxy with the name .Em Clocktown , the homeserver URL is set to -.Em https://example.org , +.Em https://localhost:8448 , the pantalaimon proxy is listening for client connections on the address .Em localhost , and port diff --git a/docs/man/pantalaimon.5.md b/docs/man/pantalaimon.5.md index b140daf..60be6f8 100644 --- a/docs/man/pantalaimon.5.md +++ b/docs/man/pantalaimon.5.md @@ -62,7 +62,14 @@ The following keys are optional in the proxy instance sections: > operation. If this is set to "No", access tokens are stored in the pantalaimon > database in plaintext. Defaults to "Yes". -Aditional to the homeserver section a special section with the name +**DropOldKeys** + +> This option configures if a proxy instance should only keep the latest version +> of a room key from a certain user around. This effectively means that only newly +> incoming messages will be decryptable, the proxy will be unable to decrypt the +> room history. Defaults to "No". + +Additional to the homeserver section a special section with the name **Default** can be used to configure the following values for all homeservers: **ListenAddress**, @@ -111,7 +118,7 @@ overridden using appropriate environment variables. The following example shows a configured pantalaimon proxy with the name *Clocktown*, the homeserver URL is set to -*https://example.org*, +*https://localhost:8448*, the pantalaimon proxy is listening for client connections on the address *localhost*, and port @@ -150,4 +157,4 @@ pantalaimon(8) was written by Damir Jelić <[poljar@termina.org.uk](mailto:poljar@termina.org.uk)>. -Linux 5.1.3-arch2-1-ARCH - May 8, 2019 +Linux 5.11.16-arch1-1 - May 8, 2019 diff --git a/docs/man/pantalaimon.8.md b/docs/man/pantalaimon.8.md index 431d5a4..004e5f2 100644 --- a/docs/man/pantalaimon.8.md +++ b/docs/man/pantalaimon.8.md @@ -24,7 +24,7 @@ behalf of the client. is supposed to run as your own user and listen to connections on a non-privileged port. A client needs to log in using the standard Matrix HTTP calls to register itself to the daemon, such a registered user is called a pan -user and will have it's own sync loop to keep up with the server. Multiple matrix +user and will have its own sync loop to keep up with the server. Multiple matrix clients can connect and use the same pan user. If user interaction is required diff --git a/pantalaimon/client.py b/pantalaimon/client.py index e705fda..f2f6895 100644 --- a/pantalaimon/client.py +++ b/pantalaimon/client.py @@ -16,7 +16,6 @@ import asyncio import os from collections import defaultdict from pprint import pformat -from typing import Any, Dict, Optional from urllib.parse import urlparse from aiohttp.client_exceptions import ClientConnectionError @@ -135,7 +134,7 @@ class InvalidLimit(Exception): class SqliteQStore(SqliteStore): def _create_database(self): return SqliteQueueDatabase( - self.database_path, pragmas=(("foregign_keys", 1), ("secure_delete", 1)) + self.database_path, pragmas=(("foreign_keys", 1), ("secure_delete", 1)) ) def close(self): @@ -208,7 +207,6 @@ class PanClient(AsyncClient): self.key_request_cb, (RoomKeyRequest, RoomKeyRequestCancellation) ) self.add_event_callback(self.undecrypted_event_cb, MegolmEvent) - self.add_event_callback(self.store_media_cb, RoomEncryptedMedia) self.add_event_callback( self.store_thumbnail_cb, (RoomEncryptedImage, RoomEncryptedVideo, RoomEncryptedFile), @@ -270,7 +268,7 @@ class PanClient(AsyncClient): self.media_info[(mxc_server, mxc_path)] = media self.pan_store.save_media(self.server_name, media) - def store_media_cb(self, room, event): + def store_event_media(self, event): try: mxc = urlparse(event.url) except ValueError: @@ -411,6 +409,10 @@ class PanClient(AsyncClient): except (asyncio.CancelledError, KeyboardInterrupt): return + @property + def has_been_synced(self) -> bool: + self.last_sync_token is not None + async def sync_tasks(self, response): if self.index: await self.index.commit_events() @@ -541,7 +543,6 @@ class PanClient(AsyncClient): timeout = 30000 sync_filter = {"room": {"state": {"lazy_load_members": True}}} next_batch = self.pan_store.load_token(self.server_name, self.user_id) - self.last_sync_token = next_batch # We don't store any room state so initial sync needs to be with the # full_state parameter. Subsequent ones are normal. @@ -552,6 +553,7 @@ class PanClient(AsyncClient): full_state=True, since=next_batch, loop_sleep_time=loop_sleep_time, + set_presence="offline", ) ) self.task = task @@ -706,7 +708,6 @@ class PanClient(AsyncClient): for share in self.get_active_key_requests( message.user_id, message.device_id ): - continued = True if not self.continue_key_share(share): @@ -733,7 +734,7 @@ class PanClient(AsyncClient): pass response = ( - f"Succesfully continued the key requests from " + f"Successfully continued the key requests from " f"{message.user_id} via {message.device_id}" ) ret = "m.ok" @@ -758,7 +759,7 @@ class PanClient(AsyncClient): if cancelled: response = ( - f"Succesfully cancelled key requests from " + f"Successfully cancelled key requests from " f"{message.user_id} via {message.device_id}" ) ret = "m.ok" @@ -808,8 +809,9 @@ class PanClient(AsyncClient): if not isinstance(event, MegolmEvent): logger.warn( - "Encrypted event is not a megolm event:" - "\n{}".format(pformat(event_dict)) + "Encrypted event is not a megolm event:" "\n{}".format( + pformat(event_dict) + ) ) return False @@ -828,12 +830,14 @@ class PanClient(AsyncClient): ) if isinstance(decrypted_event, RoomEncryptedMedia): + self.store_event_media(decrypted_event) + decrypted_event.source["content"]["url"] = decrypted_event.url if decrypted_event.thumbnail_url: - decrypted_event.source["content"]["info"][ - "thumbnail_url" - ] = decrypted_event.thumbnail_url + decrypted_event.source["content"]["info"]["thumbnail_url"] = ( + decrypted_event.thumbnail_url + ) event_dict.update(decrypted_event.source) event_dict["decrypted"] = True @@ -904,7 +908,7 @@ class PanClient(AsyncClient): self.handle_to_device_from_sync_body(body) - for room_id, room_dict in body["rooms"]["join"].items(): + for room_id, room_dict in body.get("rooms", {}).get("join", {}).items(): try: if not self.rooms[room_id].encrypted: logger.info( @@ -919,7 +923,7 @@ class PanClient(AsyncClient): # pan sync stream did. Let's assume that the room is encrypted. pass - for event in room_dict["timeline"]["events"]: + for event in room_dict.get("timeline", {}).get("events", []): if "type" not in event: continue diff --git a/pantalaimon/config.py b/pantalaimon/config.py index 2a01714..a5e59d1 100644 --- a/pantalaimon/config.py +++ b/pantalaimon/config.py @@ -31,7 +31,7 @@ class PanConfigParser(configparser.ConfigParser): "IgnoreVerification": "False", "ListenAddress": "localhost", "ListenPort": "8009", - "LogLevel": "warnig", + "LogLevel": "warning", "Notifications": "on", "UseKeyring": "yes", "SearchRequests": "off", @@ -39,6 +39,7 @@ class PanConfigParser(configparser.ConfigParser): "IndexingBatchSize": "100", "HistoryFetchDelay": "3000", "DebugEncryption": "False", + "DropOldKeys": "False", }, converters={ "address": parse_address, @@ -112,7 +113,7 @@ class ServerConfig: E2E encrypted messages. keyring (bool): Enable or disable the OS keyring for the storage of access tokens. - search_requests (bool): Enable or disable aditional Homeserver requests + search_requests (bool): Enable or disable additional Homeserver requests for the search API endpoint. index_encrypted_only (bool): Enable or disable message indexing fro non-encrypted rooms. @@ -121,6 +122,8 @@ class ServerConfig: the room history. history_fetch_delay (int): The delay between room history fetching requests in seconds. + drop_old_keys (bool): Should Pantalaimon only keep the most recent + decryption key around. """ name = attr.ib(type=str) @@ -137,6 +140,7 @@ class ServerConfig: index_encrypted_only = attr.ib(type=bool, default=True) indexing_batch_size = attr.ib(type=int, default=100) history_fetch_delay = attr.ib(type=int, default=3) + drop_old_keys = attr.ib(type=bool, default=False) @attr.s @@ -182,7 +186,6 @@ class PanConfig: try: for section_name, section in config.items(): - if section_name == "Default": continue @@ -229,6 +232,7 @@ class PanConfig: f"already defined before." ) listen_set.add(listen_tuple) + drop_old_keys = section.getboolean("DropOldKeys") server_conf = ServerConfig( section_name, @@ -243,6 +247,7 @@ class PanConfig: index_encrypted_only, indexing_batch_size, history_fetch_delay / 1000, + drop_old_keys, ) self.servers[section_name] = server_conf diff --git a/pantalaimon/daemon.py b/pantalaimon/daemon.py index 637e373..6d47b36 100755 --- a/pantalaimon/daemon.py +++ b/pantalaimon/daemon.py @@ -17,8 +17,11 @@ import json import os import urllib.parse import concurrent.futures +from io import BufferedReader, BytesIO from json import JSONDecodeError from typing import Any, Dict +from urllib.parse import urlparse +from uuid import uuid4 import aiohttp import attr @@ -34,6 +37,7 @@ from nio import ( OlmTrustError, SendRetryError, DownloadResponse, + UploadResponse, ) from nio.crypto import decrypt_attachment @@ -47,7 +51,7 @@ from pantalaimon.client import ( ) from pantalaimon.index import INDEXING_ENABLED, InvalidQueryError from pantalaimon.log import logger -from pantalaimon.store import ClientInfo, PanStore +from pantalaimon.store import ClientInfo, PanStore, MediaInfo from pantalaimon.thread_messages import ( AcceptSasMessage, CancelSasMessage, @@ -79,6 +83,12 @@ CORS_HEADERS = { } +class NotDecryptedAvailableError(Exception): + """Exception that signals that no decrypted upload is available""" + + pass + + @attr.s class ProxyDaemon: name = attr.ib() @@ -101,6 +111,7 @@ class ProxyDaemon: client_info = attr.ib(init=False, default=attr.Factory(dict), type=dict) default_session = attr.ib(init=False, default=None) media_info = attr.ib(init=False, default=None) + upload_info = attr.ib(init=False, default=None) database_name = "pan.db" def __attrs_post_init__(self): @@ -110,9 +121,12 @@ class ProxyDaemon: self.hostname = self.homeserver.hostname self.store = PanStore(self.data_dir) accounts = self.store.load_users(self.name) - self.media_info = self.store.load_media(self.name) + self.media_info = self.store.load_media_cache(self.name) + self.upload_info = self.store.load_upload(self.name) for user_id, device_id in accounts: + token = None + if self.conf.keyring: try: token = keyring.get_password( @@ -213,7 +227,8 @@ class ProxyDaemon: if ret: msg = ( - f"Device {device.id} of user " f"{device.user_id} succesfully verified." + f"Device {device.id} of user " + f"{device.user_id} successfully verified." ) await client.send_update_device(device) else: @@ -228,7 +243,7 @@ class ProxyDaemon: if ret: msg = ( f"Device {device.id} of user " - f"{device.user_id} succesfully unverified." + f"{device.user_id} successfully unverified." ) await client.send_update_device(device) else: @@ -243,7 +258,7 @@ class ProxyDaemon: if ret: msg = ( f"Device {device.id} of user " - f"{device.user_id} succesfully blacklisted." + f"{device.user_id} successfully blacklisted." ) await client.send_update_device(device) else: @@ -260,7 +275,7 @@ class ProxyDaemon: if ret: msg = ( f"Device {device.id} of user " - f"{device.user_id} succesfully unblacklisted." + f"{device.user_id} successfully unblacklisted." ) await client.send_update_device(device) else: @@ -295,7 +310,6 @@ class ProxyDaemon: DeviceUnblacklistMessage, ), ): - device = client.device_store[message.user_id].get(message.device_id, None) if not device: @@ -344,7 +358,7 @@ class ProxyDaemon: else: info_msg = ( - f"Succesfully exported keys for {client.user_id} " f"to {path}" + f"Successfully exported keys for {client.user_id} " f"to {path}" ) logger.info(info_msg) await self.send_response( @@ -367,7 +381,7 @@ class ProxyDaemon: ) else: info_msg = ( - f"Succesfully imported keys for {client.user_id} " f"from {path}" + f"Successfully imported keys for {client.user_id} " f"from {path}" ) logger.info(info_msg) await self.send_response( @@ -406,33 +420,40 @@ class ProxyDaemon: access_token = request.query.get("access_token", "") if not access_token: - access_token = request.headers.get("Authorization", "").strip("Bearer ") + access_token = request.headers.get("Authorization", "").replace( + "Bearer ", "", 1 + ) return access_token + def sanitize_subfilter(self, request_filter: Dict[Any, Any]): + types_filter = request_filter.get("types", None) + + if types_filter: + if "m.room.encrypted" not in types_filter: + types_filter.append("m.room.encrypted") + + not_types_filter = request_filter.get("not_types", None) + + if not_types_filter: + try: + not_types_filter.remove("m.room.encrypted") + except ValueError: + pass + def sanitize_filter(self, sync_filter): # type: (Dict[Any, Any]) -> Dict[Any, Any] """Make sure that a filter isn't filtering encrypted messages.""" sync_filter = dict(sync_filter) room_filter = sync_filter.get("room", None) + self.sanitize_subfilter(sync_filter) + if room_filter: timeline_filter = room_filter.get("timeline", None) if timeline_filter: - types_filter = timeline_filter.get("types", None) - - if types_filter: - if "m.room.encrypted" not in types_filter: - types_filter.append("m.room.encrypted") - - not_types_filter = timeline_filter.get("not_types", None) - - if not_types_filter: - try: - not_types_filter.remove("m.room.encrypted") - except ValueError: - pass + self.sanitize_subfilter(timeline_filter) return sync_filter @@ -443,6 +464,7 @@ class ProxyDaemon: data=None, # type: bytes session=None, # type: aiohttp.ClientSession token=None, # type: str + use_raw_path=True, # type: bool ): # type: (...) -> aiohttp.ClientResponse """Forward the given request to our configured homeserver. @@ -457,6 +479,10 @@ class ProxyDaemon: should be used to forward the request. token (str, optional): The access token that should be used for the request. + use_raw_path (str, optional): Should the raw path be used from the + request or should we use the path and re-encode it. Some may need + their filters to be sanitized, this requires the parsed version of + the path, otherwise we leave the path as is. """ if not session: if not self.default_session: @@ -465,9 +491,7 @@ class ProxyDaemon: assert session - path = urllib.parse.quote( - request.path - ) # re-encode path stuff like room aliases + path = request.raw_path if use_raw_path else urllib.parse.quote(request.path) method = request.method headers = CIMultiDict(request.headers) @@ -547,7 +571,9 @@ class ProxyDaemon: return user - async def start_pan_client(self, access_token, user, user_id, password): + async def start_pan_client( + self, access_token, user, user_id, password, device_id=None + ): client = ClientInfo(user_id, access_token) self.client_info[access_token] = client self.store.save_server_user(self.name, user_id) @@ -572,13 +598,27 @@ class ProxyDaemon: store_class=self.client_store_class, media_info=self.media_info, ) - response = await pan_client.login(password, "pantalaimon") - if not isinstance(response, LoginResponse): - await pan_client.close() - return + if password == "": + if device_id is None: + logger.warn( + "Empty password provided and device_id was also None, not " + "starting background sync client " + ) + return + # If password is blank, we cannot login normally and must + # fall back to using the provided device_id. + pan_client.restore_login(user_id, device_id, access_token) + else: + response = await pan_client.login(password, "pantalaimon") - logger.info(f"Succesfully started new background sync client for " f"{user_id}") + if not isinstance(response, LoginResponse): + await pan_client.close() + return + + logger.info( + f"Successfully started new background sync client for " f"{user_id}" + ) await self.send_ui_message( UpdateUsersMessage(self.name, user_id, pan_client.device_id) @@ -640,13 +680,16 @@ class ProxyDaemon: if response.status == 200 and json_response: user_id = json_response.get("user_id", None) access_token = json_response.get("access_token", None) + device_id = json_response.get("device_id", None) if user_id and access_token: logger.info( - f"User: {user} succesfully logged in, starting " + f"User: {user} successfully logged in, starting " f"a background sync client." ) - await self.start_pan_client(access_token, user, user_id, password) + await self.start_pan_client( + access_token, user, user_id, password, device_id + ) return web.Response( status=response.status, @@ -692,7 +735,7 @@ class ProxyDaemon: return decryption_method(body, ignore_failures=False) except EncryptionError: logger.info("Error decrypting sync, waiting for next pan " "sync") - await client.synced.wait(), + (await client.synced.wait(),) logger.info("Pan synced, retrying decryption.") try: @@ -729,7 +772,7 @@ class ProxyDaemon: try: response = await self.forward_request( - request, params=query, token=client.access_token + request, params=query, token=client.access_token, use_raw_path=False ) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) @@ -752,6 +795,27 @@ class ProxyDaemon: body=await response.read(), ) + async def createRoom(self, request): + try: + content = await request.json() + except (JSONDecodeError, ContentTypeError): + return self._not_json + + invite = content.get("invite", ()) + if invite: + access_token = self.get_access_token(request) + + if not access_token: + return self._missing_token + + client = await self._find_client(access_token) + if not client: + return self._unknown_token + + client.users_for_key_query.update(invite) + + return await self.forward_to_web(request) + async def messages(self, request): access_token = self.get_access_token(request) @@ -762,8 +826,24 @@ class ProxyDaemon: if not client: return self._unknown_token + request_filter = request.query.get("filter", None) + query = CIMultiDict(request.query) + + if request_filter: + try: + request_filter = json.loads(request_filter) + except (JSONDecodeError, TypeError): + pass + + if isinstance(request_filter, dict): + request_filter = json.dumps(self.sanitize_filter(request_filter)) + + query["filter"] = request_filter + try: - response = await self.forward_request(request) + response = await self.forward_request( + request, params=query, use_raw_path=False + ) except ClientConnectionError as e: return web.Response(status=500, text=str(e)) @@ -787,6 +867,56 @@ class ProxyDaemon: body=await response.read(), ) + def _get_upload_and_media_info(self, content_uri: str): + try: + upload_info = self.upload_info[content_uri] + except KeyError: + upload_info = self.store.load_upload(self.name, content_uri) + if not upload_info: + return None, None + + self.upload_info[content_uri] = upload_info + + mxc = urlparse(content_uri) + mxc_server = mxc.netloc.strip("/") + mxc_path = mxc.path.strip("/") + + media_info = self.store.load_media(self.name, mxc_server, mxc_path) + if not media_info: + return None, None + + self.media_info[(mxc_server, mxc_path)] = media_info + + return upload_info, media_info + + async def _decrypt_uri(self, content_uri, client): + upload_info, media_info = self._get_upload_and_media_info(content_uri) + if not upload_info or not media_info: + raise NotDecryptedAvailableError + + response, decrypted_file = await self._load_decrypted_file( + media_info.mxc_server, media_info.mxc_path, upload_info.filename + ) + + if response is None and decrypted_file is None: + raise NotDecryptedAvailableError + + if not isinstance(response, DownloadResponse): + raise NotDecryptedAvailableError + + decrypted_upload, _ = await client.upload( + data_provider=BufferedReader(BytesIO(decrypted_file)), + content_type=upload_info.mimetype, + filename=upload_info.filename, + encrypt=False, + filesize=len(decrypted_file), + ) + + if not isinstance(decrypted_upload, UploadResponse): + raise NotDecryptedAvailableError + + return decrypted_upload.content_uri + async def send_message(self, request): access_token = self.get_access_token(request) @@ -799,12 +929,36 @@ class ProxyDaemon: room_id = request.match_info["room_id"] - # The room is not in the joined rooms list, just forward it. try: room = client.rooms[room_id] encrypt = room.encrypted except KeyError: - return await self.forward_to_web(request, token=client.access_token) + # The room is not in the joined rooms list, either the pan client + # didn't manage to sync the state or we're not joined, in either + # case send an error response. + if client.has_been_synced: + return web.json_response( + { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event.", + }, + headers=CORS_HEADERS, + status=403, + ) + else: + logger.error( + "The internal Pantalaimon client did not manage " + "to sync with the server." + ) + return web.json_response( + { + "errcode": "M_UNKNOWN", + "error": "The pantalaimon client did not manage to sync with " + "the server", + }, + headers=CORS_HEADERS, + status=500, + ) # Don't encrypt reactions for now - they are weird and clients # need to support them like this. @@ -812,23 +966,84 @@ class ProxyDaemon: if request.match_info["event_type"] == "m.reaction": encrypt = False - # The room isn't encrypted just forward the message. - if not encrypt: - return await self.forward_to_web(request, token=client.access_token) - msgtype = request.match_info["event_type"] - txnid = request.match_info["txnid"] try: content = await request.json() except (JSONDecodeError, ContentTypeError): return self._not_json + # The room isn't encrypted just forward the message. + if not encrypt: + content_msgtype = content.get("msgtype") + if ( + content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] + or msgtype == "m.room.avatar" + ): + try: + content["url"] = await self._decrypt_uri(content["url"], client) + if ( + "info" in content + and "thumbnail_url" in content["info"] + and content["info"]["thumbnail_url"] is not None + ): + content["info"]["thumbnail_url"] = await self._decrypt_uri( + content["info"]["thumbnail_url"], client + ) + return await self.forward_to_web( + request, data=json.dumps(content), token=client.access_token + ) + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except (KeyError, NotDecryptedAvailableError): + return await self.forward_to_web(request, token=client.access_token) + + return await self.forward_to_web(request, token=client.access_token) + + txnid = request.match_info.get("txnid", uuid4()) + async def _send(ignore_unverified=False): try: - response = await client.room_send( - room_id, msgtype, content, txnid, ignore_unverified - ) + content_msgtype = content.get("msgtype") + if ( + content_msgtype in ["m.image", "m.video", "m.audio", "m.file"] + or msgtype == "m.room.avatar" + ): + upload_info, media_info = self._get_upload_and_media_info( + content["url"] + ) + if not upload_info or not media_info: + response = await client.room_send( + room_id, msgtype, content, txnid, ignore_unverified + ) + + return web.Response( + status=response.transport_response.status, + content_type=response.transport_response.content_type, + headers=CORS_HEADERS, + body=await response.transport_response.read(), + ) + + media_info.to_content(content, upload_info.mimetype) + if content["info"].get("thumbnail_url", False): + ( + thumb_upload_info, + thumb_media_info, + ) = self._get_upload_and_media_info( + content["info"]["thumbnail_url"] + ) + if thumb_upload_info and thumb_media_info: + thumb_media_info.to_thumbnail( + content, thumb_upload_info.mimetype + ) + + response = await client.room_send( + room_id, msgtype, content, txnid, ignore_unverified + ) + else: + response = await client.room_send( + room_id, msgtype, content, txnid, ignore_unverified + ) return web.Response( status=response.transport_response.status, @@ -841,7 +1056,7 @@ class ProxyDaemon: except SendRetryError as e: return web.Response(status=503, text=str(e)) - # Aquire a semaphore here so we only send out one + # Acquire a semaphore here so we only send out one # UnverifiedDevicesSignal sem = client.send_semaphores[room_id] @@ -1002,42 +1217,47 @@ class ProxyDaemon: return web.json_response(result, headers=CORS_HEADERS, status=200) - async def download(self, request): - server_name = request.match_info["server_name"] - media_id = request.match_info["media_id"] - file_name = request.match_info.get("file_name") - - try: - media_info = self.media_info[(server_name, media_id)] - except KeyError: - media_info = self.store.load_media(self.name, server_name, media_id) - - if not media_info: - logger.info(f"No media info found for {server_name}/{media_id}") - return await self.forward_to_web(request) - - self.media_info[(server_name, media_id)] = media_info - - try: - key = media_info.key["k"] - hash = media_info.hashes["sha256"] - except KeyError: - logger.warn( - f"Media info for {server_name}/{media_id} doesn't contain a key or hash." - ) - return await self.forward_to_web(request) - - if not self.pan_clients: - return await self.forward_to_web(request) - + async def upload(self, request): + file_name = request.query.get("filename", "") + content_type = request.headers.get("Content-Type", "application/octet-stream") client = next(iter(self.pan_clients.values())) + body = await request.read() try: - response = await client.download(server_name, media_id, file_name) - except ClientConnectionError as e: - return web.Response(status=500, text=str(e)) + response, maybe_keys = await client.upload( + data_provider=BufferedReader(BytesIO(body)), + content_type=content_type, + filename=file_name, + encrypt=True, + filesize=len(body), + ) + + if not isinstance(response, UploadResponse): + return web.Response( + status=response.transport_response.status, + content_type=response.transport_response.content_type, + headers=CORS_HEADERS, + body=await response.transport_response.read(), + ) + + self.store.save_upload( + self.name, response.content_uri, file_name, content_type + ) + + mxc = urlparse(response.content_uri) + mxc_server = mxc.netloc.strip("/") + mxc_path = mxc.path.strip("/") + + logger.info(f"Adding media info for {mxc_server}/{mxc_path} to the store") + media_info = MediaInfo( + mxc_server, + mxc_path, + maybe_keys["key"], + maybe_keys["iv"], + maybe_keys["hashes"], + ) + self.store.save_media(self.name, media_info) - if not isinstance(response, DownloadResponse): return web.Response( status=response.transport_response.status, content_type=response.transport_response.content_type, @@ -1045,6 +1265,46 @@ class ProxyDaemon: body=await response.transport_response.read(), ) + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except SendRetryError as e: + return web.Response(status=503, text=str(e)) + + async def _load_decrypted_file(self, server_name, media_id, file_name): + try: + media_info = self.media_info[(server_name, media_id)] + except KeyError: + media_info = self.store.load_media(self.name, server_name, media_id) + + if not media_info: + logger.info(f"No media info found for {server_name}/{media_id}") + return None, None + + self.media_info[(server_name, media_id)] = media_info + + try: + key = media_info.key["k"] + hash = media_info.hashes["sha256"] + except KeyError as e: + logger.warn( + f"Media info for {server_name}/{media_id} doesn't contain a key or hash." + ) + raise e + if not self.pan_clients: + return None, None + + client = next(iter(self.pan_clients.values())) + + try: + response = await client.download( + server_name=server_name, media_id=media_id, filename=file_name + ) + except ClientConnectionError as e: + raise e + + if not isinstance(response, DownloadResponse): + return response, None + logger.info(f"Decrypting media {server_name}/{media_id}") loop = asyncio.get_running_loop() @@ -1053,6 +1313,60 @@ class ProxyDaemon: pool, decrypt_attachment, response.body, key, hash, media_info.iv ) + return response, decrypted_file + + async def profile(self, request): + access_token = self.get_access_token(request) + + if not access_token: + return self._missing_token + + client = await self._find_client(access_token) + if not client: + return self._unknown_token + + try: + content = await request.json() + except (JSONDecodeError, ContentTypeError): + return self._not_json + + try: + content["avatar_url"] = await self._decrypt_uri( + content["avatar_url"], client + ) + return await self.forward_to_web( + request, data=json.dumps(content), token=client.access_token + ) + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except (KeyError, NotDecryptedAvailableError): + return await self.forward_to_web(request, token=client.access_token) + + async def download(self, request): + server_name = request.match_info["server_name"] + media_id = request.match_info["media_id"] + file_name = request.match_info.get("file_name") + + try: + response, decrypted_file = await self._load_decrypted_file( + server_name, media_id, file_name + ) + + if response is None and decrypted_file is None: + return await self.forward_to_web(request) + except ClientConnectionError as e: + return web.Response(status=500, text=str(e)) + except KeyError: + return await self.forward_to_web(request) + + if not isinstance(response, DownloadResponse): + return web.Response( + status=response.transport_response.status, + content_type=response.transport_response.content_type, + headers=CORS_HEADERS, + body=await response.transport_response.read(), + ) + return web.Response( status=response.transport_response.status, content_type=response.transport_response.content_type, diff --git a/pantalaimon/index.py b/pantalaimon/index.py index 5c8e02b..3d49614 100644 --- a/pantalaimon/index.py +++ b/pantalaimon/index.py @@ -23,7 +23,6 @@ if False: import json import os from functools import partial - from typing import Any, Dict, List, Optional, Tuple import attr import tantivy @@ -230,7 +229,6 @@ if False: ) for message in query: - event = message.event event_dict = { @@ -501,6 +499,5 @@ if False: return search_result - else: INDEXING_ENABLED = False diff --git a/pantalaimon/main.py b/pantalaimon/main.py index d6db3d1..741937e 100644 --- a/pantalaimon/main.py +++ b/pantalaimon/main.py @@ -15,7 +15,6 @@ import asyncio import os import signal -from typing import Optional import click import janus @@ -23,12 +22,13 @@ import keyring import logbook import nio from aiohttp import web -from appdirs import user_config_dir, user_data_dir +from platformdirs import user_config_dir, user_data_dir from logbook import StderrHandler from pantalaimon.config import PanConfig, PanConfigError, parse_log_level from pantalaimon.daemon import ProxyDaemon from pantalaimon.log import logger +from pantalaimon.store import KeyDroppingSqliteStore from pantalaimon.thread_messages import DaemonResponse from pantalaimon.ui import UI_ENABLED @@ -47,6 +47,8 @@ def create_dirs(data_dir, conf_dir): async def init(data_dir, server_conf, send_queue, recv_queue): """Initialize the proxy and the http server.""" + store_class = KeyDroppingSqliteStore if server_conf.drop_old_keys else None + proxy = ProxyDaemon( server_conf.name, server_conf.homeserver, @@ -56,31 +58,56 @@ async def init(data_dir, server_conf, send_queue, recv_queue): recv_queue=recv_queue.async_q if recv_queue else None, proxy=server_conf.proxy.geturl() if server_conf.proxy else None, ssl=None if server_conf.ssl is True else False, + client_store_class=store_class, ) - app = web.Application() + # 100 MB max POST size + app = web.Application(client_max_size=1024**2 * 100) app.add_routes( [ web.post("/_matrix/client/r0/login", proxy.login), + web.post("/_matrix/client/v3/login", proxy.login), web.get("/_matrix/client/r0/sync", proxy.sync), + web.get("/_matrix/client/v3/sync", proxy.sync), + web.post("/_matrix/client/r0/createRoom", proxy.createRoom), + web.post("/_matrix/client/v3/createRoom", proxy.createRoom), web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages), + web.get("/_matrix/client/v3/rooms/{room_id}/messages", proxy.messages), web.put( r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}", proxy.send_message, ), + web.put( + r"/_matrix/client/v3/rooms/{room_id}/send/{event_type}/{txnid}", + proxy.send_message, + ), + web.post( + r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}", + proxy.send_message, + ), web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter), + web.post("/_matrix/client/v3/user/{user_id}/filter", proxy.filter), web.post("/.well-known/matrix/client", proxy.well_known), web.get("/.well-known/matrix/client", proxy.well_known), web.post("/_matrix/client/r0/search", proxy.search), + web.post("/_matrix/client/v3/search", proxy.search), web.options("/_matrix/client/r0/search", proxy.search_opts), + web.options("/_matrix/client/v3/search", proxy.search_opts), web.get( "/_matrix/media/v1/download/{server_name}/{media_id}", proxy.download ), + web.get( + "/_matrix/media/v3/download/{server_name}/{media_id}", proxy.download + ), web.get( "/_matrix/media/v1/download/{server_name}/{media_id}/{file_name}", proxy.download, ), + web.get( + "/_matrix/media/v3/download/{server_name}/{media_id}/{file_name}", + proxy.download, + ), web.get( "/_matrix/media/r0/download/{server_name}/{media_id}", proxy.download ), @@ -88,6 +115,22 @@ async def init(data_dir, server_conf, send_queue, recv_queue): "/_matrix/media/r0/download/{server_name}/{media_id}/{file_name}", proxy.download, ), + web.post( + r"/_matrix/media/r0/upload", + proxy.upload, + ), + web.post( + r"/_matrix/media/v3/upload", + proxy.upload, + ), + web.put( + r"/_matrix/client/r0/profile/{userId}/avatar_url", + proxy.profile, + ), + web.put( + r"/_matrix/client/v3/profile/{userId}/avatar_url", + proxy.profile, + ), ] ) app.router.add_route("*", "/" + "{proxyPath:.*}", proxy.router) @@ -245,7 +288,7 @@ async def daemon(context, log_level, debug_encryption, config, data_path): "connect to pantalaimon." ) ) -@click.version_option(version="0.6.2", prog_name="pantalaimon") +@click.version_option(version="0.10.5", prog_name="pantalaimon") @click.option( "--log-level", type=click.Choice(["error", "warning", "info", "debug"]), diff --git a/pantalaimon/panctl.py b/pantalaimon/panctl.py index 08efc91..f37ab88 100644 --- a/pantalaimon/panctl.py +++ b/pantalaimon/panctl.py @@ -20,10 +20,16 @@ import sys from collections import defaultdict from itertools import zip_longest from typing import List +from shlex import split import attr import click -from gi.repository import GLib + +try: + from gi.repository import GLib +except ModuleNotFoundError: + from pgi.repository import GLib + from prompt_toolkit import __version__ as ptk_version from prompt_toolkit import HTML, PromptSession, print_formatted_text from prompt_toolkit.completion import Completer, Completion, PathCompleter @@ -459,7 +465,7 @@ class PanCtl: def sas_done(self, pan_user, user_id, device_id, _): print( f"Device {device_id} of user {user_id}" - f" succesfully verified for pan user {pan_user}." + f" successfully verified for pan user {pan_user}." ) def show_sas_invite(self, pan_user, user_id, device_id, _): @@ -584,7 +590,7 @@ class PanCtl: parser = PanctlParser(self.commands) try: - args = parser.parse_args(result.split()) + args = parser.parse_args(split(result, posix=False)) except ParseError: continue @@ -690,9 +696,9 @@ class PanCtl: "the pantalaimon daemon." ) ) -@click.version_option(version="0.6.2", prog_name="panctl") +@click.version_option(version="0.10.5", prog_name="panctl") def main(): - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() glib_loop = GLib.MainLoop() try: diff --git a/pantalaimon/store.py b/pantalaimon/store.py index 910df64..0dfe045 100644 --- a/pantalaimon/store.py +++ b/pantalaimon/store.py @@ -15,13 +15,15 @@ import json import os from collections import defaultdict -from typing import List, Optional, Tuple +from typing import Any, Dict import attr -from nio.crypto import TrustState +from nio.crypto import TrustState, GroupSessionStore from nio.store import ( Accounts, + MegolmInboundSessions, DeviceKeys, + SqliteStore, DeviceTrustState, use_database, use_database_atomic, @@ -29,8 +31,8 @@ from nio.store import ( from peewee import SQL, DoesNotExist, ForeignKeyField, Model, SqliteDatabase, TextField from cachetools import LRUCache - MAX_LOADED_MEDIA = 10000 +MAX_LOADED_UPLOAD = 10000 @attr.s @@ -47,6 +49,33 @@ class MediaInfo: iv = attr.ib(type=str) hashes = attr.ib(type=dict) + def to_content(self, content: Dict, mime_type: str) -> Dict[Any, Any]: + content["file"] = { + "v": "v2", + "key": self.key, + "iv": self.iv, + "hashes": self.hashes, + "url": content["url"], + "mimetype": mime_type, + } + + def to_thumbnail(self, content: Dict, mime_type: str) -> Dict[Any, Any]: + content["info"]["thumbnail_file"] = { + "v": "v2", + "key": self.key, + "iv": self.iv, + "hashes": self.hashes, + "url": content["info"]["thumbnail_url"], + "mimetype": mime_type, + } + + +@attr.s +class UploadInfo: + content_uri = attr.ib(type=str) + filename = attr.ib(type=str) + mimetype = attr.ib(type=str) + class DictField(TextField): def python_value(self, value): # pragma: no cover @@ -113,6 +142,18 @@ class PanMediaInfo(Model): constraints = [SQL("UNIQUE(server_id, mxc_server, mxc_path)")] +class PanUploadInfo(Model): + server = ForeignKeyField( + model=Servers, column_name="server_id", backref="upload", on_delete="CASCADE" + ) + content_uri = TextField() + filename = TextField() + mimetype = TextField() + + class Meta: + constraints = [SQL("UNIQUE(server_id, content_uri)")] + + @attr.s class ClientInfo: user_id = attr.ib(type=str) @@ -135,6 +176,7 @@ class PanStore: PanSyncTokens, PanFetcherTasks, PanMediaInfo, + PanUploadInfo, ] def __attrs_post_init__(self): @@ -162,6 +204,43 @@ class PanStore: except DoesNotExist: return None + @use_database + def save_upload(self, server, content_uri, filename, mimetype): + server = Servers.get(name=server) + + PanUploadInfo.insert( + server=server, + content_uri=content_uri, + filename=filename, + mimetype=mimetype, + ).on_conflict_ignore().execute() + + @use_database + def load_upload(self, server, content_uri=None): + server, _ = Servers.get_or_create(name=server) + + if not content_uri: + upload_cache = LRUCache(maxsize=MAX_LOADED_UPLOAD) + + for i, u in enumerate(server.upload): + if i > MAX_LOADED_UPLOAD: + break + + upload = UploadInfo(u.content_uri, u.filename, u.mimetype) + upload_cache[u.content_uri] = upload + + return upload_cache + else: + u = PanUploadInfo.get_or_none( + PanUploadInfo.server == server, + PanUploadInfo.content_uri == content_uri, + ) + + if not u: + return None + + return UploadInfo(u.content_uri, u.filename, u.mimetype) + @use_database def save_media(self, server, media): server = Servers.get(name=server) @@ -175,32 +254,34 @@ class PanStore: hashes=media.hashes, ).on_conflict_ignore().execute() + @use_database + def load_media_cache(self, server): + server, _ = Servers.get_or_create(name=server) + media_cache = LRUCache(maxsize=MAX_LOADED_MEDIA) + + for i, m in enumerate(server.media): + if i > MAX_LOADED_MEDIA: + break + + media = MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) + media_cache[(m.mxc_server, m.mxc_path)] = media + + return media_cache + @use_database def load_media(self, server, mxc_server=None, mxc_path=None): server, _ = Servers.get_or_create(name=server) - if not mxc_path: - media_cache = LRUCache(maxsize=MAX_LOADED_MEDIA) + m = PanMediaInfo.get_or_none( + PanMediaInfo.server == server, + PanMediaInfo.mxc_server == mxc_server, + PanMediaInfo.mxc_path == mxc_path, + ) - for i, m in enumerate(server.media): - if i > MAX_LOADED_MEDIA: - break + if not m: + return None - media = MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) - media_cache[(m.mxc_server, m.mxc_path)] = media - - return media_cache - else: - m = PanMediaInfo.get_or_none( - PanMediaInfo.server == server, - PanMediaInfo.mxc_server == mxc_server, - PanMediaInfo.mxc_path == mxc_path, - ) - - if not m: - return None - - return MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) + return MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) @use_database_atomic def replace_fetcher_task(self, server, pan_user, old_task, new_task): @@ -226,6 +307,7 @@ class PanStore: user=user, room_id=task.room_id, token=task.token ).execute() + @use_database def load_fetcher_tasks(self, server, pan_user): server = Servers.get(name=server) user = ServerUsers.get(server=server, user_id=pan_user) @@ -349,7 +431,6 @@ class PanStore: device_store = defaultdict(dict) for d in account.device_keys: - if d.deleted: continue @@ -372,3 +453,47 @@ class PanStore: store[account.user_id] = device_store return store + + +class KeyDroppingSqliteStore(SqliteStore): + @use_database + def save_inbound_group_session(self, session): + """Save the provided Megolm inbound group session to the database. + + Args: + session (InboundGroupSession): The session to save. + """ + account = self._get_account() + assert account + + MegolmInboundSessions.delete().where( + MegolmInboundSessions.sender_key == session.sender_key, + MegolmInboundSessions.account == account, + MegolmInboundSessions.room_id == session.room_id, + ).execute() + + super().save_inbound_group_session(session) + + @use_database + def load_inbound_group_sessions(self): + store = super().load_inbound_group_sessions() + + return KeyDroppingGroupSessionStore.from_group_session_store(store) + + +class KeyDroppingGroupSessionStore(GroupSessionStore): + def from_group_session_store(store): + new_store = KeyDroppingGroupSessionStore() + new_store._entries = store._entries + + return new_store + + def add(self, session) -> bool: + room_id = session.room_id + sender_key = session.sender_key + if session in self._entries[room_id][sender_key].values(): + return False + + self._entries[room_id][sender_key].clear() + self._entries[room_id][sender_key][session.id] = session + return True diff --git a/pantalaimon/ui.py b/pantalaimon/ui.py index 25eb2f8..813b67e 100644 --- a/pantalaimon/ui.py +++ b/pantalaimon/ui.py @@ -30,6 +30,7 @@ if UI_ENABLED: from gi.repository import GLib from pydbus import SessionBus from pydbus.generic import signal + from dbus.mainloop.glib import DBusGMainLoop from nio import RoomKeyRequest, RoomKeyRequestCancellation @@ -447,6 +448,7 @@ if UI_ENABLED: config = attr.ib() loop = attr.ib(init=False) + dbus_loop = attr.ib(init=False) store = attr.ib(init=False) users = attr.ib(init=False) devices = attr.ib(init=False) @@ -457,6 +459,7 @@ if UI_ENABLED: def __attrs_post_init__(self): self.loop = None + self.dbus_loop = None id_counter = IdCounter() @@ -467,14 +470,14 @@ if UI_ENABLED: self.bus.publish("org.pantalaimon1", self.control_if, self.device_if) def unverified_notification(self, message): - notificaton = notify2.Notification( + notification = notify2.Notification( "Unverified devices.", message=( f"There are unverified devices in the room " f"{message.room_display_name}." ), ) - notificaton.set_category("im") + notification.set_category("im") def send_cb(notification, action_key, user_data): message = user_data @@ -485,20 +488,20 @@ if UI_ENABLED: self.control_if.CancelSending(message.pan_user, message.room_id) if "actions" in notify2.get_server_caps(): - notificaton.add_action("send", "Send anyways", send_cb, message) - notificaton.add_action("cancel", "Cancel sending", cancel_cb, message) + notification.add_action("send", "Send anyways", send_cb, message) + notification.add_action("cancel", "Cancel sending", cancel_cb, message) - notificaton.show() + notification.show() def sas_invite_notification(self, message): - notificaton = notify2.Notification( + notification = notify2.Notification( "Key verification invite", message=( f"{message.user_id} via {message.device_id} has started " f"a key verification process." ), ) - notificaton.set_category("im") + notification.set_category("im") def accept_cb(notification, action_key, user_data): message = user_data @@ -513,17 +516,17 @@ if UI_ENABLED: ) if "actions" in notify2.get_server_caps(): - notificaton.add_action("accept", "Accept", accept_cb, message) - notificaton.add_action("cancel", "Cancel", cancel_cb, message) + notification.add_action("accept", "Accept", accept_cb, message) + notification.add_action("cancel", "Cancel", cancel_cb, message) - notificaton.show() + notification.show() def sas_show_notification(self, message): emojis = [x[0] for x in message.emoji] emoji_str = " ".join(emojis) - notificaton = notify2.Notification( + notification = notify2.Notification( "Short authentication string", message=( f"Short authentication string for the key verification of" @@ -531,7 +534,7 @@ if UI_ENABLED: f"{emoji_str}" ), ) - notificaton.set_category("im") + notification.set_category("im") def confirm_cb(notification, action_key, user_data): message = user_data @@ -546,21 +549,21 @@ if UI_ENABLED: ) if "actions" in notify2.get_server_caps(): - notificaton.add_action("confirm", "Confirm", confirm_cb, message) - notificaton.add_action("cancel", "Cancel", cancel_cb, message) + notification.add_action("confirm", "Confirm", confirm_cb, message) + notification.add_action("cancel", "Cancel", cancel_cb, message) - notificaton.show() + notification.show() def sas_done_notification(self, message): - notificaton = notify2.Notification( + notification = notify2.Notification( "Device successfully verified.", message=( f"Device {message.device_id} of user {message.user_id} " f"successfully verified." ), ) - notificaton.set_category("im") - notificaton.show() + notification.set_category("im") + notification.show() def message_callback(self): try: @@ -632,11 +635,12 @@ if UI_ENABLED: return True def run(self): + self.dbus_loop = DBusGMainLoop() self.loop = GLib.MainLoop() if self.config.notifications: try: - notify2.init("pantalaimon", mainloop=self.loop) + notify2.init("pantalaimon", mainloop=self.dbus_loop) self.notifications = True except dbus.DBusException: logger.error( @@ -646,6 +650,7 @@ if UI_ENABLED: self.notifications = False GLib.timeout_add(100, self.message_callback) + if not self.loop: return diff --git a/setup.py b/setup.py index eabb128..9f0584e 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,66 @@ # -*- coding: utf-8 -*- from setuptools import find_packages, setup +import os with open("README.md", encoding="utf-8") as f: long_description = f.read() + +def get_manpages(): + """ + This function goes and gets all the man pages so they can be installed when + the package is installed. + """ + man_pages = [] + for root, _, files in os.walk("docs/man"): + for file in files: + if file.endswith((".1", ".5", ".8")): + man_section = file.split(".")[-1] + dest_dir = os.path.join("share", "man", f"man{man_section}") + man_pages.append((dest_dir, [os.path.join(root, file)])) + return man_pages + + setup( name="pantalaimon", - version="0.6.2", + version="0.10.5", url="https://github.com/matrix-org/pantalaimon", author="The Matrix.org Team", author_email="poljar@termina.org.uk", - description=("A Matrix proxy daemon that adds E2E encryption " - "capabilities."), + description=("A Matrix proxy daemon that adds E2E encryption capabilities."), long_description=long_description, long_description_content_type="text/markdown", license="Apache License, Version 2.0", packages=find_packages(), install_requires=[ - "attrs <= 19.3", - "aiohttp <= 3.6", - "appdirs <= 1.4", - "click <= 7.1", - "keyring <= 21.2", - "logbook <= 1.5", - "peewee <= 3.13", - "janus <= 0.5", - "cachetools >= 3.0" - "prompt_toolkit>2<4", + "attrs >= 19.3.0", + "aiohttp >= 3.6, < 4.0", + "platformdirs >= 4.3.6", + "click >= 7.1.2", + "keyring >= 21.2.1", + "logbook >= 1.5.3", + "peewee >= 3.13.1", + "janus >= 0.5", + "cachetools >= 3.0.0", + "prompt_toolkit > 2, < 4", "typing;python_version<'3.5'", - "matrix-nio[e2e] <= 0.12" + "matrix-nio[e2e] >= 0.24, < 0.25.2", ], extras_require={ "ui": [ - "dbus-python <= 1.2", - "PyGObject <= 3.36", - "pydbus <= 0.6", - "notify2 <= 0.3", + "dbus-python >= 1.2, < 1.3", + "PyGObject >= 3.46, < 3.50", + "pydbus >= 0.6, < 0.7", + "notify2 >= 0.3, < 0.4", ] }, entry_points={ - "console_scripts": ["pantalaimon=pantalaimon.main:main", - "panctl=pantalaimon.panctl:main"], + "console_scripts": [ + "pantalaimon=pantalaimon.main:main", + "panctl=pantalaimon.panctl:main", + ], }, - zip_safe=False + zip_safe=False, + data_files=get_manpages(), ) diff --git a/test-requirements.txt b/test-requirements.txt index 36816bf..bdbc671 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,9 @@ -pytest -pytest-flake8 -pytest-isort -pytest-cov -faker -aiohttp -pytest-aiohttp -aioresponses +pytest==8.3.5 +pytest-flake8==1.2.2 +pytest-isort==4.0.0 +pytest-cov==5.0.0 +faker<=37.1.0 +aiohttp<=3.11.16 +pytest-aiohttp<=1.1.0 +pytest-asyncio<=0.26.0 +aioresponses<=0.7.8 diff --git a/tests/conftest.py b/tests/conftest.py index c79de19..ceb8902 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,11 +34,9 @@ class Provider(BaseProvider): def client(self): return ClientInfo(faker.mx_id(), faker.access_token()) - def avatar_url(self): return "mxc://{}/{}#auto".format( - faker.hostname(), - "".join(choices(ascii_letters) for i in range(24)) + faker.hostname(), "".join(choices(ascii_letters) for i in range(24)) ) def olm_key_pair(self): @@ -56,7 +54,6 @@ class Provider(BaseProvider): ) - faker.add_provider(Provider) @@ -80,13 +77,7 @@ def tempdir(): @pytest.fixture def panstore(tempdir): for _ in range(10): - store = SqliteStore( - faker.mx_id(), - faker.device_id(), - tempdir, - "", - "pan.db" - ) + store = SqliteStore(faker.mx_id(), faker.device_id(), tempdir, "", "pan.db") account = OlmAccount() store.save_account(account) @@ -116,7 +107,7 @@ async def pan_proxy_server(tempdir, aiohttp_server): server_name = faker.hostname() - config = ServerConfig(server_name, urlparse("https://example.org")) + config = ServerConfig(server_name, urlparse("https://example.org"), keyring=False) pan_queue = janus.Queue() ui_queue = janus.Queue() @@ -130,21 +121,23 @@ async def pan_proxy_server(tempdir, aiohttp_server): recv_queue=ui_queue.async_q, proxy=None, ssl=False, - client_store_class=SqliteStore + client_store_class=SqliteStore, ) - app.add_routes([ - web.post("/_matrix/client/r0/login", proxy.login), - web.get("/_matrix/client/r0/sync", proxy.sync), - web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages), - web.put( - r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}", - proxy.send_message - ), - web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter), - web.post("/_matrix/client/r0/search", proxy.search), - web.options("/_matrix/client/r0/search", proxy.search_opts), - ]) + app.add_routes( + [ + web.post("/_matrix/client/r0/login", proxy.login), + web.get("/_matrix/client/r0/sync", proxy.sync), + web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages), + web.put( + r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}", + proxy.send_message, + ), + web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter), + web.post("/_matrix/client/r0/search", proxy.search), + web.options("/_matrix/client/r0/search", proxy.search_opts), + ] + ) server = await aiohttp_server(app) @@ -161,7 +154,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client): "access_token": "abc123", "device_id": "GHTYAJCE", "home_server": "example.org", - "user_id": "@example:example.org" + "user_id": "@example:example.org", } aioclient = await aiohttp_client(server) @@ -170,7 +163,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client): "https://example.org/_matrix/client/r0/login", status=200, payload=login_response, - repeat=True + repeat=True, ) await aioclient.post( @@ -179,7 +172,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client): "type": "m.login.password", "user": "example", "password": "wordpass", - } + }, ) yield server, aioclient, proxy, queues diff --git a/tests/pan_client_test.py b/tests/pan_client_test.py index 7202698..67f130f 100644 --- a/tests/pan_client_test.py +++ b/tests/pan_client_test.py @@ -25,10 +25,10 @@ ALICE_ID = "@alice:example.org" @pytest.fixture -async def client(tmpdir, loop): +async def client(tmpdir): store = PanStore(tmpdir) queue = janus.Queue() - conf = ServerConfig("example", "https://exapmle.org") + conf = ServerConfig("example", "https://example.org") conf.history_fetch_delay = 0.1 store.save_server_user("example", "@example:example.org") @@ -371,7 +371,7 @@ class TestClass(object): await client.loop_stop() - async def test_history_fetching_tasks(self, client, aioresponse, loop): + async def test_history_fetching_tasks(self, client, aioresponse): if not INDEXING_ENABLED: pytest.skip("Indexing needs to be enabled to test this") @@ -380,7 +380,9 @@ class TestClass(object): ) aioresponse.get( - sync_url, status=200, payload=self.initial_sync_response, + sync_url, + status=200, + payload=self.initial_sync_response, ) aioresponse.get(sync_url, status=200, payload=self.empty_sync, repeat=True) @@ -421,7 +423,7 @@ class TestClass(object): tasks = client.pan_store.load_fetcher_tasks(client.server_name, client.user_id) assert len(tasks) == 1 - # Check that the task is our prev_batch from the sync resposne + # Check that the task is our prev_batch from the sync response assert tasks[0].room_id == TEST_ROOM_ID assert tasks[0].token == "t392-516_47314_0_7_1_1_1_11444_1" @@ -431,7 +433,7 @@ class TestClass(object): tasks = client.pan_store.load_fetcher_tasks(client.server_name, client.user_id) assert len(tasks) == 1 - # Check that the task is our end token from the messages resposne + # Check that the task is our end token from the messages response assert tasks[0].room_id == TEST_ROOM_ID assert tasks[0].token == "t47409-4357353_219380_26003_2265" @@ -445,7 +447,7 @@ class TestClass(object): await client.loop_stop() - async def test_history_fetching_resume(self, client, aioresponse, loop): + async def test_history_fetching_resume(self, client, aioresponse): if not INDEXING_ENABLED: pytest.skip("Indexing needs to be enabled to test this") @@ -454,7 +456,9 @@ class TestClass(object): ) aioresponse.get( - sync_url, status=200, payload=self.initial_sync_response, + sync_url, + status=200, + payload=self.initial_sync_response, ) aioresponse.get(sync_url, status=200, payload=self.empty_sync, repeat=True) @@ -519,7 +523,7 @@ class TestClass(object): ) assert len(tasks) == 1 - # Check that the task is our end token from the messages resposne + # Check that the task is our end token from the messages response assert tasks[0].room_id == TEST_ROOM_ID assert tasks[0].token == "t47409-4357353_219380_26003_2265" diff --git a/tests/proxy_test.py b/tests/proxy_test.py index b50379e..ecf515e 100644 --- a/tests/proxy_test.py +++ b/tests/proxy_test.py @@ -1,9 +1,7 @@ -import asyncio import json import re from collections import defaultdict -from aiohttp import web from nio.crypto import OlmDevice from conftest import faker @@ -27,7 +25,7 @@ class TestClass(object): "access_token": "abc123", "device_id": "GHTYAJCE", "home_server": "example.org", - "user_id": "@example:example.org" + "user_id": "@example:example.org", } @property @@ -36,12 +34,7 @@ class TestClass(object): @property def keys_upload_response(self): - return { - "one_time_key_counts": { - "curve25519": 10, - "signed_curve25519": 20 - } - } + return {"one_time_key_counts": {"curve25519": 10, "signed_curve25519": 20}} @property def example_devices(self): @@ -52,10 +45,7 @@ class TestClass(object): devices[device.user_id][device.id] = device bob_device = OlmDevice( - BOB_ID, - BOB_DEVICE, - {"ed25519": BOB_ONETIME, - "curve25519": BOB_CURVE} + BOB_ID, BOB_DEVICE, {"ed25519": BOB_ONETIME, "curve25519": BOB_CURVE} ) devices[BOB_ID][BOB_DEVICE] = bob_device @@ -71,7 +61,7 @@ class TestClass(object): "https://example.org/_matrix/client/r0/login", status=200, payload=self.login_response, - repeat=True + repeat=True, ) assert not daemon.pan_clients @@ -82,7 +72,7 @@ class TestClass(object): "type": "m.login.password", "user": "example", "password": "wordpass", - } + }, ) assert resp.status == 200 @@ -105,11 +95,11 @@ class TestClass(object): "https://example.org/_matrix/client/r0/login", status=200, payload=self.login_response, - repeat=True + repeat=True, ) sync_url = re.compile( - r'^https://example\.org/_matrix/client/r0/sync\?access_token=.*' + r"^https://example\.org/_matrix/client/r0/sync\?access_token=.*" ) aioresponse.get( @@ -124,14 +114,16 @@ class TestClass(object): "type": "m.login.password", "user": "example", "password": "wordpass", - } + }, ) # Check that the pan client started to sync after logging in. pan_client = list(daemon.pan_clients.values())[0] assert len(pan_client.rooms) == 1 - async def test_pan_client_keys_upload(self, pan_proxy_server, aiohttp_client, aioresponse): + async def test_pan_client_keys_upload( + self, pan_proxy_server, aiohttp_client, aioresponse + ): server, daemon, _ = pan_proxy_server client = await aiohttp_client(server) @@ -140,11 +132,11 @@ class TestClass(object): "https://example.org/_matrix/client/r0/login", status=200, payload=self.login_response, - repeat=True + repeat=True, ) sync_url = re.compile( - r'^https://example\.org/_matrix/client/r0/sync\?access_token=.*' + r"^https://example\.org/_matrix/client/r0/sync\?access_token=.*" ) aioresponse.get( @@ -169,7 +161,7 @@ class TestClass(object): "type": "m.login.password", "user": "example", "password": "wordpass", - } + }, ) pan_client = list(daemon.pan_clients.values())[0] diff --git a/tests/store_test.py b/tests/store_test.py index 2f31587..2ecef85 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -1,14 +1,12 @@ import asyncio -import pdb import pprint import pytest from nio import RoomMessage, RoomEncryptedMedia from urllib.parse import urlparse -from conftest import faker from pantalaimon.index import INDEXING_ENABLED -from pantalaimon.store import FetchTask, MediaInfo +from pantalaimon.store import FetchTask, MediaInfo, UploadInfo TEST_ROOM = "!SVkFJHzfwvuaIEawgC:localhost" TEST_ROOM2 = "!testroom:localhost" @@ -27,7 +25,7 @@ class TestClass(object): "type": "m.room.message", "unsigned": {"age": 43289803095}, "user_id": "@example2:localhost", - "age": 43289803095 + "age": 43289803095, } ) @@ -43,43 +41,44 @@ class TestClass(object): "type": "m.room.message", "unsigned": {"age": 43289803095}, "user_id": "@example2:localhost", - "age": 43289803095 + "age": 43289803095, } ) @property def encrypted_media_event(self): - return RoomEncryptedMedia.from_dict({ - "room_id": "!testroom:localhost", - "event_id": "$15163622445EBvZK:localhost", - "origin_server_ts": 1516362244030, - "sender": "@example2:localhost", - "type": "m.room.message", - "content": { - "body": "orange_cat.jpg", - "msgtype": "m.image", - "file": { - "v": "v2", - "key": { - "alg": "A256CTR", - "ext": True, - "k": "yx0QvkgYlasdWEsdalkejaHBzCkKEBAp3tB7dGtWgrs", - "key_ops": ["encrypt", "decrypt"], - "kty": "oct" + return RoomEncryptedMedia.from_dict( + { + "room_id": "!testroom:localhost", + "event_id": "$15163622445EBvZK:localhost", + "origin_server_ts": 1516362244030, + "sender": "@example2:localhost", + "type": "m.room.message", + "content": { + "body": "orange_cat.jpg", + "msgtype": "m.image", + "file": { + "v": "v2", + "key": { + "alg": "A256CTR", + "ext": True, + "k": "yx0QvkgYlasdWEsdalkejaHBzCkKEBAp3tB7dGtWgrs", + "key_ops": ["encrypt", "decrypt"], + "kty": "oct", + }, + "iv": "0pglXX7fspIBBBBAEERLFd", + "hashes": { + "sha256": "eXRDFvh+aXsQRj8a+5ZVVWUQ9Y6u9DYiz4tq1NvbLu8" + }, + "url": "mxc://localhost/maDtasSiPFjROFMnlwxIhhyW", + "mimetype": "image/jpeg", }, - "iv": "0pglXX7fspIBBBBAEERLFd", - "hashes": { - "sha256": "eXRDFvh+aXsQRj8a+5ZVVWUQ9Y6u9DYiz4tq1NvbLu8" - }, - "url": "mxc://localhost/maDtasSiPFjROFMnlwxIhhyW", - "mimetype": "image/jpeg" - } + }, } - }) + ) def test_account_loading(self, panstore): accounts = panstore.load_all_users() - # pdb.set_trace() assert len(accounts) == 10 def test_token_saving(self, panstore, access_token): @@ -130,7 +129,8 @@ class TestClass(object): if not INDEXING_ENABLED: pytest.skip("Indexing needs to be enabled to test this") - from pantalaimon.index import Index, IndexStore + from pantalaimon.index import IndexStore + loop = asyncio.get_event_loop() store = IndexStore("example", tempdir) @@ -148,12 +148,14 @@ class TestClass(object): assert len(result["results"]) == 1 assert result["count"] == 1 assert result["results"][0]["result"] == self.test_event.source - assert (result["results"][0]["context"]["events_after"][0] - == self.another_event.source) + assert ( + result["results"][0]["context"]["events_after"][0] + == self.another_event.source + ) def test_media_storage(self, panstore): server_name = "test" - media_cache = panstore.load_media(server_name) + media_cache = panstore.load_media_cache(server_name) assert not media_cache event = self.encrypted_media_event @@ -171,9 +173,31 @@ class TestClass(object): panstore.save_media(server_name, media) - media_cache = panstore.load_media(server_name) + media_cache = panstore.load_media_cache(server_name) assert (mxc_server, mxc_path) in media_cache media_info = media_cache[(mxc_server, mxc_path)] assert media_info == media assert media_info == panstore.load_media(server_name, mxc_server, mxc_path) + + def test_upload_storage(self, panstore): + server_name = "test" + upload_cache = panstore.load_upload(server_name) + assert not upload_cache + + filename = "orange_cat.jpg" + mimetype = "image/jpeg" + event = self.encrypted_media_event + + assert not panstore.load_upload(server_name, event.url) + + upload = UploadInfo(event.url, filename, mimetype) + + panstore.save_upload(server_name, event.url, filename, mimetype) + + upload_cache = panstore.load_upload(server_name) + + assert (event.url) in upload_cache + upload_info = upload_cache[event.url] + assert upload_info == upload + assert upload_info == panstore.load_upload(server_name, event.url) diff --git a/tox.ini b/tox.ini index fb97e24..c19ce56 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,14 @@ -# content of: tox.ini , put in same dir as setup.py [tox] -envlist = py36,py37,coverage -[testenv] -basepython = - py36: python3.6 - py37: python3.7 - py3: python3.7 +envlist = coverage +[testenv] deps = -rtest-requirements.txt install_command = pip install {opts} {packages} -passenv = TOXENV CI TRAVIS TRAVIS_* +passenv = TOXENV,CI commands = pytest -usedevelop = True [testenv:coverage] -basepython = python3.7 commands = pytest --cov=pantalaimon --cov-report term-missing coverage xml @@ -25,5 +18,6 @@ deps = -rtest-requirements.txt coverage codecov>=1.4.0 + pytest-asyncio setenv = COVERAGE_FILE=.coverage