Merge branch 'matrix-org:master' into decrypt-initial-dm

This commit is contained in:
Igor Artemenko 2024-10-30 10:19:53 +00:00 committed by GitHub
commit a82652f6ad
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 165 additions and 128 deletions

View File

@ -25,6 +25,14 @@ Installing pantalaimon works like usually with python packages:
python setup.py install
or you can use `pip` and install it with:
```
pip install .[ui]
```
It is recommended that you create a virtual environment first or install dependencies
via your package manager. They are usually found with `python-<package-name>`.
Pantalaimon can also be found on pypi:
pip install pantalaimon
@ -111,7 +119,7 @@ specifies one or more homeservers for pantalaimon to connect to.
A minimal pantalaimon configuration looks like this:
```dosini
[local-matrix]
Homeserver = https://localhost:8448
Homeserver = https://localhost:443
ListenAddress = localhost
ListenPort = 8009
```
@ -140,3 +148,41 @@ To control the daemon an interactive utility is provided in the form of
`panctl` can be used to verify, blacklist or ignore devices, import or export
session keys, or to introspect devices of users that we share encrypted rooms
with.
### Setup
This is all coming from an excellent comment that you can find [here](https://github.com/matrix-org/pantalaimon/issues/154#issuecomment-1951591191).
1) Ensure you have an OS keyring installed. In my case I installed `gnome-keyring`. You may also want a GUI like `seahorse` to inspect the keyring. (pantalaimon will work without a keyring but your client will have to log in with the password every time `pantalaimon` is restarted, instead of being able to reuse the access token from the previous successful login.)
2) In case you have prior attempts, clean the slate by deleting the `~/.local/share/pantalaimon` directory.
3) Start `pantalaimon`.
4) Connect a client to the `ListenAddress:ListenPort` you specified in `pantalaimon.conf`, eg to `127.0.0.1:8009`, using the same username and password you would've used to login to your homeserver directly.
5) The login should succeed, but at this point all encrypted messages will fail to decrypt. This is fine.
6) Start another client that you were already using for your encrypted chats previously. In my case this was `app.element.io`, so the rest of the steps here assume that.
7) Run `panctl`. At the prompt, run `start-verification <user ID> <user ID> <Element's device ID>`. `<user ID>` here is the full user ID like `@arnavion:arnavion.dev`. If you only have the one Element session, `panctl` will show you the device ID as an autocomplete hint so you don't have to look it up. If you do need to look it up, go to Element -> profile icon -> All Settings -> Sessions, expand the "Current session" item, and the "Session ID" is the device ID.
8) In Element you will see a popup "Incoming Verification Request". Click "Continue". It will change to a popup containing some emojis, and `panctl` will print the same emojis. Click the "They match" button. It will now change to a popup like "Waiting for other client to confirm..."
9) In `panctl`, run `confirm-verification <user ID> <user ID> <Element's device ID>`, ie the same command as before but with `confirm-verification` instead of `start-verification`.
10) At this point, if you look at all your sessions in Element (profile icon -> All Settings -> Sessions), you should see "pantalaimon" in the "Other sessions" list as a "Verified" session.
11) Export the E2E room keys that Element was using via profile icon -> Security & Privacy -> Export E2E room keys. Pick any password and then save the file to some path.
12) Back in `panctl`, run `import-keys <user ID> <path of file> <password you used to encrypt the file>`. After a few seconds, in the output of `pantalaimon`, you should see a log like `INFO: pantalaimon: Successfully imported keys for <user ID> from <path of file>`.
13) Close and restart the client you had used in step 5, ie the one you want to connect to `pantalaimon`. Now, finally, you should be able to see the encrypted chats be decrypted.
14) Delete the E2E room keys backup file from step 12. You don't need it any more.
15) If in step 11 you had other unverified sessions from pantalaimon from your prior attempts, you can sign out of them too.
You will probably have to repeat steps 11-14 any time you start a new encrypted chat in Element.

View File

@ -16,7 +16,6 @@ import asyncio
import os
from collections import defaultdict
from pprint import pformat
from typing import Any, Dict, Optional
from urllib.parse import urlparse
from aiohttp.client_exceptions import ClientConnectionError
@ -135,7 +134,7 @@ class InvalidLimit(Exception):
class SqliteQStore(SqliteStore):
def _create_database(self):
return SqliteQueueDatabase(
self.database_path, pragmas=(("foregign_keys", 1), ("secure_delete", 1))
self.database_path, pragmas=(("foreign_keys", 1), ("secure_delete", 1))
)
def close(self):
@ -554,6 +553,7 @@ class PanClient(AsyncClient):
full_state=True,
since=next_batch,
loop_sleep_time=loop_sleep_time,
set_presence="offline",
)
)
self.task = task
@ -708,7 +708,6 @@ class PanClient(AsyncClient):
for share in self.get_active_key_requests(
message.user_id, message.device_id
):
continued = True
if not self.continue_key_share(share):
@ -810,8 +809,9 @@ class PanClient(AsyncClient):
if not isinstance(event, MegolmEvent):
logger.warn(
"Encrypted event is not a megolm event:"
"\n{}".format(pformat(event_dict))
"Encrypted event is not a megolm event:" "\n{}".format(
pformat(event_dict)
)
)
return False
@ -835,9 +835,9 @@ class PanClient(AsyncClient):
decrypted_event.source["content"]["url"] = decrypted_event.url
if decrypted_event.thumbnail_url:
decrypted_event.source["content"]["info"][
"thumbnail_url"
] = decrypted_event.thumbnail_url
decrypted_event.source["content"]["info"]["thumbnail_url"] = (
decrypted_event.thumbnail_url
)
event_dict.update(decrypted_event.source)
event_dict["decrypted"] = True

View File

@ -186,7 +186,6 @@ class PanConfig:
try:
for section_name, section in config.items():
if section_name == "Default":
continue

View File

@ -227,7 +227,8 @@ class ProxyDaemon:
if ret:
msg = (
f"Device {device.id} of user " f"{device.user_id} successfully verified."
f"Device {device.id} of user "
f"{device.user_id} successfully verified."
)
await client.send_update_device(device)
else:
@ -309,7 +310,6 @@ class ProxyDaemon:
DeviceUnblacklistMessage,
),
):
device = client.device_store[message.user_id].get(message.device_id, None)
if not device:
@ -616,7 +616,9 @@ class ProxyDaemon:
await pan_client.close()
return
logger.info(f"Successfully started new background sync client for " f"{user_id}")
logger.info(
f"Successfully started new background sync client for " f"{user_id}"
)
await self.send_ui_message(
UpdateUsersMessage(self.name, user_id, pan_client.device_id)
@ -733,7 +735,7 @@ class ProxyDaemon:
return decryption_method(body, ignore_failures=False)
except EncryptionError:
logger.info("Error decrypting sync, waiting for next pan " "sync")
await client.synced.wait(),
(await client.synced.wait(),)
logger.info("Pan synced, retrying decryption.")
try:
@ -1294,7 +1296,9 @@ class ProxyDaemon:
client = next(iter(self.pan_clients.values()))
try:
response = await client.download(server_name, media_id, file_name)
response = await client.download(
server_name=server_name, media_id=media_id, filename=file_name
)
except ClientConnectionError as e:
raise e

View File

@ -23,7 +23,6 @@ if False:
import json
import os
from functools import partial
from typing import Any, Dict, List, Optional, Tuple
import attr
import tantivy
@ -230,7 +229,6 @@ if False:
)
for message in query:
event = message.event
event_dict = {

View File

@ -15,7 +15,6 @@
import asyncio
import os
import signal
from typing import Optional
import click
import janus

View File

@ -34,7 +34,7 @@ from prompt_toolkit import HTML, PromptSession, print_formatted_text
from prompt_toolkit.completion import Completer, Completion, PathCompleter
from prompt_toolkit.document import Document
from prompt_toolkit.patch_stdout import patch_stdout
from pydbus import SessionBus
from dasbus.connection import SessionMessageBus
PTK2 = ptk_version.startswith("2.")
@ -404,8 +404,8 @@ class PanCtl:
commands = list(command_help.keys())
def __attrs_post_init__(self):
self.bus = SessionBus()
self.pan_bus = self.bus.get("org.pantalaimon1")
self.bus = SessionMessageBus()
self.pan_bus = self.bus.get_connection("org.pantalaimon1")
self.ctl = self.pan_bus["org.pantalaimon1.control"]
self.devices = self.pan_bus["org.pantalaimon1.devices"]

View File

@ -15,7 +15,7 @@
import json
import os
from collections import defaultdict
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict
import attr
from nio.crypto import TrustState, GroupSessionStore
@ -431,7 +431,6 @@ class PanStore:
device_store = defaultdict(dict)
for d in account.device_keys:
if d.deleted:
continue

View File

@ -17,7 +17,7 @@ from importlib import util
UI_ENABLED = (
util.find_spec("gi") is not None
and util.find_spec("gi.repository") is not None
and util.find_spec("pydbus") is not None
and util.find_spec("dasbus") is not None
)
if UI_ENABLED:
@ -28,8 +28,8 @@ if UI_ENABLED:
import dbus
import notify2
from gi.repository import GLib
from pydbus import SessionBus
from pydbus.generic import signal
from dasbus import SessionMessageBus
from dasbus.signal import Signal
from dbus.mainloop.glib import DBusGMainLoop
from nio import RoomKeyRequest, RoomKeyRequestCancellation
@ -123,8 +123,8 @@ if UI_ENABLED:
</node>
"""
Response = signal()
UnverifiedDevices = signal()
Response = Signal()
UnverifiedDevices = Signal()
def __init__(self, queue, server_list, id_counter):
self.queue = queue
@ -297,13 +297,13 @@ if UI_ENABLED:
</node>
"""
VerificationInvite = signal()
VerificationCancel = signal()
VerificationString = signal()
VerificationDone = signal()
VerificationInvite = Signal()
VerificationCancel = Signal()
VerificationString = Signal()
VerificationDone = Signal()
KeyRequest = signal()
KeyRequestCancel = signal()
KeyRequest = Signal()
KeyRequestCancel = Signal()
def __init__(self, queue, id_counter):
self.device_list = dict()
@ -466,8 +466,8 @@ if UI_ENABLED:
self.control_if = Control(self.send_queue, self.server_list, id_counter)
self.device_if = Devices(self.send_queue, id_counter)
self.bus = SessionBus()
self.bus.publish("org.pantalaimon1", self.control_if, self.device_if)
self.bus = SessionMessageBus()
self.bus.publish_object("org.pantalaimon1", self.control_if, self.device_if)
def unverified_notification(self, message):
notification = notify2.Notification(

View File

@ -11,8 +11,7 @@ setup(
url="https://github.com/matrix-org/pantalaimon",
author="The Matrix.org Team",
author_email="poljar@termina.org.uk",
description=("A Matrix proxy daemon that adds E2E encryption "
"capabilities."),
description=("A Matrix proxy daemon that adds E2E encryption " "capabilities."),
long_description=long_description,
long_description_content_type="text/markdown",
license="Apache License, Version 2.0",
@ -29,19 +28,21 @@ setup(
"cachetools >= 3.0.0",
"prompt_toolkit > 2, < 4",
"typing;python_version<'3.5'",
"matrix-nio[e2e] >= 0.20, < 0.21"
"matrix-nio[e2e] >= 0.24, < 0.25.2",
],
extras_require={
"ui": [
"dbus-python >= 1.2, < 1.3",
"PyGObject >= 3.36, < 3.39",
"pydbus >= 0.6, < 0.7",
"dasbus == 1.7",
"notify2 >= 0.3, < 0.4",
]
},
entry_points={
"console_scripts": ["pantalaimon=pantalaimon.main:main",
"panctl=pantalaimon.panctl:main"],
"console_scripts": [
"pantalaimon=pantalaimon.main:main",
"panctl=pantalaimon.panctl:main",
],
},
zip_safe=False
zip_safe=False,
)

View File

@ -34,11 +34,9 @@ class Provider(BaseProvider):
def client(self):
return ClientInfo(faker.mx_id(), faker.access_token())
def avatar_url(self):
return "mxc://{}/{}#auto".format(
faker.hostname(),
"".join(choices(ascii_letters) for i in range(24))
faker.hostname(), "".join(choices(ascii_letters) for i in range(24))
)
def olm_key_pair(self):
@ -56,7 +54,6 @@ class Provider(BaseProvider):
)
faker.add_provider(Provider)
@ -80,13 +77,7 @@ def tempdir():
@pytest.fixture
def panstore(tempdir):
for _ in range(10):
store = SqliteStore(
faker.mx_id(),
faker.device_id(),
tempdir,
"",
"pan.db"
)
store = SqliteStore(faker.mx_id(), faker.device_id(), tempdir, "", "pan.db")
account = OlmAccount()
store.save_account(account)
@ -130,21 +121,23 @@ async def pan_proxy_server(tempdir, aiohttp_server):
recv_queue=ui_queue.async_q,
proxy=None,
ssl=False,
client_store_class=SqliteStore
client_store_class=SqliteStore,
)
app.add_routes([
web.post("/_matrix/client/r0/login", proxy.login),
web.get("/_matrix/client/r0/sync", proxy.sync),
web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages),
web.put(
r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}",
proxy.send_message
),
web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter),
web.post("/_matrix/client/r0/search", proxy.search),
web.options("/_matrix/client/r0/search", proxy.search_opts),
])
app.add_routes(
[
web.post("/_matrix/client/r0/login", proxy.login),
web.get("/_matrix/client/r0/sync", proxy.sync),
web.get("/_matrix/client/r0/rooms/{room_id}/messages", proxy.messages),
web.put(
r"/_matrix/client/r0/rooms/{room_id}/send/{event_type}/{txnid}",
proxy.send_message,
),
web.post("/_matrix/client/r0/user/{user_id}/filter", proxy.filter),
web.post("/_matrix/client/r0/search", proxy.search),
web.options("/_matrix/client/r0/search", proxy.search_opts),
]
)
server = await aiohttp_server(app)
@ -161,7 +154,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client):
"access_token": "abc123",
"device_id": "GHTYAJCE",
"home_server": "example.org",
"user_id": "@example:example.org"
"user_id": "@example:example.org",
}
aioclient = await aiohttp_client(server)
@ -170,7 +163,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client):
"https://example.org/_matrix/client/r0/login",
status=200,
payload=login_response,
repeat=True
repeat=True,
)
await aioclient.post(
@ -179,7 +172,7 @@ async def running_proxy(pan_proxy_server, aioresponse, aiohttp_client):
"type": "m.login.password",
"user": "example",
"password": "wordpass",
}
},
)
yield server, aioclient, proxy, queues

View File

@ -380,7 +380,9 @@ class TestClass(object):
)
aioresponse.get(
sync_url, status=200, payload=self.initial_sync_response,
sync_url,
status=200,
payload=self.initial_sync_response,
)
aioresponse.get(sync_url, status=200, payload=self.empty_sync, repeat=True)
@ -454,7 +456,9 @@ class TestClass(object):
)
aioresponse.get(
sync_url, status=200, payload=self.initial_sync_response,
sync_url,
status=200,
payload=self.initial_sync_response,
)
aioresponse.get(sync_url, status=200, payload=self.empty_sync, repeat=True)

View File

@ -1,9 +1,7 @@
import asyncio
import json
import re
from collections import defaultdict
from aiohttp import web
from nio.crypto import OlmDevice
from conftest import faker
@ -27,7 +25,7 @@ class TestClass(object):
"access_token": "abc123",
"device_id": "GHTYAJCE",
"home_server": "example.org",
"user_id": "@example:example.org"
"user_id": "@example:example.org",
}
@property
@ -36,12 +34,7 @@ class TestClass(object):
@property
def keys_upload_response(self):
return {
"one_time_key_counts": {
"curve25519": 10,
"signed_curve25519": 20
}
}
return {"one_time_key_counts": {"curve25519": 10, "signed_curve25519": 20}}
@property
def example_devices(self):
@ -52,10 +45,7 @@ class TestClass(object):
devices[device.user_id][device.id] = device
bob_device = OlmDevice(
BOB_ID,
BOB_DEVICE,
{"ed25519": BOB_ONETIME,
"curve25519": BOB_CURVE}
BOB_ID, BOB_DEVICE, {"ed25519": BOB_ONETIME, "curve25519": BOB_CURVE}
)
devices[BOB_ID][BOB_DEVICE] = bob_device
@ -71,7 +61,7 @@ class TestClass(object):
"https://example.org/_matrix/client/r0/login",
status=200,
payload=self.login_response,
repeat=True
repeat=True,
)
assert not daemon.pan_clients
@ -82,7 +72,7 @@ class TestClass(object):
"type": "m.login.password",
"user": "example",
"password": "wordpass",
}
},
)
assert resp.status == 200
@ -105,11 +95,11 @@ class TestClass(object):
"https://example.org/_matrix/client/r0/login",
status=200,
payload=self.login_response,
repeat=True
repeat=True,
)
sync_url = re.compile(
r'^https://example\.org/_matrix/client/r0/sync\?access_token=.*'
r"^https://example\.org/_matrix/client/r0/sync\?access_token=.*"
)
aioresponse.get(
@ -124,14 +114,16 @@ class TestClass(object):
"type": "m.login.password",
"user": "example",
"password": "wordpass",
}
},
)
# Check that the pan client started to sync after logging in.
pan_client = list(daemon.pan_clients.values())[0]
assert len(pan_client.rooms) == 1
async def test_pan_client_keys_upload(self, pan_proxy_server, aiohttp_client, aioresponse):
async def test_pan_client_keys_upload(
self, pan_proxy_server, aiohttp_client, aioresponse
):
server, daemon, _ = pan_proxy_server
client = await aiohttp_client(server)
@ -140,11 +132,11 @@ class TestClass(object):
"https://example.org/_matrix/client/r0/login",
status=200,
payload=self.login_response,
repeat=True
repeat=True,
)
sync_url = re.compile(
r'^https://example\.org/_matrix/client/r0/sync\?access_token=.*'
r"^https://example\.org/_matrix/client/r0/sync\?access_token=.*"
)
aioresponse.get(
@ -169,7 +161,7 @@ class TestClass(object):
"type": "m.login.password",
"user": "example",
"password": "wordpass",
}
},
)
pan_client = list(daemon.pan_clients.values())[0]

View File

@ -1,12 +1,10 @@
import asyncio
import pdb
import pprint
import pytest
from nio import RoomMessage, RoomEncryptedMedia
from urllib.parse import urlparse
from conftest import faker
from pantalaimon.index import INDEXING_ENABLED
from pantalaimon.store import FetchTask, MediaInfo, UploadInfo
@ -27,7 +25,7 @@ class TestClass(object):
"type": "m.room.message",
"unsigned": {"age": 43289803095},
"user_id": "@example2:localhost",
"age": 43289803095
"age": 43289803095,
}
)
@ -43,43 +41,44 @@ class TestClass(object):
"type": "m.room.message",
"unsigned": {"age": 43289803095},
"user_id": "@example2:localhost",
"age": 43289803095
"age": 43289803095,
}
)
@property
def encrypted_media_event(self):
return RoomEncryptedMedia.from_dict({
"room_id": "!testroom:localhost",
"event_id": "$15163622445EBvZK:localhost",
"origin_server_ts": 1516362244030,
"sender": "@example2:localhost",
"type": "m.room.message",
"content": {
"body": "orange_cat.jpg",
"msgtype": "m.image",
"file": {
"v": "v2",
"key": {
"alg": "A256CTR",
"ext": True,
"k": "yx0QvkgYlasdWEsdalkejaHBzCkKEBAp3tB7dGtWgrs",
"key_ops": ["encrypt", "decrypt"],
"kty": "oct"
return RoomEncryptedMedia.from_dict(
{
"room_id": "!testroom:localhost",
"event_id": "$15163622445EBvZK:localhost",
"origin_server_ts": 1516362244030,
"sender": "@example2:localhost",
"type": "m.room.message",
"content": {
"body": "orange_cat.jpg",
"msgtype": "m.image",
"file": {
"v": "v2",
"key": {
"alg": "A256CTR",
"ext": True,
"k": "yx0QvkgYlasdWEsdalkejaHBzCkKEBAp3tB7dGtWgrs",
"key_ops": ["encrypt", "decrypt"],
"kty": "oct",
},
"iv": "0pglXX7fspIBBBBAEERLFd",
"hashes": {
"sha256": "eXRDFvh+aXsQRj8a+5ZVVWUQ9Y6u9DYiz4tq1NvbLu8"
},
"url": "mxc://localhost/maDtasSiPFjROFMnlwxIhhyW",
"mimetype": "image/jpeg",
},
"iv": "0pglXX7fspIBBBBAEERLFd",
"hashes": {
"sha256": "eXRDFvh+aXsQRj8a+5ZVVWUQ9Y6u9DYiz4tq1NvbLu8"
},
"url": "mxc://localhost/maDtasSiPFjROFMnlwxIhhyW",
"mimetype": "image/jpeg"
}
},
}
})
)
def test_account_loading(self, panstore):
accounts = panstore.load_all_users()
# pdb.set_trace()
assert len(accounts) == 10
def test_token_saving(self, panstore, access_token):
@ -130,7 +129,8 @@ class TestClass(object):
if not INDEXING_ENABLED:
pytest.skip("Indexing needs to be enabled to test this")
from pantalaimon.index import Index, IndexStore
from pantalaimon.index import IndexStore
loop = asyncio.get_event_loop()
store = IndexStore("example", tempdir)
@ -148,8 +148,10 @@ class TestClass(object):
assert len(result["results"]) == 1
assert result["count"] == 1
assert result["results"][0]["result"] == self.test_event.source
assert (result["results"][0]["context"]["events_after"][0]
== self.another_event.source)
assert (
result["results"][0]["context"]["events_after"][0]
== self.another_event.source
)
def test_media_storage(self, panstore):
server_name = "test"

View File

@ -5,7 +5,7 @@ envlist = coverage
deps = -rtest-requirements.txt
install_command = pip install {opts} {packages}
passenv = TOXENV CI
passenv = TOXENV,CI
commands = pytest
[testenv:coverage]