From db7de4d182e68d5ab0b6a1bd0802e0a692214214 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 4 Sep 2020 09:10:33 -0400 Subject: [PATCH 1/3] Fix a regression from calling read_templates. (#8252) Regressed in #8037. --- changelog.d/8252.feature | 1 + synapse/config/saml2_config.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/8252.feature diff --git a/changelog.d/8252.feature b/changelog.d/8252.feature new file mode 100644 index 000000000..7e69b7242 --- /dev/null +++ b/changelog.d/8252.feature @@ -0,0 +1 @@ +Use the default template file when its equivalent is not found in a custom template directory. diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py index 036f8c0e9..cc7401888 100644 --- a/synapse/config/saml2_config.py +++ b/synapse/config/saml2_config.py @@ -171,7 +171,7 @@ class SAML2Config(Config): self.saml2_error_html_template = self.read_templates( ["saml_error.html"], saml2_config.get("template_dir") - ) + )[0] def _default_saml_config_dict( self, required_attributes: set, optional_attributes: set From 041ee971c9b029dcad0dbfb55d4b9da5711d4cb0 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 4 Sep 2020 14:14:22 +0100 Subject: [PATCH 2/3] Unread counts fixes (#8254) * Fixup `ALTER TABLE` database queries Make the new columns nullable, because doing otherwise can wedge a server with a big database, as setting a default value rewrites the table. * Switch back to using the notifications count in the push badge Clients are likely to be confused if we send a push but the badge count is the unread messages one, and not the notifications one. * Changelog --- changelog.d/8254.feature | 1 + synapse/push/push_tools.py | 2 +- .../databases/main/schema/delta/58/15unread_count.sql | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/8254.feature diff --git a/changelog.d/8254.feature b/changelog.d/8254.feature new file mode 100644 index 000000000..feb02be23 --- /dev/null +++ b/changelog.d/8254.feature @@ -0,0 +1 @@ +Add unread messages count to sync responses, as specified in [MSC2654](https://github.com/matrix-org/matrix-doc/pull/2654). diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py index f7a25571f..d0145666b 100644 --- a/synapse/push/push_tools.py +++ b/synapse/push/push_tools.py @@ -36,7 +36,7 @@ async def get_badge_count(store, user_id): ) # return one badge count per conversation, as count per # message is so noisy as to be almost useless - badge += 1 if notifs["unread_count"] else 0 + badge += 1 if notifs["notify_count"] else 0 return badge diff --git a/synapse/storage/databases/main/schema/delta/58/15unread_count.sql b/synapse/storage/databases/main/schema/delta/58/15unread_count.sql index b451e8663..317fba8a5 100644 --- a/synapse/storage/databases/main/schema/delta/58/15unread_count.sql +++ b/synapse/storage/databases/main/schema/delta/58/15unread_count.sql @@ -19,8 +19,8 @@ -- Add columns to event_push_actions and event_push_actions_staging to track unread -- messages and calculate unread counts. -ALTER TABLE event_push_actions_staging ADD COLUMN unread SMALLINT NOT NULL DEFAULT 0; -ALTER TABLE event_push_actions ADD COLUMN unread SMALLINT NOT NULL DEFAULT 0; +ALTER TABLE event_push_actions_staging ADD COLUMN unread SMALLINT; +ALTER TABLE event_push_actions ADD COLUMN unread SMALLINT; -- Add column to event_push_summary -ALTER TABLE event_push_summary ADD COLUMN unread_count BIGINT NOT NULL DEFAULT 0; \ No newline at end of file +ALTER TABLE event_push_summary ADD COLUMN unread_count BIGINT; \ No newline at end of file From f25af1f9c72f1bf8645eca14fc2d8db13f51a9f3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 4 Sep 2020 15:06:05 +0100 Subject: [PATCH 3/3] Add cross-signing sigs to the `keys` object (#8234) All the callers want this info in the same place, so let's reduce the duplication by doing it here. --- changelog.d/8234.misc | 1 + synapse/storage/databases/main/devices.py | 12 ++---- .../storage/databases/main/end_to_end_keys.py | 39 +++++++------------ 3 files changed, 18 insertions(+), 34 deletions(-) create mode 100644 changelog.d/8234.misc diff --git a/changelog.d/8234.misc b/changelog.d/8234.misc new file mode 100644 index 000000000..979c8b227 --- /dev/null +++ b/changelog.d/8234.misc @@ -0,0 +1 @@ +Refactor queries for device keys and cross-signatures. diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index f8fe94812..add4e3ea0 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -291,15 +291,9 @@ class DeviceWorkerStore(SQLBaseStore): prev_id = stream_id if device is not None: - key_json = device.key_json - if key_json: - result["keys"] = db_to_json(key_json) - - if device.signatures: - for sig_user_id, sigs in device.signatures.items(): - result["keys"].setdefault("signatures", {}).setdefault( - sig_user_id, {} - ).update(sigs) + keys = device.keys + if keys: + result["keys"] = keys device_display_name = device.display_name if device_display_name: diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 09af03323..fba3098ea 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -43,12 +43,8 @@ class DeviceKeyLookupResult: # the key data from e2e_device_keys_json. Typically includes fields like # "algorithm", "keys" (including the curve25519 identity key and the ed25519 signing - # key) and "signatures" (a signature of the structure by the ed25519 key) - key_json = attr.ib(type=Optional[str]) - - # cross-signing sigs on this device. - # dict from (signing user_id)->(signing device_id)->sig - signatures = attr.ib(type=Optional[Dict[str, Dict[str, str]]], factory=dict) + # key) and "signatures" (a map from (user id) to (key id/device_id) to signature.) + keys = attr.ib(type=Optional[JsonDict]) class EndToEndKeyWorkerStore(SQLBaseStore): @@ -70,15 +66,9 @@ class EndToEndKeyWorkerStore(SQLBaseStore): for device_id, device in user_devices.items(): result = {"device_id": device_id} - key_json = device.key_json - if key_json: - result["keys"] = db_to_json(key_json) - - if device.signatures: - for sig_user_id, sigs in device.signatures.items(): - result["keys"].setdefault("signatures", {}).setdefault( - sig_user_id, {} - ).update(sigs) + keys = device.keys + if keys: + result["keys"] = keys device_display_name = device.display_name if device_display_name: @@ -114,16 +104,11 @@ class EndToEndKeyWorkerStore(SQLBaseStore): for user_id, device_keys in results.items(): rv[user_id] = {} for device_id, device_info in device_keys.items(): - r = db_to_json(device_info.key_json) + r = device_info.keys r["unsigned"] = {} display_name = device_info.display_name if display_name is not None: r["unsigned"]["device_display_name"] = display_name - if device_info.signatures: - for sig_user_id, sigs in device_info.signatures.items(): - r.setdefault("signatures", {}).setdefault( - sig_user_id, {} - ).update(sigs) rv[user_id][device_id] = r return rv @@ -140,6 +125,9 @@ class EndToEndKeyWorkerStore(SQLBaseStore): Any cross-signatures made on the keys by the owner of the device are also included. + The cross-signatures are added to the `signatures` field within the `keys` + object in the response. + Args: query_list: List of pairs of user_ids and device_ids. Device id can be None to indicate "all devices for this user" @@ -170,7 +158,7 @@ class EndToEndKeyWorkerStore(SQLBaseStore): (user_id, device_id) for user_id, dev in result.items() for device_id, d in dev.items() - if d is not None + if d is not None and d.keys is not None ) for batch in batch_iter(signature_query, 50): @@ -183,8 +171,9 @@ class EndToEndKeyWorkerStore(SQLBaseStore): # add each cross-signing signature to the correct device in the result dict. for (user_id, key_id, device_id, signature) in cross_sigs_result: target_device_result = result[user_id][device_id] - target_device_signatures = target_device_result.signatures - + target_device_signatures = target_device_result.keys.setdefault( + "signatures", {} + ) signing_user_signatures = target_device_signatures.setdefault( user_id, {} ) @@ -240,7 +229,7 @@ class EndToEndKeyWorkerStore(SQLBaseStore): if include_deleted_devices: deleted_devices.remove((user_id, device_id)) result.setdefault(user_id, {})[device_id] = DeviceKeyLookupResult( - display_name, key_json + display_name, db_to_json(key_json) if key_json else None ) if include_deleted_devices: