Synapse 1.39.0rc3 (2021-07-28)

==============================
 
 Bugfixes
 --------
 
 - Fix a bug introduced in Synapse 1.38 which caused an exception at startup when SAML authentication was enabled. ([\#10477](https://github.com/matrix-org/synapse/issues/10477))
 - Fix a long-standing bug where Synapse would not inform clients that a device had exhausted its one-time-key pool, potentially causing problems decrypting events. ([\#10485](https://github.com/matrix-org/synapse/issues/10485))
 - Fix reporting old R30 stats as R30v2 stats. Introduced in v1.39.0rc1. ([\#10486](https://github.com/matrix-org/synapse/issues/10486))
 
 Internal Changes
 ----------------
 
 - Fix an error which prevented the Github Actions workflow to build the docker images from running. ([\#10461](https://github.com/matrix-org/synapse/issues/10461))
 - Fix release script to correctly version debian changelog when doing RCs. ([\#10465](https://github.com/matrix-org/synapse/issues/10465))
 -----BEGIN PGP SIGNATURE-----
 
 iQFEBAABCgAuFiEEBTGR3/RnAzBGUif3pULk7RsPrAkFAmEBUJUQHGVyaWtAbWF0
 cml4Lm9yZwAKCRClQuTtGw+sCWi6CACfA3B7rXfQIO893mctSZkNhtAG/e4o310M
 etvjAtovKmKeFqBvm89FPmFoXvGA2ypoz7Jz2fdkP56DNKIXvihnDc8EeQ3gGawf
 hQ8GBjICOL1e7R/4qzuOCWYdppBGoGNAbz7qyxAUxZ/XOzsPJ2VBGkgyXWZkyPe/
 nJYsuMuMp117HUwTqPqs3oZuTN0MRTy6xgaDfbFbcX9UC/amLmGIWOhlme8iV15r
 HQAW0X7F2Un5h7eQwogWTDb9gBHKnJ4ApaSJLfZl9E72i0Sv7o9UFcLIdy2DRVSq
 gxlRBVzVU6ZndCQ4vQXFSjQ20VGNXc2vpKVRE1D6YXxmxVNbBQB4
 =lb6H
 -----END PGP SIGNATURE-----

Merge tag 'v1.39.0rc3' into develop

Synapse 1.39.0rc3 (2021-07-28)
==============================

Bugfixes
--------

- Fix a bug introduced in Synapse 1.38 which caused an exception at startup when SAML authentication was enabled. ([\#10477](https://github.com/matrix-org/synapse/issues/10477))
- Fix a long-standing bug where Synapse would not inform clients that a device had exhausted its one-time-key pool, potentially causing problems decrypting events. ([\#10485](https://github.com/matrix-org/synapse/issues/10485))
- Fix reporting old R30 stats as R30v2 stats. Introduced in v1.39.0rc1. ([\#10486](https://github.com/matrix-org/synapse/issues/10486))

Internal Changes
----------------

- Fix an error which prevented the Github Actions workflow to build the docker images from running. ([\#10461](https://github.com/matrix-org/synapse/issues/10461))
- Fix release script to correctly version debian changelog when doing RCs. ([\#10465](https://github.com/matrix-org/synapse/issues/10465))

CHANGES.md

@@ -1,3 +1,21 @@
+Synapse 1.39.0rc3 (2021-07-28)
+==============================
+
+Bugfixes
+--------
+
+- Fix a bug introduced in Synapse 1.38 which caused an exception at startup when SAML authentication was enabled. ([\#10477](https://github.com/matrix-org/synapse/issues/10477))
+- Fix a long-standing bug where Synapse would not inform clients that a device had exhausted its one-time-key pool, potentially causing problems decrypting events. ([\#10485](https://github.com/matrix-org/synapse/issues/10485))
+- Fix reporting old R30 stats as R30v2 stats. Introduced in v1.39.0rc1. ([\#10486](https://github.com/matrix-org/synapse/issues/10486))
+
+
+Internal Changes
+----------------
+
+- Fix an error which prevented the Github Actions workflow to build the docker images from running. ([\#10461](https://github.com/matrix-org/synapse/issues/10461))
+- Fix release script to correctly version debian changelog when doing RCs. ([\#10465](https://github.com/matrix-org/synapse/issues/10465))
+
+
 Synapse 1.39.0rc2 (2021-07-22)
 ==============================
 

changelog.d/10461.misc

@@ -1 +0,0 @@
-Fix an error which prevented the Github Actions workflow to build the docker images from running.

debian/changelog

@@ -4,6 +4,12 @@ matrix-synapse-py3 (1.39.0ubuntu1) UNRELEASED; urgency=medium
 
  -- Richard van der Hoff <richard@matrix.org>  Tue, 20 Jul 2021 00:10:03 +0100
 
+matrix-synapse-py3 (1.39.0~rc3) stable; urgency=medium
+
+  * New synapse release 1.39.0~rc3.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 28 Jul 2021 13:30:58 +0100
+
 matrix-synapse-py3 (1.38.1) stable; urgency=medium
 
   * New synapse release 1.38.1.

scripts-dev/release.py

@@ -139,6 +139,11 @@ def run():
 
     # Switch to the release branch.
     parsed_new_version = version.parse(new_version)
+
+    # We assume for debian changelogs that we only do RCs or full releases.
+    assert not parsed_new_version.is_devrelease
+    assert not parsed_new_version.is_postrelease
+
     release_branch_name = (
         f"release-v{parsed_new_version.major}.{parsed_new_version.minor}"
     )
@@ -190,10 +195,19 @@ def run():
     # Generate changelogs
     subprocess.run("python3 -m towncrier", shell=True)
 
-    # Generate debian changelogs if its not an RC.
+    # Generate debian changelogs
-    if not rc:
+    if parsed_new_version.pre is not None:
+        # If this is an RC then we need to coerce the version string to match
+        # Debian norms, e.g. 1.39.0rc2 gets converted to 1.39.0~rc2.
+        base_ver = parsed_new_version.base_version
+        pre_type, pre_num = parsed_new_version.pre
+        debian_version = f"{base_ver}~{pre_type}{pre_num}"
+    else:
+        debian_version = new_version
+
     subprocess.run(
-            f'dch -M -v {new_version} "New synapse release {new_version}."', shell=True
+        f'dch -M -v {debian_version} "New synapse release {debian_version}."',
+        shell=True,
     )
     subprocess.run('dch -M -r -D stable ""', shell=True)
 

synapse/__init__.py

@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.39.0rc2"
+__version__ = "1.39.0rc3"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/api/constants.py

@@ -128,6 +128,14 @@ class ToDeviceEventTypes:
     RoomKeyRequest = "m.room_key_request"
 
 
+class DeviceKeyAlgorithms:
+    """Spec'd algorithms for the generation of per-device keys"""
+
+    ED25519 = "ed25519"
+    CURVE25519 = "curve25519"
+    SIGNED_CURVE25519 = "signed_curve25519"
+
+
 class EduTypes:
     Presence = "m.presence"
 

synapse/app/phone_stats_home.py

@@ -109,7 +109,7 @@ async def phone_stats_home(hs, stats, stats_process=_stats_process):
     for name, count in r30_results.items():
         stats["r30_users_" + name] = count
 
-    r30v2_results = await store.count_r30_users()
+    r30v2_results = await store.count_r30v2_users()
     for name, count in r30v2_results.items():
         stats["r30v2_users_" + name] = count
 

synapse/handlers/_base.py

@@ -15,8 +15,6 @@
 import logging
 from typing import TYPE_CHECKING, Optional
 
-import synapse.state
-import synapse.storage
 import synapse.types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.ratelimiting import Ratelimiter

synapse/handlers/sync.py

@@ -1093,6 +1093,10 @@ class SyncHandler:
         one_time_key_counts: JsonDict = {}
         unused_fallback_key_types: List[str] = []
         if device_id:
+            # TODO: We should have a way to let clients differentiate between the states of:
+            #   * no change in OTK count since the provided since token
+            #   * the server has zero OTKs left for this device
+            #  Spec issue: https://github.com/matrix-org/matrix-doc/issues/3298
             one_time_key_counts = await self.store.count_e2e_one_time_keys(
                 user_id, device_id
             )

synapse/storage/databases/main/end_to_end_keys.py

@@ -21,6 +21,7 @@ from canonicaljson import encode_canonical_json
 
 from twisted.enterprise.adbapi import Connection
 
+from synapse.api.constants import DeviceKeyAlgorithms
 from synapse.logging.opentracing import log_kv, set_tag, trace
 from synapse.storage._base import SQLBaseStore, db_to_json
 from synapse.storage.database import DatabasePool, make_in_list_sql_clause
@@ -381,9 +382,15 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore):
                 " GROUP BY algorithm"
             )
             txn.execute(sql, (user_id, device_id))
-            result = {}
+
+            # Initially set the key count to 0. This ensures that the client will always
+            # receive *some count*, even if it's 0.
+            result = {DeviceKeyAlgorithms.SIGNED_CURVE25519: 0}
+
+            # Override entries with the count of any keys we pulled from the database
             for algorithm, key_count in txn:
                 result[algorithm] = key_count
+
             return result
 
         return await self.db_pool.runInteraction(

tests/handlers/test_e2e_keys.py

@@ -47,12 +47,16 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
             "alg2:k3": {"key": "key3"},
         }
 
+        # Note that "signed_curve25519" is always returned in key count responses. This is necessary until
+        # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed.
         res = self.get_success(
             self.handler.upload_keys_for_user(
                 local_user, device_id, {"one_time_keys": keys}
             )
         )
-        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
+        self.assertDictEqual(
+            res, {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}
+        )
 
         # we should be able to change the signature without a problem
         keys["alg2:k2"]["signatures"]["k1"] = "sig2"
@@ -61,7 +65,9 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
                 local_user, device_id, {"one_time_keys": keys}
             )
         )
-        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
+        self.assertDictEqual(
+            res, {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}
+        )
 
     def test_change_one_time_keys(self):
         """attempts to change one-time-keys should be rejected"""
@@ -79,7 +85,9 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
                 local_user, device_id, {"one_time_keys": keys}
             )
         )
-        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
+        self.assertDictEqual(
+            res, {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}
+        )
 
         # Error when changing string key
         self.get_failure(
@@ -89,7 +97,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
             SynapseError,
         )
 
-        # Error when replacing dict key with strin
+        # Error when replacing dict key with string
         self.get_failure(
             self.handler.upload_keys_for_user(
                 local_user, device_id, {"one_time_keys": {"alg2:k3": "key2"}}
@@ -131,7 +139,9 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
                 local_user, device_id, {"one_time_keys": keys}
             )
         )
-        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1}})
+        self.assertDictEqual(
+            res, {"one_time_key_counts": {"alg1": 1, "signed_curve25519": 0}}
+        )
 
         res2 = self.get_success(
             self.handler.claim_one_time_keys(