Merge remote-tracking branch 'upstream/release-v1.56'

CHANGES.md

@@ -1,3 +1,27 @@
+Synapse 1.56.0 (2022-04-05)
+===========================
+
+Synapse will now refuse to start up if open registration is enabled, in order to help mitigate
+abuse across the federation. If you would like
+to provide registration to anyone, consider adding [email](https://github.com/matrix-org/synapse/blob/8a519f8abc6de772167c2cca101d22ee2052fafc/docs/sample_config.yaml#L1285),
+[recaptcha](https://matrix-org.github.io/synapse/v1.56/CAPTCHA_SETUP.html)
+or [token-based](https://matrix-org.github.io/synapse/v1.56/usage/administration/admin_api/registration_tokens.html) verification
+in order to prevent automated registration from bad actors.
+This check can be disabled by setting the `enable_registration_without_verification` option in your
+homeserver configuration file to `true`. More details are available in the
+[upgrade notes](https://matrix-org.github.io/synapse/v1.56/upgrade.html#open-registration-without-verification-is-now-disabled-by-default).
+
+Synapse will additionally now refuse to start when using PostgreSQL with a non-`C` values for `COLLATE` and `CTYPE`, unless
+the config flag `allow_unsafe_locale`, found in the database section of the configuration file, is set to `true`. See the
+[upgrade notes](https://matrix-org.github.io/synapse/v1.56/upgrade#change-in-behaviour-for-postgresql-databases-with-unsafe-locale)
+for details.
+
+Internal Changes
+----------------
+
+- Bump the version of `black` for compatibility with the latest `click` release. ([\#12320](https://github.com/matrix-org/synapse/issues/12320))
+
+
 Synapse 1.56.0rc1 (2022-03-29)
 ==============================
 
@@ -33,7 +57,7 @@ Internal Changes
 - Rename `shared_rooms` to `mutual_rooms` ([MSC2666](https://github.com/matrix-org/matrix-doc/pull/2666)), as per proposal changes. ([\#12036](https://github.com/matrix-org/synapse/issues/12036))
 - Remove check on `update_user_directory` for shared rooms handler ([MSC2666](https://github.com/matrix-org/matrix-doc/pull/2666)), and update/expand documentation. ([\#12038](https://github.com/matrix-org/synapse/issues/12038))
 - Refactor `create_new_client_event` to use a new parameter, `state_event_ids`, which accurately describes the usage with [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) instead of abusing `auth_event_ids`. ([\#12083](https://github.com/matrix-org/synapse/issues/12083), [\#12304](https://github.com/matrix-org/synapse/issues/12304))
-- Refuse to start if registration is enabled without email, captcha, or token-based verification unless the new config flag `enable_registration_without_verification` is set. ([\#12091](https://github.com/matrix-org/synapse/issues/12091))
+- Refuse to start if registration is enabled without email, captcha, or token-based verification unless the new config flag `enable_registration_without_verification` is set to `true`. ([\#12091](https://github.com/matrix-org/synapse/issues/12091), [\#12322](https://github.com/matrix-org/synapse/issues/12322))
 - Add tests for database transaction callbacks. ([\#12198](https://github.com/matrix-org/synapse/issues/12198))
 - Handle cancellation in `DatabasePool.runInteraction`. ([\#12199](https://github.com/matrix-org/synapse/issues/12199))
 - Add missing type hints for cache storage. ([\#12216](https://github.com/matrix-org/synapse/issues/12216))

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,12 +193,15 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
+            ssrcMsg = (
-                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
-                "nick": self.userId,
+                % {
-                "assrc": self.ssrcs["audio"],
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                "vssrc": self.ssrcs["video"],
+                    "nick": self.userId,
-            }
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

debian/changelog

@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.56.0) stable; urgency=medium
+
+  * New synapse release 1.56.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Apr 2022 12:38:39 +0100
+
 matrix-synapse-py3 (1.56.0~rc1) stable; urgency=medium
 
   * New synapse release 1.56.0~rc1.

docs/upgrade.md

@@ -87,6 +87,11 @@ process, for example:
 
 # Upgrading to v1.56.0
 
+## Open registration without verification is now disabled by default
+
+Synapse will refuse to start if registration is enabled without email, captcha, or token-based verification unless the new config
+flag `enable_registration_without_verification` is set to "true".
+
 ## Groups/communities feature has been deprecated
 
 The non-standard groups/communities feature in Synapse has been deprecated and will
@@ -108,12 +113,6 @@ for more information and instructions on how to fix a database with incorrect va
 
 # Upgrading to v1.55.0
 
-## Open registration without verification is now disabled by default
-
-Synapse will refuse to start if registration is enabled without email, captcha, or token-based verification unless the new config 
-flag `enable_registration_without_verification` is set to "true".
-
-
 ## `synctl` script has been moved
 
 The `synctl` script

setup.py

@@ -95,7 +95,7 @@ CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS)
 # We pin black so that our tests don't start failing on new releases.
 CONDITIONAL_REQUIREMENTS["lint"] = [
     "isort==5.7.0",
-    "black==21.12b0",
+    "black==22.3.0",
     "flake8-comprehensions",
     "flake8-bugbear==21.3.2",
     "flake8",
@@ -128,7 +128,7 @@ CONDITIONAL_REQUIREMENTS["dev"] = (
     + CONDITIONAL_REQUIREMENTS["test"]
     + [
         # The following are used by the release script
-        "click==7.1.2",
+        "click==8.1.0",
         "redbaron==0.9.2",
         "GitPython==3.1.14",
         "commonmark==0.9.1",

synapse/__init__.py

@@ -68,7 +68,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.56.0rc1"
+__version__ = "1.56.0"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/api/constants.py

@@ -23,7 +23,7 @@ from typing_extensions import Final
 MAX_PDU_SIZE = 65536
 
 # the "depth" field on events is limited to 2**63 - 1
-MAX_DEPTH = 2 ** 63 - 1
+MAX_DEPTH = 2**63 - 1
 
 # the maximum length for a room alias is 255 characters
 MAX_ALIAS_LENGTH = 255

synapse/appservice/scheduler.py

@@ -428,7 +428,7 @@ class _Recoverer:
                 "as-recoverer-%s" % (self.service.id,), self.retry
             )
 
-        delay = 2 ** self.backoff_counter
+        delay = 2**self.backoff_counter
         logger.info("Scheduling retries on %s in %fs", self.service.id, delay)
         self.clock.call_later(delay, _retry)
 

synapse/crypto/keyring.py

@@ -182,7 +182,7 @@ class Keyring:
         vk = get_verify_key(hs.signing_key)
         self._local_verify_keys[f"{vk.alg}:{vk.version}"] = FetchKeyResult(
             verify_key=vk,
-            valid_until_ts=2 ** 63,  # fake future timestamp
+            valid_until_ts=2**63,  # fake future timestamp
         )
 
     async def verify_json_for_server(

synapse/events/utils.py

@@ -49,7 +49,7 @@ if TYPE_CHECKING:
 #       the literal fields "foo\" and "bar" but will instead be treated as "foo\\.bar"
 SPLIT_FIELD_REGEX = re.compile(r"(?<!\\)\.")
 
-CANONICALJSON_MAX_INT = (2 ** 53) - 1
+CANONICALJSON_MAX_INT = (2**53) - 1
 CANONICALJSON_MIN_INT = -CANONICALJSON_MAX_INT
 
 

synapse/http/proxyagent.py

@@ -41,7 +41,7 @@ from synapse.types import ISynapseReactor
 
 logger = logging.getLogger(__name__)
 
-_VALID_URI = re.compile(br"\A[\x21-\x7e]+\Z")
+_VALID_URI = re.compile(rb"\A[\x21-\x7e]+\Z")
 
 
 @implementer(IAgent)

synapse/python_dependencies.py

@@ -48,7 +48,7 @@ REQUIREMENTS = [
     "unpaddedbase64>=1.1.0",
     "canonicaljson>=1.4.0",
     # we use the type definitions added in signedjson 1.1.
-    "signedjson>=1.1.0",
+    "signedjson>=1.1.0,<=1.1.1",
     "pynacl>=1.2.1",
     "idna>=2.5",
     # validating SSL certs for IP addresses requires service_identity 18.1.

synapse/replication/http/_base.py

@@ -275,7 +275,7 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta):
                             if attempts > cls.RETRY_ON_CONNECT_ERROR_ATTEMPTS:
                                 raise
 
-                            delay = 2 ** attempts
+                            delay = 2**attempts
                             logger.warning(
                                 "%s request connection failed; retrying in %ds: %r",
                                 cls.NAME,

synapse/rest/media/v1/media_storage.py

@@ -352,7 +352,7 @@ class ReadableFileWrapper:
     `IConsumer`.
     """
 
-    CHUNK_SIZE = 2 ** 14
+    CHUNK_SIZE = 2**14
 
     clock: Clock
     path: str

synapse/rest/media/v1/preview_html.py

@@ -23,10 +23,10 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 _charset_match = re.compile(
-    br'<\s*meta[^>]*charset\s*=\s*"?([a-z0-9_-]+)"?', flags=re.I
+    rb'<\s*meta[^>]*charset\s*=\s*"?([a-z0-9_-]+)"?', flags=re.I
 )
 _xml_encoding_match = re.compile(
-    br'\s*<\s*\?\s*xml[^>]*encoding="([a-z0-9_-]+)"', flags=re.I
+    rb'\s*<\s*\?\s*xml[^>]*encoding="([a-z0-9_-]+)"', flags=re.I
 )
 _content_type_match = re.compile(r'.*; *charset="?(.*?)"?(;|$)', flags=re.I)
 

synapse/storage/database.py

@@ -63,7 +63,7 @@ if TYPE_CHECKING:
     from synapse.server import HomeServer
 
 # python 3 does not have a maximum int value
-MAX_TXN_ID = 2 ** 63 - 1
+MAX_TXN_ID = 2**63 - 1
 
 logger = logging.getLogger(__name__)
 

synapse/util/patch_inline_callbacks.py

@@ -217,13 +217,16 @@ def _check_yield_points(
                 # We don't raise here as its perfectly valid for contexts to
                 # change in a function, as long as it sets the correct context
                 # on resolving (which is checked separately).
-                err = "%s changed context from %s to %s, happened between lines %d and %d in %s" % (
+                err = (
-                    frame.f_code.co_name,
+                    "%s changed context from %s to %s, happened between lines %d and %d in %s"
-                    expected_context,
+                    % (
-                    current_context(),
+                        frame.f_code.co_name,
-                    last_yield_line_no,
+                        expected_context,
-                    frame.f_lineno,
+                        current_context(),
-                    frame.f_code.co_filename,
+                        last_yield_line_no,
+                        frame.f_lineno,
+                        frame.f_code.co_filename,
+                    )
                 )
                 changes.append(err)
 

synapse/util/retryutils.py

@@ -30,7 +30,7 @@ MIN_RETRY_INTERVAL = 10 * 60 * 1000
 RETRY_MULTIPLIER = 5
 
 # a cap on the backoff. (Essentially none)
-MAX_RETRY_INTERVAL = 2 ** 62
+MAX_RETRY_INTERVAL = 2**62
 
 
 class NotRetryingDestination(Exception):

tests/handlers/test_federation.py

@@ -496,8 +496,8 @@ class EventFromPduTestCase(TestCase):
     def test_invalid_numbers(self) -> None:
         """Invalid values for an integer should be rejected, all floats should be rejected."""
         for value in [
-            -(2 ** 53),
+            -(2**53),
-            2 ** 53,
+            2**53,
             1.0,
             float("inf"),
             float("-inf"),
@@ -524,7 +524,7 @@ class EventFromPduTestCase(TestCase):
             event_from_pdu_json(
                 {
                     "type": EventTypes.Message,
-                    "content": {"foo": [{"bar": 2 ** 56}]},
+                    "content": {"foo": [{"bar": 2**56}]},
                     "room_id": "!room:test",
                     "sender": "@user:test",
                     "depth": 1,

tests/replication/_base.py

@@ -206,7 +206,7 @@ class BaseStreamTestCase(unittest.HomeserverTestCase):
         path: bytes = request.path  # type: ignore
         self.assertRegex(
             path,
-            br"^/_synapse/replication/get_repl_stream_updates/%s/[^/]+$"
+            rb"^/_synapse/replication/get_repl_stream_updates/%s/[^/]+$"
             % (stream_name.encode("ascii"),),
         )