From 76aa5537ad4f41cad130862a230c1f4cc4bfcbcf Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 20 Dec 2021 16:33:35 +0000
Subject: [PATCH 1/8] Disable aggregation bundling on `/sync` responses
 (#11583)

* Disable aggregation bundling on `/sync` responses

A partial revert of #11478. This turns out to have had a significant CPU impact
on initial-sync handling. For now, let's disable it, until we find a more
efficient way of achieving this.

* Fix tests.

Co-authored-by: Patrick Cloke <patrickc@matrix.org>
---
 changelog.d/11583.bugfix            |  1 +
 synapse/rest/client/sync.py         | 10 +++++++++-
 tests/rest/client/test_relations.py | 10 +++++-----
 3 files changed, 15 insertions(+), 6 deletions(-)
 create mode 100644 changelog.d/11583.bugfix

diff --git a/changelog.d/11583.bugfix b/changelog.d/11583.bugfix
new file mode 100644
index 000000000..d2ed113e2
--- /dev/null
+++ b/changelog.d/11583.bugfix
@@ -0,0 +1 @@
+Fix a performance regression in `/sync` handling, introduced in 1.49.0.
diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py
index 88e4f5e06..7f5846d38 100644
--- a/synapse/rest/client/sync.py
+++ b/synapse/rest/client/sync.py
@@ -522,7 +522,15 @@ class SyncRestServlet(RestServlet):
                 time_now=time_now,
                 # Don't bother to bundle aggregations if the timeline is unlimited,
                 # as clients will have all the necessary information.
-                bundle_aggregations=room.timeline.limited,
+                # bundle_aggregations=room.timeline.limited,
+                #
+                # richvdh 2021-12-15: disable this temporarily as it has too high an
+                # overhead for initialsyncs. We need to figure out a way that the
+                # bundling can be done *before* the events are stored in the
+                # SyncResponseCache so that this part can be synchronous.
+                #
+                # Ensure to re-enable the test at tests/rest/client/test_relations.py::RelationsTestCase.test_bundled_aggregations.
+                bundle_aggregations=False,
                 token_id=token_id,
                 event_format=event_formatter,
                 only_event_fields=only_fields,
diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py
index 397c12c2a..1b58b7313 100644
--- a/tests/rest/client/test_relations.py
+++ b/tests/rest/client/test_relations.py
@@ -574,11 +574,11 @@ class RelationsTestCase(unittest.HomeserverTestCase):
         assert_bundle(channel.json_body["event"]["unsigned"].get("m.relations"))
 
         # Request sync.
-        channel = self.make_request("GET", "/sync", access_token=self.user_token)
-        self.assertEquals(200, channel.code, channel.json_body)
-        room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"]
-        self.assertTrue(room_timeline["limited"])
-        _find_and_assert_event(room_timeline["events"])
+        # channel = self.make_request("GET", "/sync", access_token=self.user_token)
+        # self.assertEquals(200, channel.code, channel.json_body)
+        # room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"]
+        # self.assertTrue(room_timeline["limited"])
+        # _find_and_assert_event(room_timeline["events"])
 
         # Note that /relations is tested separately in test_aggregation_get_event_for_thread
         # since it needs different data configured.

From 8c36d332d598f55f471196fb9415bf27db3e6fcd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Dec 2021 11:07:41 +0000
Subject: [PATCH 2/8] 1.49.1

---
 CHANGES.md               | 9 +++++++++
 changelog.d/11583.bugfix | 1 -
 debian/changelog         | 6 ++++++
 synapse/__init__.py      | 2 +-
 4 files changed, 16 insertions(+), 2 deletions(-)
 delete mode 100644 changelog.d/11583.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index 58217e655..bdfa40686 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,12 @@
+Synapse 1.49.1 (2021-12-21)
+===========================
+
+Bugfixes
+--------
+
+- Fix a performance regression in `/sync` handling, introduced in 1.49.0. ([\#11583](https://github.com/matrix-org/synapse/issues/11583))
+
+
 Synapse 1.49.0 (2021-12-14)
 ===========================
 
diff --git a/changelog.d/11583.bugfix b/changelog.d/11583.bugfix
deleted file mode 100644
index d2ed113e2..000000000
--- a/changelog.d/11583.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a performance regression in `/sync` handling, introduced in 1.49.0.
diff --git a/debian/changelog b/debian/changelog
index 794d8efa4..61a1d003b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.49.1) stable; urgency=medium
+
+  * New synapse release 1.49.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Dec 2021 11:07:30 +0000
+
 matrix-synapse-py3 (1.49.0) stable; urgency=medium
 
   * New synapse release 1.49.0.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index d44a03a2c..d45e0f188 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.49.0"
+__version__ = "1.49.1"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

From b9f2f6d3c414a5b0d2bbce15ef12d669dbcea82a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Dec 2021 11:23:35 +0000
Subject: [PATCH 3/8] more words

---
 CHANGES.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGES.md b/CHANGES.md
index bdfa40686..0fcfb3c78 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,6 +1,8 @@
 Synapse 1.49.1 (2021-12-21)
 ===========================
 
+This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it imposssible for such users to log in on a new client.
+
 Bugfixes
 --------
 

From aa58e8a28a05c5a71003570f8bdcd1256276f243 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Dec 2021 11:24:24 +0000
Subject: [PATCH 4/8] typopo

---
 CHANGES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 0fcfb3c78..693204584 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,7 +1,7 @@
 Synapse 1.49.1 (2021-12-21)
 ===========================
 
-This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it imposssible for such users to log in on a new client.
+This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client.
 
 Bugfixes
 --------

From 57ca8ab10f74e9f36b3a93ff31438cabb146fb5f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Dec 2021 12:06:31 +0000
Subject: [PATCH 5/8] Add notes about dropping support for Python 3.6 and
 Postgres 9.6.

---
 CHANGES.md                 | 4 ++++
 docs/deprecation_policy.md | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 693204584..dc6fa1779 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -3,6 +3,10 @@ Synapse 1.49.1 (2021-12-21)
 
 This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client.
 
+**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10.
+
+**Note:** We will also stop producing packages for Ubuntu 18.04 (Bionic Beaver) after this release, as it uses Python 3.6.
+
 Bugfixes
 --------
 
diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md
index 06ea34055..359dac07c 100644
--- a/docs/deprecation_policy.md
+++ b/docs/deprecation_policy.md
@@ -14,8 +14,8 @@ i.e. when a version reaches End of Life Synapse will withdraw support for that
 version in future releases.
 
 Details on the upstream support life cycles for Python and PostgreSQL are
-documented at https://endoflife.date/python and
-https://endoflife.date/postgresql.
+documented at [https://endoflife.date/python](https://endoflife.date/python) and
+[https://endoflife.date/postgresql](https://endoflife.date/postgresql).
 
 
 Context

From 2bf31f7807c7a0c229170803c97090d612dc16f9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 21 Dec 2021 17:26:04 +0000
Subject: [PATCH 6/8] Pin to frozendict<2.1.2 (#11625)

... to work around breakage on buster
(https://github.com/Marco-Sulla/python-frozendict/issues/41)
---
 changelog.d/11625.misc         | 1 +
 synapse/python_dependencies.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11625.misc

diff --git a/changelog.d/11625.misc b/changelog.d/11625.misc
new file mode 100644
index 000000000..82ee78e72
--- /dev/null
+++ b/changelog.d/11625.misc
@@ -0,0 +1 @@
+Work around a build problem on Debian Buster.
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 7d2695424..13fb69460 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -50,7 +50,8 @@ logger = logging.getLogger(__name__)
 REQUIREMENTS = [
     # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0
     "jsonschema>=3.0.0",
-    "frozendict>=1",
+    # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41
+    "frozendict>=1,<2.1.2",
     "unpaddedbase64>=1.1.0",
     "canonicaljson>=1.4.0",
     # we use the type definitions added in signedjson 1.1.

From aa874a13902fcf7f3e4024cd95a8491f308367bb Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Dec 2021 17:32:16 +0000
Subject: [PATCH 7/8] 1.49.2

---
 CHANGES.md             | 13 ++++++++++++-
 changelog.d/11625.misc |  1 -
 debian/changelog       |  6 ++++++
 synapse/__init__.py    |  2 +-
 4 files changed, 19 insertions(+), 3 deletions(-)
 delete mode 100644 changelog.d/11625.misc

diff --git a/CHANGES.md b/CHANGES.md
index dc6fa1779..9301ec880 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,4 +1,4 @@
-Synapse 1.49.1 (2021-12-21)
+Synapse 1.49.2 (2021-12-21)
 ===========================
 
 This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client.
@@ -12,6 +12,17 @@ Bugfixes
 
 - Fix a performance regression in `/sync` handling, introduced in 1.49.0. ([\#11583](https://github.com/matrix-org/synapse/issues/11583))
 
+Internal Changes
+----------------
+
+- Work around a build problem on Debian Buster. ([\#11625](https://github.com/matrix-org/synapse/issues/11625))
+
+
+Synapse 1.49.1 (2021-12-21)
+===========================
+
+Not released due to problems building the debian packages.
+
 
 Synapse 1.49.0 (2021-12-14)
 ===========================
diff --git a/changelog.d/11625.misc b/changelog.d/11625.misc
deleted file mode 100644
index 82ee78e72..000000000
--- a/changelog.d/11625.misc
+++ /dev/null
@@ -1 +0,0 @@
-Work around a build problem on Debian Buster.
diff --git a/debian/changelog b/debian/changelog
index 61a1d003b..ebe3e0cbf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.49.2) stable; urgency=medium
+
+  * New synapse release 1.49.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Dec 2021 17:31:03 +0000
+
 matrix-synapse-py3 (1.49.1) stable; urgency=medium
 
   * New synapse release 1.49.1.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index d45e0f188..95a49c20b 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.49.1"
+__version__ = "1.49.2"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

From 6b6dcdc338a3fcfa1da6c3e2a38f62c268df1cbc Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Dec 2021 17:41:34 +0000
Subject: [PATCH 8/8] update changelog

postgres 10 _+_
---
 CHANGES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 9301ec880..9f6e29631 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -3,7 +3,7 @@ Synapse 1.49.2 (2021-12-21)
 
 This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client.
 
-**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10.
+**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10+.
 
 **Note:** We will also stop producing packages for Ubuntu 18.04 (Bionic Beaver) after this release, as it uses Python 3.6.