From 7f9841bdec349075eb424c943db5569439acb83c Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Wed, 24 Nov 2021 20:21:44 +0100
Subject: [PATCH] Lower minumum batch size to 1 for background updates (#11422)

Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 changelog.d/11422.bugfix                    |  1 +
 synapse/storage/background_updates.py       |  2 +-
 tests/rest/admin/test_background_updates.py | 25 ++++++++++++++-------
 tests/storage/test_background_update.py     |  8 +++----
 4 files changed, 23 insertions(+), 13 deletions(-)
 create mode 100644 changelog.d/11422.bugfix

diff --git a/changelog.d/11422.bugfix b/changelog.d/11422.bugfix
new file mode 100644
index 000000000..28ac65ea7
--- /dev/null
+++ b/changelog.d/11422.bugfix
@@ -0,0 +1 @@
+Improve performance of various background database schema updates.
diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index b104f9032..bc8364400 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -82,7 +82,7 @@ class BackgroundUpdater:
     process and autotuning the batch size.
     """
 
-    MINIMUM_BACKGROUND_BATCH_SIZE = 100
+    MINIMUM_BACKGROUND_BATCH_SIZE = 1
     DEFAULT_BACKGROUND_BATCH_SIZE = 100
     BACKGROUND_UPDATE_INTERVAL_MS = 1000
     BACKGROUND_UPDATE_DURATION_MS = 100
diff --git a/tests/rest/admin/test_background_updates.py b/tests/rest/admin/test_background_updates.py
index 178631676..cd5c60b65 100644
--- a/tests/rest/admin/test_background_updates.py
+++ b/tests/rest/admin/test_background_updates.py
@@ -20,6 +20,7 @@ import synapse.rest.admin
 from synapse.api.errors import Codes
 from synapse.rest.client import login
 from synapse.server import HomeServer
+from synapse.storage.background_updates import BackgroundUpdater
 
 from tests import unittest
 
@@ -150,9 +151,11 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase):
                 "current_updates": {
                     "master": {
                         "name": "test_update",
-                        "average_items_per_ms": 0.1,
+                        "average_items_per_ms": 0.001,
                         "total_duration_ms": 1000.0,
-                        "total_item_count": 100,
+                        "total_item_count": (
+                            BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
+                        ),
                     }
                 },
                 "enabled": True,
@@ -203,9 +206,11 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase):
                 "current_updates": {
                     "master": {
                         "name": "test_update",
-                        "average_items_per_ms": 0.1,
+                        "average_items_per_ms": 0.001,
                         "total_duration_ms": 1000.0,
-                        "total_item_count": 100,
+                        "total_item_count": (
+                            BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
+                        ),
                     }
                 },
                 "enabled": False,
@@ -230,9 +235,11 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase):
                 "current_updates": {
                     "master": {
                         "name": "test_update",
-                        "average_items_per_ms": 0.1,
+                        "average_items_per_ms": 0.001,
                         "total_duration_ms": 1000.0,
-                        "total_item_count": 100,
+                        "total_item_count": (
+                            BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
+                        ),
                     }
                 },
                 "enabled": False,
@@ -267,9 +274,11 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase):
                 "current_updates": {
                     "master": {
                         "name": "test_update",
-                        "average_items_per_ms": 0.1,
+                        "average_items_per_ms": 0.001,
                         "total_duration_ms": 2000.0,
-                        "total_item_count": 200,
+                        "total_item_count": (
+                            2 * BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
+                        ),
                     }
                 },
                 "enabled": True,
diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py
index 0da42b5ac..a5f5ebad4 100644
--- a/tests/storage/test_background_update.py
+++ b/tests/storage/test_background_update.py
@@ -19,11 +19,11 @@ class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
         )
 
     def test_do_background_update(self):
-        # the time we claim each update takes
-        duration_ms = 42
+        # the time we claim it takes to update one item when running the update
+        duration_ms = 4200
 
         # the target runtime for each bg update
-        target_background_update_duration_ms = 50000
+        target_background_update_duration_ms = 5000000
 
         store = self.hs.get_datastore()
         self.get_success(
@@ -57,7 +57,7 @@ class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
 
         # on the first call, we should get run with the default background update size
         self.update_handler.assert_called_once_with(
-            {"my_key": 1}, self.updates.DEFAULT_BACKGROUND_BATCH_SIZE
+            {"my_key": 1}, self.updates.MINIMUM_BACKGROUND_BATCH_SIZE
         )
 
         # second step: complete the update