diff --git a/changelog.d/13694.bugfix b/changelog.d/13694.bugfix
new file mode 100644
index 000000000..48b9bb5f0
--- /dev/null
+++ b/changelog.d/13694.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled.
diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py
index 260db49ca..702b81e63 100644
--- a/synapse/config/experimental.py
+++ b/synapse/config/experimental.py
@@ -71,6 +71,9 @@ class ExperimentalConfig(Config):
         self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False)
 
         # MSC2654: Unread counts
+        #
+        # Note that enabling this will result in an incorrect unread count for
+        # previously calculated push actions.
         self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False)
 
         # MSC2815 (allow room moderators to view redacted event content)
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index ccd512be5..d1caf8a0f 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -262,7 +262,12 @@ class BulkPushRuleEvaluator:
             # This can happen due to out of band memberships
             return
 
-        count_as_unread = _should_count_as_unread(event, context)
+        # Disable counting as unread unless the experimental configuration is
+        # enabled, as it can cause additional (unwanted) rows to be added to the
+        # event_push_actions table.
+        count_as_unread = False
+        if self.hs.config.experimental.msc2654_enabled:
+            count_as_unread = _should_count_as_unread(event, context)
 
         rules_by_user = await self._get_rules_for_event(event)
         actions_by_user: Dict[str, Collection[Union[Mapping, str]]] = {}
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 62fd4aeb2..fc43d7edd 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -67,9 +67,7 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):
 
         last_event_id: str
 
-        def _assert_counts(
-            noitf_count: int, unread_count: int, highlight_count: int
-        ) -> None:
+        def _assert_counts(noitf_count: int, highlight_count: int) -> None:
             counts = self.get_success(
                 self.store.db_pool.runInteraction(
                     "get-unread-counts",
@@ -82,7 +80,7 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):
                 counts,
                 NotifCounts(
                     notify_count=noitf_count,
-                    unread_count=unread_count,
+                    unread_count=0,
                     highlight_count=highlight_count,
                 ),
             )
@@ -112,27 +110,27 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):
                 )
             )
 
-        _assert_counts(0, 0, 0)
+        _assert_counts(0, 0)
         _create_event()
-        _assert_counts(1, 1, 0)
+        _assert_counts(1, 0)
         _rotate()
-        _assert_counts(1, 1, 0)
+        _assert_counts(1, 0)
 
         event_id = _create_event()
-        _assert_counts(2, 2, 0)
+        _assert_counts(2, 0)
         _rotate()
-        _assert_counts(2, 2, 0)
+        _assert_counts(2, 0)
 
         _create_event()
         _mark_read(event_id)
-        _assert_counts(1, 1, 0)
+        _assert_counts(1, 0)
 
         _mark_read(last_event_id)
-        _assert_counts(0, 0, 0)
+        _assert_counts(0, 0)
 
         _create_event()
         _rotate()
-        _assert_counts(1, 1, 0)
+        _assert_counts(1, 0)
 
         # Delete old event push actions, this should not affect the (summarised) count.
         #
@@ -151,35 +149,35 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):
             )
         )
         self.assertEqual(result, [])
-        _assert_counts(1, 1, 0)
+        _assert_counts(1, 0)
 
         _mark_read(last_event_id)
-        _assert_counts(0, 0, 0)
+        _assert_counts(0, 0)
 
         event_id = _create_event(True)
-        _assert_counts(1, 1, 1)
+        _assert_counts(1, 1)
         _rotate()
-        _assert_counts(1, 1, 1)
+        _assert_counts(1, 1)
 
         # Check that adding another notification and rotating after highlight
         # works.
         _create_event()
         _rotate()
-        _assert_counts(2, 2, 1)
+        _assert_counts(2, 1)
 
         # Check that sending read receipts at different points results in the
         # right counts.
         _mark_read(event_id)
-        _assert_counts(1, 1, 0)
+        _assert_counts(1, 0)
         _mark_read(last_event_id)
-        _assert_counts(0, 0, 0)
+        _assert_counts(0, 0)
 
         _create_event(True)
-        _assert_counts(1, 1, 1)
+        _assert_counts(1, 1)
         _mark_read(last_event_id)
-        _assert_counts(0, 0, 0)
+        _assert_counts(0, 0)
         _rotate()
-        _assert_counts(0, 0, 0)
+        _assert_counts(0, 0)
 
     def test_find_first_stream_ordering_after_ts(self) -> None:
         def add_event(so: int, ts: int) -> None: