diff --git a/changelog.d/12155.misc b/changelog.d/12155.misc
new file mode 100644
index 000000000..9f333e718
--- /dev/null
+++ b/changelog.d/12155.misc
@@ -0,0 +1 @@
+Avoid trying to calculate the state at outlier events.
diff --git a/synapse/visibility.py b/synapse/visibility.py
index 1b970ce47..281cbe4d8 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -81,8 +81,9 @@ async def filter_events_for_client(
 
     types = ((EventTypes.RoomHistoryVisibility, ""), (EventTypes.Member, user_id))
 
+    # we exclude outliers at this point, and then handle them separately later
     event_id_to_state = await storage.state.get_state_for_events(
-        frozenset(e.event_id for e in events),
+        frozenset(e.event_id for e in events if not e.internal_metadata.outlier),
         state_filter=StateFilter.from_types(types),
     )
 
@@ -154,6 +155,17 @@ async def filter_events_for_client(
         if event.event_id in always_include_ids:
             return event
 
+        # we need to handle outliers separately, since we don't have the room state.
+        if event.internal_metadata.outlier:
+            # Normally these can't be seen by clients, but we make an exception for
+            # for out-of-band membership events (eg, incoming invites, or rejections of
+            # said invite) for the user themselves.
+            if event.type == EventTypes.Member and event.state_key == user_id:
+                logger.debug("Returning out-of-band-membership event %s", event)
+                return event
+
+            return None
+
         state = event_id_to_state[event.event_id]
 
         # get the room_visibility at the time of the event.
@@ -198,6 +210,9 @@ async def filter_events_for_client(
 
             # Always allow the user to see their own leave events, otherwise
             # they won't see the room disappear if they reject the invite
+            #
+            # (Note this doesn't work for out-of-band invite rejections, which don't
+            # have prev_state populated. They are handled above in the outlier code.)
             if membership == "leave" and (
                 prev_membership == "join" or prev_membership == "invite"
             ):
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 219b5660b..532e3fe9c 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -13,11 +13,12 @@
 # limitations under the License.
 import logging
 from typing import Optional
+from unittest.mock import patch
 
 from synapse.api.room_versions import RoomVersions
-from synapse.events import EventBase
-from synapse.types import JsonDict
-from synapse.visibility import filter_events_for_server
+from synapse.events import EventBase, make_event_from_dict
+from synapse.types import JsonDict, create_requester
+from synapse.visibility import filter_events_for_client, filter_events_for_server
 
 from tests import unittest
 from tests.utils import create_room
@@ -185,3 +186,72 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
 
         self.get_success(self.storage.persistence.persist_event(event, context))
         return event
+
+
+class FilterEventsForClientTestCase(unittest.FederatingHomeserverTestCase):
+    def test_out_of_band_invite_rejection(self):
+        # this is where we have received an invite event over federation, and then
+        # rejected it.
+        invite_pdu = {
+            "room_id": "!room:id",
+            "depth": 1,
+            "auth_events": [],
+            "prev_events": [],
+            "origin_server_ts": 1,
+            "sender": "@someone:" + self.OTHER_SERVER_NAME,
+            "type": "m.room.member",
+            "state_key": "@user:test",
+            "content": {"membership": "invite"},
+        }
+        self.add_hashes_and_signatures(invite_pdu)
+        invite_event_id = make_event_from_dict(invite_pdu, RoomVersions.V9).event_id
+
+        self.get_success(
+            self.hs.get_federation_server().on_invite_request(
+                self.OTHER_SERVER_NAME,
+                invite_pdu,
+                "9",
+            )
+        )
+
+        # stub out do_remotely_reject_invite so that we fall back to a locally-
+        # generated rejection
+        with patch.object(
+            self.hs.get_federation_handler(),
+            "do_remotely_reject_invite",
+            side_effect=Exception(),
+        ):
+            reject_event_id, _ = self.get_success(
+                self.hs.get_room_member_handler().remote_reject_invite(
+                    invite_event_id,
+                    txn_id=None,
+                    requester=create_requester("@user:test"),
+                    content={},
+                )
+            )
+
+        invite_event, reject_event = self.get_success(
+            self.hs.get_datastores().main.get_events_as_list(
+                [invite_event_id, reject_event_id]
+            )
+        )
+
+        # the invited user should be able to see both the invite and the rejection
+        self.assertEqual(
+            self.get_success(
+                filter_events_for_client(
+                    self.hs.get_storage(), "@user:test", [invite_event, reject_event]
+                )
+            ),
+            [invite_event, reject_event],
+        )
+
+        # other users should see neither
+        self.assertEqual(
+            self.get_success(
+                filter_events_for_client(
+                    self.hs.get_storage(), "@other:test", [invite_event, reject_event]
+                )
+            ),
+            [],
+        )