Fix rejecting invites over federation (#12409)

Currently causes future incremental syncs to fail.

Broke by #12191

changelog.d/12409.misc

@@ -0,0 +1 @@
+Avoid trying to calculate the state at outlier events.

synapse/handlers/sync.py

@@ -1851,6 +1851,7 @@ class SyncHandler:
                         full_state=False,
                         since_token=since_token,
                         upto_token=leave_token,
+                        out_of_band=leave_event.internal_metadata.is_out_of_band_membership(),
                     )
                 )
 
@@ -2117,6 +2118,7 @@ class SyncHandler:
             ):
                 return
 
+            if not room_builder.out_of_band:
                 state = await self.compute_state_delta(
                     room_id,
                     batch,
@@ -2125,13 +2127,19 @@ class SyncHandler:
                     now_token,
                     full_state=full_state,
                 )
+            else:
+                # An out of band room won't have any state changes.
+                state = {}
 
             summary: Optional[JsonDict] = {}
 
             # we include a summary in room responses when we're lazy loading
             # members (as the client otherwise doesn't have enough info to form
             # the name itself).
-            if sync_config.filter_collection.lazy_load_members() and (
+            if (
+                not room_builder.out_of_band
+                and sync_config.filter_collection.lazy_load_members()
+                and (
                     # we recalculate the summary:
                     #   if there are membership changes in the timeline, or
                     #   if membership has changed during a gappy sync, or
@@ -2144,6 +2152,7 @@ class SyncHandler:
                         and any(t == EventTypes.Member for (t, k) in state)
                     )
                     or since_token is None
+                )
             ):
                 summary = await self.compute_summary(
                     room_id, sync_config, batch, state, now_token
@@ -2387,6 +2396,8 @@ class RoomSyncResultBuilder:
         full_state: Whether the full state should be sent in result
         since_token: Earliest point to return events from, or None
         upto_token: Latest point to return events from.
+        out_of_band: whether the events in the room are "out of band" events
+            and the server isn't in the room.
     """
 
     room_id: str
@@ -2396,3 +2407,5 @@ class RoomSyncResultBuilder:
     full_state: bool
     since_token: Optional[StreamToken]
     upto_token: StreamToken
+
+    out_of_band: bool = False