diff --git a/changelog.d/10116.bugfix b/changelog.d/10116.bugfix
new file mode 100644
index 000000000..90ef70755
--- /dev/null
+++ b/changelog.d/10116.bugfix
@@ -0,0 +1 @@
+Fix bug where the server would attempt to fetch the same history in the room from a remote server multiple times in parallel.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index f3f97db2f..b802822ba 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -178,6 +178,8 @@ class FederationHandler(BaseHandler):
         self.room_queues = {}  # type: Dict[str, List[Tuple[EventBase, str]]]
         self._room_pdu_linearizer = Linearizer("fed_room_pdu")
 
+        self._room_backfill = Linearizer("room_backfill")
+
         self.third_party_event_rules = hs.get_third_party_event_rules()
 
         self._ephemeral_messages_enabled = hs.config.enable_ephemeral_messages
@@ -1041,6 +1043,12 @@ class FederationHandler(BaseHandler):
                 return. This is used as part of the heuristic to decide if we
                 should back paginate.
         """
+        with (await self._room_backfill.queue(room_id)):
+            return await self._maybe_backfill_inner(room_id, current_depth, limit)
+
+    async def _maybe_backfill_inner(
+        self, room_id: str, current_depth: int, limit: int
+    ) -> bool:
         extremities = await self.store.get_oldest_events_with_depth_in_room(room_id)
 
         if not extremities: