Time out HTTP federation requests

synapse/http/matrixfederationclient.py

@@ -22,7 +22,7 @@ from twisted.web._newclient import ResponseDone
 
 from synapse.http.agent_name import AGENT_NAME
 from synapse.http.endpoint import matrix_federation_endpoint
-from synapse.util.async import sleep
+from synapse.util.async import sleep, time_bound_deferred
 from synapse.util.logcontext import PreserveLoggingContext
 
 from syutil.jsonutil import encode_canonical_json
@@ -78,6 +78,7 @@ class MatrixFederationHttpClient(object):
         self.signing_key = hs.config.signing_key[0]
         self.server_name = hs.hostname
         self.agent = MatrixFederationHttpAgent(reactor)
+        self.clock = hs.get_clock()
 
     @defer.inlineCallbacks
     def _create_request(self, destination, method, path_bytes,
@@ -117,7 +118,7 @@ class MatrixFederationHttpClient(object):
 
             try:
                 with PreserveLoggingContext():
-                    response = yield self.agent.request(
+                    request_deferred = self.agent.request(
                         destination,
                         endpoint,
                         method,
@@ -128,6 +129,12 @@ class MatrixFederationHttpClient(object):
                         producer
                     )
 
+                    response = yield time_bound_deferred(
+                        request_deferred,
+                        clock=self.clock,
+                        time_out=60,
+                    )
+
                 logger.debug("Got response to %s", method)
                 break
             except Exception as e:

synapse/util/async.py

@@ -32,3 +32,23 @@ def run_on_reactor():
     iteration of the main loop
     """
     return sleep(0)
+
+
+def time_bound_deferred(given_deferred, clock, time_out):
+    ret_deferred = defer.Deferred()
+
+    def timed_out():
+        if not given_deferred.called:
+            given_deferred.cancel()
+            ret_deferred.errback(RuntimeError("Timed out"))
+
+    timer = clock.call_later(time_out, timed_out)
+
+    def succeed(result):
+        clock.cancel_call_later(timer)
+        ret_deferred.callback(result)
+
+    given_deferred.addCallback(succeed)
+    given_deferred.addErrback(ret_deferred.errback)
+
+    return ret_deferred