diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py
index 2a8bceed9..4d674cdb9 100644
--- a/synapse/http/federation/matrix_federation_agent.py
+++ b/synapse/http/federation/matrix_federation_agent.py
@@ -15,6 +15,7 @@
 import logging
 
 import attr
+from netaddr import IPAddress
 from zope.interface import implementer
 
 from twisted.internet import defer
@@ -137,6 +138,24 @@ class MatrixFederationAgent(object):
         Returns:
             Deferred[_RoutingResult]
         """
+        # check for an IP literal
+        try:
+            ip_address = IPAddress(parsed_uri.host.decode("ascii"))
+        except Exception:
+            # not an IP address
+            ip_address = None
+
+        if ip_address:
+            port = parsed_uri.port
+            if port == -1:
+                port = 8448
+            defer.returnValue(_RoutingResult(
+                host_header=parsed_uri.netloc,
+                tls_server_name=parsed_uri.host,
+                target_host=parsed_uri.host,
+                target_port=port,
+            ))
+
         if parsed_uri.port != -1:
             # there is an explicit port
             defer.returnValue(_RoutingResult(
diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py
index f144092a5..8257594fb 100644
--- a/tests/http/federation/test_matrix_federation_agent.py
+++ b/tests/http/federation/test_matrix_federation_agent.py
@@ -166,11 +166,7 @@ class MatrixFederationAgentTests(TestCase):
         """
         Test the behaviour when the server name contains an explicit IP (with no port)
         """
-
-        # the SRV lookup will return an empty list (XXX: why do we even do an SRV lookup?)
-        self.mock_resolver.resolve_service.side_effect = lambda _: []
-
-        # then there will be a getaddrinfo on the IP
+        # there will be a getaddrinfo on the IP
         self.reactor.lookups["1.2.3.4"] = "1.2.3.4"
 
         test_d = self._make_get_request(b"matrix://1.2.3.4/foo/bar")
@@ -178,10 +174,6 @@ class MatrixFederationAgentTests(TestCase):
         # Nothing happened yet
         self.assertNoResult(test_d)
 
-        self.mock_resolver.resolve_service.assert_called_once_with(
-            b"_matrix._tcp.1.2.3.4",
-        )
-
         # Make sure treq is trying to connect
         clients = self.reactor.tcpClients
         self.assertEqual(len(clients), 1)
@@ -215,10 +207,7 @@ class MatrixFederationAgentTests(TestCase):
         (with no port)
         """
 
-        # the SRV lookup will return an empty list (XXX: why do we even do an SRV lookup?)
-        self.mock_resolver.resolve_service.side_effect = lambda _: []
-
-        # then there will be a getaddrinfo on the IP
+        # there will be a getaddrinfo on the IP
         self.reactor.lookups["::1"] = "::1"
 
         test_d = self._make_get_request(b"matrix://[::1]/foo/bar")
@@ -226,10 +215,6 @@ class MatrixFederationAgentTests(TestCase):
         # Nothing happened yet
         self.assertNoResult(test_d)
 
-        self.mock_resolver.resolve_service.assert_called_once_with(
-            b"_matrix._tcp.::1",
-        )
-
         # Make sure treq is trying to connect
         clients = self.reactor.tcpClients
         self.assertEqual(len(clients), 1)