allow specifying https:// proxy (#10411)

This commit is contained in:
Dirk Klimpel 2021-07-27 18:31:06 +02:00 committed by GitHub
parent e16eab29d6
commit 076deade02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 455 additions and 138 deletions

View File

@ -0,0 +1 @@
Add support for https connections to a proxy server. Contributed by @Bubu and @dklimpel.

View File

@ -14,21 +14,32 @@
import base64 import base64
import logging import logging
import re import re
from typing import Optional, Tuple from typing import Any, Dict, Optional, Tuple
from urllib.request import getproxies_environment, proxy_bypass_environment from urllib.parse import urlparse
from urllib.request import ( # type: ignore[attr-defined]
getproxies_environment,
proxy_bypass_environment,
)
import attr import attr
from zope.interface import implementer from zope.interface import implementer
from twisted.internet import defer from twisted.internet import defer
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
from twisted.internet.interfaces import IReactorCore, IStreamClientEndpoint
from twisted.python.failure import Failure from twisted.python.failure import Failure
from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _AgentBase from twisted.web.client import (
URI,
BrowserLikePolicyForHTTPS,
HTTPConnectionPool,
_AgentBase,
)
from twisted.web.error import SchemeNotSupported from twisted.web.error import SchemeNotSupported
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
from twisted.web.iweb import IAgent, IPolicyForHTTPS from twisted.web.iweb import IAgent, IBodyProducer, IPolicyForHTTPS
from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint
from synapse.types import ISynapseReactor
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -63,35 +74,38 @@ class ProxyAgent(_AgentBase):
reactor might have some blacklisting applied (i.e. for DNS queries), reactor might have some blacklisting applied (i.e. for DNS queries),
but we need unblocked access to the proxy. but we need unblocked access to the proxy.
contextFactory (IPolicyForHTTPS): A factory for TLS contexts, to control the contextFactory: A factory for TLS contexts, to control the
verification parameters of OpenSSL. The default is to use a verification parameters of OpenSSL. The default is to use a
`BrowserLikePolicyForHTTPS`, so unless you have special `BrowserLikePolicyForHTTPS`, so unless you have special
requirements you can leave this as-is. requirements you can leave this as-is.
connectTimeout (Optional[float]): The amount of time that this Agent will wait connectTimeout: The amount of time that this Agent will wait
for the peer to accept a connection, in seconds. If 'None', for the peer to accept a connection, in seconds. If 'None',
HostnameEndpoint's default (30s) will be used. HostnameEndpoint's default (30s) will be used.
This is used for connections to both proxies and destination servers. This is used for connections to both proxies and destination servers.
bindAddress (bytes): The local address for client sockets to bind to. bindAddress: The local address for client sockets to bind to.
pool (HTTPConnectionPool|None): connection pool to be used. If None, a pool: connection pool to be used. If None, a
non-persistent pool instance will be created. non-persistent pool instance will be created.
use_proxy (bool): Whether proxy settings should be discovered and used use_proxy: Whether proxy settings should be discovered and used
from conventional environment variables. from conventional environment variables.
Raises:
ValueError if use_proxy is set and the environment variables
contain an invalid proxy specification.
""" """
def __init__( def __init__(
self, self,
reactor, reactor: IReactorCore,
proxy_reactor=None, proxy_reactor: Optional[ISynapseReactor] = None,
contextFactory: Optional[IPolicyForHTTPS] = None, contextFactory: Optional[IPolicyForHTTPS] = None,
connectTimeout=None, connectTimeout: Optional[float] = None,
bindAddress=None, bindAddress: Optional[bytes] = None,
pool=None, pool: Optional[HTTPConnectionPool] = None,
use_proxy=False, use_proxy: bool = False,
): ):
contextFactory = contextFactory or BrowserLikePolicyForHTTPS() contextFactory = contextFactory or BrowserLikePolicyForHTTPS()
@ -102,7 +116,7 @@ class ProxyAgent(_AgentBase):
else: else:
self.proxy_reactor = proxy_reactor self.proxy_reactor = proxy_reactor
self._endpoint_kwargs = {} self._endpoint_kwargs: Dict[str, Any] = {}
if connectTimeout is not None: if connectTimeout is not None:
self._endpoint_kwargs["timeout"] = connectTimeout self._endpoint_kwargs["timeout"] = connectTimeout
if bindAddress is not None: if bindAddress is not None:
@ -117,16 +131,12 @@ class ProxyAgent(_AgentBase):
https_proxy = proxies["https"].encode() if "https" in proxies else None https_proxy = proxies["https"].encode() if "https" in proxies else None
no_proxy = proxies["no"] if "no" in proxies else None no_proxy = proxies["no"] if "no" in proxies else None
# Parse credentials from http and https proxy connection string if present self.http_proxy_endpoint, self.http_proxy_creds = _http_proxy_endpoint(
self.http_proxy_creds, http_proxy = parse_username_password(http_proxy) http_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs
self.https_proxy_creds, https_proxy = parse_username_password(https_proxy)
self.http_proxy_endpoint = _http_proxy_endpoint(
http_proxy, self.proxy_reactor, **self._endpoint_kwargs
) )
self.https_proxy_endpoint = _http_proxy_endpoint( self.https_proxy_endpoint, self.https_proxy_creds = _http_proxy_endpoint(
https_proxy, self.proxy_reactor, **self._endpoint_kwargs https_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs
) )
self.no_proxy = no_proxy self.no_proxy = no_proxy
@ -134,7 +144,13 @@ class ProxyAgent(_AgentBase):
self._policy_for_https = contextFactory self._policy_for_https = contextFactory
self._reactor = reactor self._reactor = reactor
def request(self, method, uri, headers=None, bodyProducer=None): def request(
self,
method: bytes,
uri: bytes,
headers: Optional[Headers] = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> defer.Deferred:
""" """
Issue a request to the server indicated by the given uri. Issue a request to the server indicated by the given uri.
@ -146,16 +162,15 @@ class ProxyAgent(_AgentBase):
See also: twisted.web.iweb.IAgent.request See also: twisted.web.iweb.IAgent.request
Args: Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc method: The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request. uri: The location of the resource to request.
headers (Headers|None): Extra headers to send with the request headers: Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to bodyProducer: An object which can generate bytes to make up the body of
make up the body of this request (for example, the properly encoded this request (for example, the properly encoded contents of a file for
contents of a file for a file upload). Or, None if the request is to a file upload). Or, None if the request is to have no body.
have no body.
Returns: Returns:
Deferred[IResponse]: completes when the header of the response has Deferred[IResponse]: completes when the header of the response has
@ -253,70 +268,89 @@ class ProxyAgent(_AgentBase):
) )
def _http_proxy_endpoint(proxy: Optional[bytes], reactor, **kwargs): def _http_proxy_endpoint(
proxy: Optional[bytes],
reactor: IReactorCore,
tls_options_factory: IPolicyForHTTPS,
**kwargs,
) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]:
"""Parses an http proxy setting and returns an endpoint for the proxy """Parses an http proxy setting and returns an endpoint for the proxy
Args: Args:
proxy: the proxy setting in the form: [<username>:<password>@]<host>[:<port>] proxy: the proxy setting in the form: [scheme://][<username>:<password>@]<host>[:<port>]
Note that compared to other apps, this function currently lacks support This currently supports http:// and https:// proxies.
for specifying a protocol schema (i.e. protocol://...). A hostname without scheme is assumed to be http.
reactor: reactor to be used to connect to the proxy reactor: reactor to be used to connect to the proxy
tls_options_factory: the TLS options to use when connecting through a https proxy
kwargs: other args to be passed to HostnameEndpoint kwargs: other args to be passed to HostnameEndpoint
Returns: Returns:
interfaces.IStreamClientEndpoint|None: endpoint to use to connect to the proxy, a tuple of
or None endpoint to use to connect to the proxy, or None
ProxyCredentials or if no credentials were found, or None
Raise:
ValueError if proxy has no hostname or unsupported scheme.
""" """
if proxy is None: if proxy is None:
return None return None, None
# Parse the connection string # Note: urlsplit/urlparse cannot be used here as that does not work (for Python
host, port = parse_host_port(proxy, default_port=1080) # 3.9+) on scheme-less proxies, e.g. host:port.
return HostnameEndpoint(reactor, host, port, **kwargs) scheme, host, port, credentials = parse_proxy(proxy)
proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs)
if scheme == b"https":
tls_options = tls_options_factory.creatorForNetloc(host, port)
proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint)
return proxy_endpoint, credentials
def parse_username_password(proxy: bytes) -> Tuple[Optional[ProxyCredentials], bytes]: def parse_proxy(
proxy: bytes, default_scheme: bytes = b"http", default_port: int = 1080
) -> Tuple[bytes, bytes, int, Optional[ProxyCredentials]]:
""" """
Parses the username and password from a proxy declaration e.g Parse a proxy connection string.
username:password@hostname:port.
Given a HTTP proxy URL, breaks it down into components and checks that it
has a hostname (otherwise it is not useful to us when trying to find a
proxy) and asserts that the URL has a scheme we support.
Args: Args:
proxy: The proxy connection string. proxy: The proxy connection string. Must be in the form '[scheme://][<username>:<password>@]host[:port]'.
default_scheme: The default scheme to return if one is not found in `proxy`. Defaults to http
Returns default_port: The default port to return if one is not found in `proxy`. Defaults to 1080
An instance of ProxyCredentials and the proxy connection string with any credentials
stripped, i.e u:p@host:port -> host:port. If no credentials were found, the
ProxyCredentials instance is replaced with None.
"""
if proxy and b"@" in proxy:
# We use rsplit here as the password could contain an @ character
credentials, proxy_without_credentials = proxy.rsplit(b"@", 1)
return ProxyCredentials(credentials), proxy_without_credentials
return None, proxy
def parse_host_port(hostport: bytes, default_port: int = None) -> Tuple[bytes, int]:
"""
Parse the hostname and port from a proxy connection byte string.
Args:
hostport: The proxy connection string. Must be in the form 'host[:port]'.
default_port: The default port to return if one is not found in `hostport`.
Returns: Returns:
A tuple containing the hostname and port. Uses `default_port` if one was not found. A tuple containing the scheme, hostname, port and ProxyCredentials.
""" If no credentials were found, the ProxyCredentials instance is replaced with None.
if b":" in hostport:
host, port = hostport.rsplit(b":", 1)
try:
port = int(port)
return host, port
except ValueError:
# the thing after the : wasn't a valid port; presumably this is an
# IPv6 address.
pass
return hostport, default_port Raise:
ValueError if proxy has no hostname or unsupported scheme.
"""
# First check if we have a scheme present
# Note: urlsplit/urlparse cannot be used (for Python # 3.9+) on scheme-less proxies, e.g. host:port.
if b"://" not in proxy:
proxy = b"".join([default_scheme, b"://", proxy])
url = urlparse(proxy)
if not url.hostname:
raise ValueError("Proxy URL did not contain a hostname! Please specify one.")
if url.scheme not in (b"http", b"https"):
raise ValueError(
f"Unknown proxy scheme {url.scheme!s}; only 'http' and 'https' is supported."
)
credentials = None
if url.username and url.password:
credentials = ProxyCredentials(b"".join([url.username, b":", url.password]))
return url.scheme, url.hostname, url.port or default_port, credentials

View File

@ -14,19 +14,22 @@
import base64 import base64
import logging import logging
import os import os
from typing import Optional from typing import Iterable, Optional
from unittest.mock import patch from unittest.mock import patch
import treq import treq
from netaddr import IPSet from netaddr import IPSet
from parameterized import parameterized
from twisted.internet import interfaces # noqa: F401 from twisted.internet import interfaces # noqa: F401
from twisted.internet.endpoints import HostnameEndpoint, _WrapperEndpoint
from twisted.internet.interfaces import IProtocol, IProtocolFactory
from twisted.internet.protocol import Factory from twisted.internet.protocol import Factory
from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
from twisted.web.http import HTTPChannel from twisted.web.http import HTTPChannel
from synapse.http.client import BlacklistingReactorWrapper from synapse.http.client import BlacklistingReactorWrapper
from synapse.http.proxyagent import ProxyAgent from synapse.http.proxyagent import ProxyAgent, ProxyCredentials, parse_proxy
from tests.http import TestServerTLSConnectionFactory, get_test_https_policy from tests.http import TestServerTLSConnectionFactory, get_test_https_policy
from tests.server import FakeTransport, ThreadedMemoryReactorClock from tests.server import FakeTransport, ThreadedMemoryReactorClock
@ -37,33 +40,208 @@ logger = logging.getLogger(__name__)
HTTPFactory = Factory.forProtocol(HTTPChannel) HTTPFactory = Factory.forProtocol(HTTPChannel)
class ProxyParserTests(TestCase):
"""
Values for test
[
proxy_string,
expected_scheme,
expected_hostname,
expected_port,
expected_credentials,
]
"""
@parameterized.expand(
[
# host
[b"localhost", b"http", b"localhost", 1080, None],
[b"localhost:9988", b"http", b"localhost", 9988, None],
# host+scheme
[b"https://localhost", b"https", b"localhost", 1080, None],
[b"https://localhost:1234", b"https", b"localhost", 1234, None],
# ipv4
[b"1.2.3.4", b"http", b"1.2.3.4", 1080, None],
[b"1.2.3.4:9988", b"http", b"1.2.3.4", 9988, None],
# ipv4+scheme
[b"https://1.2.3.4", b"https", b"1.2.3.4", 1080, None],
[b"https://1.2.3.4:9988", b"https", b"1.2.3.4", 9988, None],
# ipv6 - without brackets is broken
# [
# b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
# b"http",
# b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
# 1080,
# None,
# ],
# [
# b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
# b"http",
# b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
# 1080,
# None,
# ],
# [b"::1", b"http", b"::1", 1080, None],
# [b"::ffff:0.0.0.0", b"http", b"::ffff:0.0.0.0", 1080, None],
# ipv6 - with brackets
[
b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
b"http",
b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
1080,
None,
],
[
b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
b"http",
b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
1080,
None,
],
[b"[::1]", b"http", b"::1", 1080, None],
[b"[::ffff:0.0.0.0]", b"http", b"::ffff:0.0.0.0", 1080, None],
# ipv6+port
[
b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
b"http",
b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
9988,
None,
],
[
b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
b"http",
b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
9988,
None,
],
[b"[::1]:9988", b"http", b"::1", 9988, None],
[b"[::ffff:0.0.0.0]:9988", b"http", b"::ffff:0.0.0.0", 9988, None],
# ipv6+scheme
[
b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
b"https",
b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
1080,
None,
],
[
b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
b"https",
b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
1080,
None,
],
[b"https://[::1]", b"https", b"::1", 1080, None],
[b"https://[::ffff:0.0.0.0]", b"https", b"::ffff:0.0.0.0", 1080, None],
# ipv6+scheme+port
[
b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
b"https",
b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
9988,
None,
],
[
b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
b"https",
b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
9988,
None,
],
[b"https://[::1]:9988", b"https", b"::1", 9988, None],
# with credentials
[
b"https://user:pass@1.2.3.4:9988",
b"https",
b"1.2.3.4",
9988,
b"user:pass",
],
[b"user:pass@1.2.3.4:9988", b"http", b"1.2.3.4", 9988, b"user:pass"],
[
b"https://user:pass@proxy.local:9988",
b"https",
b"proxy.local",
9988,
b"user:pass",
],
[
b"user:pass@proxy.local:9988",
b"http",
b"proxy.local",
9988,
b"user:pass",
],
]
)
def test_parse_proxy(
self,
proxy_string: bytes,
expected_scheme: bytes,
expected_hostname: bytes,
expected_port: int,
expected_credentials: Optional[bytes],
):
"""
Tests that a given proxy URL will be broken into the components.
Args:
proxy_string: The proxy connection string.
expected_scheme: Expected value of proxy scheme.
expected_hostname: Expected value of proxy hostname.
expected_port: Expected value of proxy port.
expected_credentials: Expected value of credentials.
Must be in form '<username>:<password>' or None
"""
proxy_cred = None
if expected_credentials:
proxy_cred = ProxyCredentials(expected_credentials)
self.assertEqual(
(
expected_scheme,
expected_hostname,
expected_port,
proxy_cred,
),
parse_proxy(proxy_string),
)
class MatrixFederationAgentTests(TestCase): class MatrixFederationAgentTests(TestCase):
def setUp(self): def setUp(self):
self.reactor = ThreadedMemoryReactorClock() self.reactor = ThreadedMemoryReactorClock()
def _make_connection( def _make_connection(
self, client_factory, server_factory, ssl=False, expected_sni=None self,
): client_factory: IProtocolFactory,
server_factory: IProtocolFactory,
ssl: bool = False,
expected_sni: Optional[bytes] = None,
tls_sanlist: Optional[Iterable[bytes]] = None,
) -> IProtocol:
"""Builds a test server, and completes the outgoing client connection """Builds a test server, and completes the outgoing client connection
Args: Args:
client_factory (interfaces.IProtocolFactory): the the factory that the client_factory: the the factory that the
application is trying to use to make the outbound connection. We will application is trying to use to make the outbound connection. We will
invoke it to build the client Protocol invoke it to build the client Protocol
server_factory (interfaces.IProtocolFactory): a factory to build the server_factory: a factory to build the
server-side protocol server-side protocol
ssl (bool): If true, we will expect an ssl connection and wrap ssl: If true, we will expect an ssl connection and wrap
server_factory with a TLSMemoryBIOFactory server_factory with a TLSMemoryBIOFactory
expected_sni (bytes|None): the expected SNI value expected_sni: the expected SNI value
tls_sanlist: list of SAN entries for the TLS cert presented by the server.
Defaults to [b'DNS:test.com']
Returns: Returns:
IProtocol: the server Protocol returned by server_factory the server Protocol returned by server_factory
""" """
if ssl: if ssl:
server_factory = _wrap_server_factory_for_tls(server_factory) server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist)
server_protocol = server_factory.buildProtocol(None) server_protocol = server_factory.buildProtocol(None)
@ -98,22 +276,28 @@ class MatrixFederationAgentTests(TestCase):
self.assertEqual( self.assertEqual(
server_name, server_name,
expected_sni, expected_sni,
"Expected SNI %s but got %s" % (expected_sni, server_name), f"Expected SNI {expected_sni!s} but got {server_name!s}",
) )
return http_protocol return http_protocol
def _test_request_direct_connection(self, agent, scheme, hostname, path): def _test_request_direct_connection(
self,
agent: ProxyAgent,
scheme: bytes,
hostname: bytes,
path: bytes,
):
"""Runs a test case for a direct connection not going through a proxy. """Runs a test case for a direct connection not going through a proxy.
Args: Args:
agent (ProxyAgent): the proxy agent being tested agent: the proxy agent being tested
scheme (bytes): expected to be either "http" or "https" scheme: expected to be either "http" or "https"
hostname (bytes): the hostname to connect to in the test hostname: the hostname to connect to in the test
path (bytes): the path to connect to in the test path: the path to connect to in the test
""" """
is_https = scheme == b"https" is_https = scheme == b"https"
@ -208,7 +392,7 @@ class MatrixFederationAgentTests(TestCase):
""" """
Tests that requests can be made through a proxy. Tests that requests can be made through a proxy.
""" """
self._do_http_request_via_proxy(auth_credentials=None) self._do_http_request_via_proxy(ssl=False, auth_credentials=None)
@patch.dict( @patch.dict(
os.environ, os.environ,
@ -218,12 +402,28 @@ class MatrixFederationAgentTests(TestCase):
""" """
Tests that authenticated requests can be made through a proxy. Tests that authenticated requests can be made through a proxy.
""" """
self._do_http_request_via_proxy(auth_credentials="bob:pinkponies") self._do_http_request_via_proxy(ssl=False, auth_credentials=b"bob:pinkponies")
@patch.dict(
os.environ, {"http_proxy": "https://proxy.com:8888", "no_proxy": "unused.com"}
)
def test_http_request_via_https_proxy(self):
self._do_http_request_via_proxy(ssl=True, auth_credentials=None)
@patch.dict(
os.environ,
{
"http_proxy": "https://bob:pinkponies@proxy.com:8888",
"no_proxy": "unused.com",
},
)
def test_http_request_via_https_proxy_with_auth(self):
self._do_http_request_via_proxy(ssl=True, auth_credentials=b"bob:pinkponies")
@patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"}) @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"})
def test_https_request_via_proxy(self): def test_https_request_via_proxy(self):
"""Tests that TLS-encrypted requests can be made through a proxy""" """Tests that TLS-encrypted requests can be made through a proxy"""
self._do_https_request_via_proxy(auth_credentials=None) self._do_https_request_via_proxy(ssl=False, auth_credentials=None)
@patch.dict( @patch.dict(
os.environ, os.environ,
@ -231,15 +431,39 @@ class MatrixFederationAgentTests(TestCase):
) )
def test_https_request_via_proxy_with_auth(self): def test_https_request_via_proxy_with_auth(self):
"""Tests that authenticated, TLS-encrypted requests can be made through a proxy""" """Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
self._do_https_request_via_proxy(auth_credentials="bob:pinkponies") self._do_https_request_via_proxy(ssl=False, auth_credentials=b"bob:pinkponies")
@patch.dict(
os.environ, {"https_proxy": "https://proxy.com", "no_proxy": "unused.com"}
)
def test_https_request_via_https_proxy(self):
"""Tests that TLS-encrypted requests can be made through a proxy"""
self._do_https_request_via_proxy(ssl=True, auth_credentials=None)
@patch.dict(
os.environ,
{"https_proxy": "https://bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
)
def test_https_request_via_https_proxy_with_auth(self):
"""Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
self._do_https_request_via_proxy(ssl=True, auth_credentials=b"bob:pinkponies")
def _do_http_request_via_proxy( def _do_http_request_via_proxy(
self, self,
auth_credentials: Optional[str] = None, ssl: bool = False,
auth_credentials: Optional[bytes] = None,
): ):
"""Send a http request via an agent and check that it is correctly received at
the proxy. The proxy can use either http or https.
Args:
ssl: True if we expect the request to connect via https to proxy
auth_credentials: credentials to authenticate at proxy
""" """
Tests that requests can be made through a proxy. if ssl:
""" agent = ProxyAgent(
self.reactor, use_proxy=True, contextFactory=get_test_https_policy()
)
else:
agent = ProxyAgent(self.reactor, use_proxy=True) agent = ProxyAgent(self.reactor, use_proxy=True)
self.reactor.lookups["proxy.com"] = "1.2.3.5" self.reactor.lookups["proxy.com"] = "1.2.3.5"
@ -254,7 +478,11 @@ class MatrixFederationAgentTests(TestCase):
# make a test server, and wire up the client # make a test server, and wire up the client
http_server = self._make_connection( http_server = self._make_connection(
client_factory, _get_test_protocol_factory() client_factory,
_get_test_protocol_factory(),
ssl=ssl,
tls_sanlist=[b"DNS:proxy.com"] if ssl else None,
expected_sni=b"proxy.com" if ssl else None,
) )
# the FakeTransport is async, so we need to pump the reactor # the FakeTransport is async, so we need to pump the reactor
@ -272,7 +500,7 @@ class MatrixFederationAgentTests(TestCase):
if auth_credentials is not None: if auth_credentials is not None:
# Compute the correct header value for Proxy-Authorization # Compute the correct header value for Proxy-Authorization
encoded_credentials = base64.b64encode(b"bob:pinkponies") encoded_credentials = base64.b64encode(auth_credentials)
expected_header_value = b"Basic " + encoded_credentials expected_header_value = b"Basic " + encoded_credentials
# Validate the header's value # Validate the header's value
@ -295,8 +523,15 @@ class MatrixFederationAgentTests(TestCase):
def _do_https_request_via_proxy( def _do_https_request_via_proxy(
self, self,
auth_credentials: Optional[str] = None, ssl: bool = False,
auth_credentials: Optional[bytes] = None,
): ):
"""Send a https request via an agent and check that it is correctly received at
the proxy and client. The proxy can use either http or https.
Args:
ssl: True if we expect the request to connect via https to proxy
auth_credentials: credentials to authenticate at proxy
"""
agent = ProxyAgent( agent = ProxyAgent(
self.reactor, self.reactor,
contextFactory=get_test_https_policy(), contextFactory=get_test_https_policy(),
@ -313,18 +548,15 @@ class MatrixFederationAgentTests(TestCase):
self.assertEqual(host, "1.2.3.5") self.assertEqual(host, "1.2.3.5")
self.assertEqual(port, 1080) self.assertEqual(port, 1080)
# make a test HTTP server, and wire up the client # make a test server to act as the proxy, and wire up the client
proxy_server = self._make_connection( proxy_server = self._make_connection(
client_factory, _get_test_protocol_factory() client_factory,
_get_test_protocol_factory(),
ssl=ssl,
tls_sanlist=[b"DNS:proxy.com"] if ssl else None,
expected_sni=b"proxy.com" if ssl else None,
) )
assert isinstance(proxy_server, HTTPChannel)
# fish the transports back out so that we can do the old switcheroo
s2c_transport = proxy_server.transport
client_protocol = s2c_transport.other
c2s_transport = client_protocol.transport
# the FakeTransport is async, so we need to pump the reactor
self.reactor.advance(0)
# now there should be a pending CONNECT request # now there should be a pending CONNECT request
self.assertEqual(len(proxy_server.requests), 1) self.assertEqual(len(proxy_server.requests), 1)
@ -340,7 +572,7 @@ class MatrixFederationAgentTests(TestCase):
if auth_credentials is not None: if auth_credentials is not None:
# Compute the correct header value for Proxy-Authorization # Compute the correct header value for Proxy-Authorization
encoded_credentials = base64.b64encode(b"bob:pinkponies") encoded_credentials = base64.b64encode(auth_credentials)
expected_header_value = b"Basic " + encoded_credentials expected_header_value = b"Basic " + encoded_credentials
# Validate the header's value # Validate the header's value
@ -352,31 +584,49 @@ class MatrixFederationAgentTests(TestCase):
# tell the proxy server not to close the connection # tell the proxy server not to close the connection
proxy_server.persistent = True proxy_server.persistent = True
# this just stops the http Request trying to do a chunked response
# request.setHeader(b"Content-Length", b"0")
request.finish() request.finish()
# now we can replace the proxy channel with a new, SSL-wrapped HTTP channel # now we make another test server to act as the upstream HTTP server.
ssl_factory = _wrap_server_factory_for_tls(_get_test_protocol_factory()) server_ssl_protocol = _wrap_server_factory_for_tls(
ssl_protocol = ssl_factory.buildProtocol(None) _get_test_protocol_factory()
http_server = ssl_protocol.wrappedProtocol ).buildProtocol(None)
ssl_protocol.makeConnection( # Tell the HTTP server to send outgoing traffic back via the proxy's transport.
FakeTransport(client_protocol, self.reactor, ssl_protocol) proxy_server_transport = proxy_server.transport
) server_ssl_protocol.makeConnection(proxy_server_transport)
c2s_transport.other = ssl_protocol
# ... and replace the protocol on the proxy's transport with the
# TLSMemoryBIOProtocol for the test server, so that incoming traffic
# to the proxy gets sent over to the HTTP(s) server.
#
# This needs a bit of gut-wrenching, which is different depending on whether
# the proxy is using TLS or not.
#
# (an alternative, possibly more elegant, approach would be to use a custom
# Protocol to implement the proxy, which starts out by forwarding to an
# HTTPChannel (to implement the CONNECT command) and can then be switched
# into a mode where it forwards its traffic to another Protocol.)
if ssl:
assert isinstance(proxy_server_transport, TLSMemoryBIOProtocol)
proxy_server_transport.wrappedProtocol = server_ssl_protocol
else:
assert isinstance(proxy_server_transport, FakeTransport)
client_protocol = proxy_server_transport.other
c2s_transport = client_protocol.transport
c2s_transport.other = server_ssl_protocol
self.reactor.advance(0) self.reactor.advance(0)
server_name = ssl_protocol._tlsConnection.get_servername() server_name = server_ssl_protocol._tlsConnection.get_servername()
expected_sni = b"test.com" expected_sni = b"test.com"
self.assertEqual( self.assertEqual(
server_name, server_name,
expected_sni, expected_sni,
"Expected SNI %s but got %s" % (expected_sni, server_name), f"Expected SNI {expected_sni!s} but got {server_name!s}",
) )
# now there should be a pending request # now there should be a pending request
http_server = server_ssl_protocol.wrappedProtocol
self.assertEqual(len(http_server.requests), 1) self.assertEqual(len(http_server.requests), 1)
request = http_server.requests[0] request = http_server.requests[0]
@ -510,7 +760,7 @@ class MatrixFederationAgentTests(TestCase):
self.assertEqual( self.assertEqual(
server_name, server_name,
expected_sni, expected_sni,
"Expected SNI %s but got %s" % (expected_sni, server_name), f"Expected SNI {expected_sni!s} but got {server_name!s}",
) )
# now there should be a pending request # now there should be a pending request
@ -529,16 +779,48 @@ class MatrixFederationAgentTests(TestCase):
body = self.successResultOf(treq.content(resp)) body = self.successResultOf(treq.content(resp))
self.assertEqual(body, b"result") self.assertEqual(body, b"result")
@patch.dict(os.environ, {"http_proxy": "proxy.com:8888"})
def test_proxy_with_no_scheme(self):
http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
self.assertIsInstance(http_proxy_agent.http_proxy_endpoint, HostnameEndpoint)
self.assertEqual(http_proxy_agent.http_proxy_endpoint._hostStr, "proxy.com")
self.assertEqual(http_proxy_agent.http_proxy_endpoint._port, 8888)
def _wrap_server_factory_for_tls(factory, sanlist=None): @patch.dict(os.environ, {"http_proxy": "socks://proxy.com:8888"})
def test_proxy_with_unsupported_scheme(self):
with self.assertRaises(ValueError):
ProxyAgent(self.reactor, use_proxy=True)
@patch.dict(os.environ, {"http_proxy": "http://proxy.com:8888"})
def test_proxy_with_http_scheme(self):
http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
self.assertIsInstance(http_proxy_agent.http_proxy_endpoint, HostnameEndpoint)
self.assertEqual(http_proxy_agent.http_proxy_endpoint._hostStr, "proxy.com")
self.assertEqual(http_proxy_agent.http_proxy_endpoint._port, 8888)
@patch.dict(os.environ, {"http_proxy": "https://proxy.com:8888"})
def test_proxy_with_https_scheme(self):
https_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
self.assertIsInstance(https_proxy_agent.http_proxy_endpoint, _WrapperEndpoint)
self.assertEqual(
https_proxy_agent.http_proxy_endpoint._wrappedEndpoint._hostStr, "proxy.com"
)
self.assertEqual(
https_proxy_agent.http_proxy_endpoint._wrappedEndpoint._port, 8888
)
def _wrap_server_factory_for_tls(
factory: IProtocolFactory, sanlist: Iterable[bytes] = None
) -> IProtocolFactory:
"""Wrap an existing Protocol Factory with a test TLSMemoryBIOFactory """Wrap an existing Protocol Factory with a test TLSMemoryBIOFactory
The resultant factory will create a TLS server which presents a certificate The resultant factory will create a TLS server which presents a certificate
signed by our test CA, valid for the domains in `sanlist` signed by our test CA, valid for the domains in `sanlist`
Args: Args:
factory (interfaces.IProtocolFactory): protocol factory to wrap factory: protocol factory to wrap
sanlist (iterable[bytes]): list of domains the cert should be valid for sanlist: list of domains the cert should be valid for
Returns: Returns:
interfaces.IProtocolFactory interfaces.IProtocolFactory
@ -552,7 +834,7 @@ def _wrap_server_factory_for_tls(factory, sanlist=None):
) )
def _get_test_protocol_factory(): def _get_test_protocol_factory() -> IProtocolFactory:
"""Get a protocol Factory which will build an HTTPChannel """Get a protocol Factory which will build an HTTPChannel
Returns: Returns:
@ -566,6 +848,6 @@ def _get_test_protocol_factory():
return server_factory return server_factory
def _log_request(request): def _log_request(request: str):
"""Implements Factory.log, which is expected by Request.finish""" """Implements Factory.log, which is expected by Request.finish"""
logger.info("Completed request %s", request) logger.info(f"Completed request {request}")