mirror of
https://git.anonymousland.org/anonymousland/synapse.git
synced 2025-05-03 02:14:49 -04:00
Port http/ to Python 3 (#3771)
This commit is contained in:
Amber Brown
GitHub
parent
a6cf7d9d9a
commit
2d2828dcbc
8 changed files with 134 additions and 186 deletions
|
|
@ -13,24 +13,25 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import urllib
|
||||
|
||||
from six import StringIO
|
||||
from six import text_type
|
||||
from six.moves import urllib
|
||||
|
||||
import treq
|
||||
from canonicaljson import encode_canonical_json, json
|
||||
from prometheus_client import Counter
|
||||
|
||||
from OpenSSL import SSL
|
||||
from OpenSSL.SSL import VERIFY_NONE
|
||||
from twisted.internet import defer, protocol, reactor, ssl, task
|
||||
from twisted.internet import defer, protocol, reactor, ssl
|
||||
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
|
||||
from twisted.web._newclient import ResponseDone
|
||||
from twisted.web.client import (
|
||||
Agent,
|
||||
BrowserLikeRedirectAgent,
|
||||
ContentDecoderAgent,
|
||||
FileBodyProducer as TwistedFileBodyProducer,
|
||||
GzipDecoder,
|
||||
HTTPConnectionPool,
|
||||
PartialDownloadError,
|
||||
|
|
@ -83,18 +84,20 @@ class SimpleHttpClient(object):
|
|||
if hs.config.user_agent_suffix:
|
||||
self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,)
|
||||
|
||||
self.user_agent = self.user_agent.encode('ascii')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def request(self, method, uri, *args, **kwargs):
|
||||
def request(self, method, uri, data=b'', headers=None):
|
||||
# A small wrapper around self.agent.request() so we can easily attach
|
||||
# counters to it
|
||||
outgoing_requests_counter.labels(method).inc()
|
||||
|
||||
# log request but strip `access_token` (AS requests for example include this)
|
||||
logger.info("Sending request %s %s", method, redact_uri(uri))
|
||||
logger.info("Sending request %s %s", method, redact_uri(uri.encode('ascii')))
|
||||
|
||||
try:
|
||||
request_deferred = self.agent.request(
|
||||
method, uri, *args, **kwargs
|
||||
request_deferred = treq.request(
|
||||
method, uri, agent=self.agent, data=data, headers=headers
|
||||
)
|
||||
add_timeout_to_deferred(
|
||||
request_deferred, 60, self.hs.get_reactor(),
|
||||
|
|
@ -105,14 +108,14 @@ class SimpleHttpClient(object):
|
|||
incoming_responses_counter.labels(method, response.code).inc()
|
||||
logger.info(
|
||||
"Received response to %s %s: %s",
|
||||
method, redact_uri(uri), response.code
|
||||
method, redact_uri(uri.encode('ascii')), response.code
|
||||
)
|
||||
defer.returnValue(response)
|
||||
except Exception as e:
|
||||
incoming_responses_counter.labels(method, "ERR").inc()
|
||||
logger.info(
|
||||
"Error sending request to %s %s: %s %s",
|
||||
method, redact_uri(uri), type(e).__name__, e.message
|
||||
method, redact_uri(uri.encode('ascii')), type(e).__name__, e.args[0]
|
||||
)
|
||||
raise
|
||||
|
||||
|
|
@ -137,7 +140,8 @@ class SimpleHttpClient(object):
|
|||
# TODO: Do we ever want to log message contents?
|
||||
logger.debug("post_urlencoded_get_json args: %s", args)
|
||||
|
||||
query_bytes = urllib.urlencode(encode_urlencode_args(args), True)
|
||||
query_bytes = urllib.parse.urlencode(
|
||||
encode_urlencode_args(args), True).encode("utf8")
|
||||
|
||||
actual_headers = {
|
||||
b"Content-Type": [b"application/x-www-form-urlencoded"],
|
||||
|
|
@ -148,15 +152,14 @@ class SimpleHttpClient(object):
|
|||
|
||||
response = yield self.request(
|
||||
"POST",
|
||||
uri.encode("ascii"),
|
||||
uri,
|
||||
headers=Headers(actual_headers),
|
||||
bodyProducer=FileBodyProducer(StringIO(query_bytes))
|
||||
data=query_bytes
|
||||
)
|
||||
|
||||
body = yield make_deferred_yieldable(readBody(response))
|
||||
|
||||
if 200 <= response.code < 300:
|
||||
defer.returnValue(json.loads(body))
|
||||
body = yield make_deferred_yieldable(treq.json_content(response))
|
||||
defer.returnValue(body)
|
||||
else:
|
||||
raise HttpResponseException(response.code, response.phrase, body)
|
||||
|
||||
|
|
@ -191,9 +194,9 @@ class SimpleHttpClient(object):
|
|||
|
||||
response = yield self.request(
|
||||
"POST",
|
||||
uri.encode("ascii"),
|
||||
uri,
|
||||
headers=Headers(actual_headers),
|
||||
bodyProducer=FileBodyProducer(StringIO(json_str))
|
||||
data=json_str
|
||||
)
|
||||
|
||||
body = yield make_deferred_yieldable(readBody(response))
|
||||
|
|
@ -248,7 +251,7 @@ class SimpleHttpClient(object):
|
|||
ValueError: if the response was not JSON
|
||||
"""
|
||||
if len(args):
|
||||
query_bytes = urllib.urlencode(args, True)
|
||||
query_bytes = urllib.parse.urlencode(args, True)
|
||||
uri = "%s?%s" % (uri, query_bytes)
|
||||
|
||||
json_str = encode_canonical_json(json_body)
|
||||
|
|
@ -262,9 +265,9 @@ class SimpleHttpClient(object):
|
|||
|
||||
response = yield self.request(
|
||||
"PUT",
|
||||
uri.encode("ascii"),
|
||||
uri,
|
||||
headers=Headers(actual_headers),
|
||||
bodyProducer=FileBodyProducer(StringIO(json_str))
|
||||
data=json_str
|
||||
)
|
||||
|
||||
body = yield make_deferred_yieldable(readBody(response))
|
||||
|
|
@ -293,7 +296,7 @@ class SimpleHttpClient(object):
|
|||
HttpResponseException on a non-2xx HTTP response.
|
||||
"""
|
||||
if len(args):
|
||||
query_bytes = urllib.urlencode(args, True)
|
||||
query_bytes = urllib.parse.urlencode(args, True)
|
||||
uri = "%s?%s" % (uri, query_bytes)
|
||||
|
||||
actual_headers = {
|
||||
|
|
@ -304,7 +307,7 @@ class SimpleHttpClient(object):
|
|||
|
||||
response = yield self.request(
|
||||
"GET",
|
||||
uri.encode("ascii"),
|
||||
uri,
|
||||
headers=Headers(actual_headers),
|
||||
)
|
||||
|
||||
|
|
@ -339,7 +342,7 @@ class SimpleHttpClient(object):
|
|||
|
||||
response = yield self.request(
|
||||
"GET",
|
||||
url.encode("ascii"),
|
||||
url,
|
||||
headers=Headers(actual_headers),
|
||||
)
|
||||
|
||||
|
|
@ -434,12 +437,12 @@ class CaptchaServerHttpClient(SimpleHttpClient):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def post_urlencoded_get_raw(self, url, args={}):
|
||||
query_bytes = urllib.urlencode(encode_urlencode_args(args), True)
|
||||
query_bytes = urllib.parse.urlencode(encode_urlencode_args(args), True)
|
||||
|
||||
response = yield self.request(
|
||||
"POST",
|
||||
url.encode("ascii"),
|
||||
bodyProducer=FileBodyProducer(StringIO(query_bytes)),
|
||||
url,
|
||||
data=query_bytes,
|
||||
headers=Headers({
|
||||
b"Content-Type": [b"application/x-www-form-urlencoded"],
|
||||
b"User-Agent": [self.user_agent],
|
||||
|
|
@ -510,7 +513,7 @@ def encode_urlencode_args(args):
|
|||
|
||||
|
||||
def encode_urlencode_arg(arg):
|
||||
if isinstance(arg, unicode):
|
||||
if isinstance(arg, text_type):
|
||||
return arg.encode('utf-8')
|
||||
elif isinstance(arg, list):
|
||||
return [encode_urlencode_arg(i) for i in arg]
|
||||
|
|
@ -542,26 +545,3 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory):
|
|||
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
return self
|
||||
|
||||
|
||||
class FileBodyProducer(TwistedFileBodyProducer):
|
||||
"""Workaround for https://twistedmatrix.com/trac/ticket/8473
|
||||
|
||||
We override the pauseProducing and resumeProducing methods in twisted's
|
||||
FileBodyProducer so that they do not raise exceptions if the task has
|
||||
already completed.
|
||||
"""
|
||||
|
||||
def pauseProducing(self):
|
||||
try:
|
||||
super(FileBodyProducer, self).pauseProducing()
|
||||
except task.TaskDone:
|
||||
# task has already completed
|
||||
pass
|
||||
|
||||
def resumeProducing(self):
|
||||
try:
|
||||
super(FileBodyProducer, self).resumeProducing()
|
||||
except task.NotPaused:
|
||||
# task was not paused (probably because it had already completed)
|
||||
pass
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue