2014-08-12 22:32:18 -04:00
|
|
|
#
|
2023-11-21 15:29:58 -05:00
|
|
|
# This file is licensed under the Affero General Public License (AGPL) version 3.
|
|
|
|
#
|
|
|
|
# Copyright (C) 2023 New Vector, Ltd
|
|
|
|
#
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU Affero General Public License as
|
|
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
|
|
# License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# See the GNU Affero General Public License for more details:
|
|
|
|
# <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
|
|
#
|
|
|
|
# Originally licensed under the Apache License, Version 2.0:
|
|
|
|
# <http://www.apache.org/licenses/LICENSE-2.0>.
|
|
|
|
#
|
|
|
|
# [This file includes modifications made by New Vector Limited]
|
2014-08-12 22:32:18 -04:00
|
|
|
#
|
|
|
|
#
|
|
|
|
|
2014-08-12 10:10:52 -04:00
|
|
|
import json
|
|
|
|
import urllib
|
2020-07-20 16:43:49 -04:00
|
|
|
from pprint import pformat
|
2021-04-08 17:38:54 -04:00
|
|
|
from typing import Optional
|
2020-07-20 16:43:49 -04:00
|
|
|
|
|
|
|
from twisted.internet import defer, reactor
|
|
|
|
from twisted.web.client import Agent, readBody
|
|
|
|
from twisted.web.http_headers import Headers
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
|
2020-09-04 06:54:56 -04:00
|
|
|
class HttpClient:
|
2021-02-16 17:32:34 -05:00
|
|
|
"""Interface for talking json over http"""
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
def put_json(self, url, data):
|
2021-02-16 17:32:34 -05:00
|
|
|
"""Sends the specifed json data using PUT
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
Args:
|
|
|
|
url (str): The URL to PUT data to.
|
|
|
|
data (dict): A dict containing the data that will be used as
|
|
|
|
the request body. This will be encoded as JSON.
|
|
|
|
|
|
|
|
Returns:
|
2017-04-20 19:46:54 -04:00
|
|
|
Deferred: Succeeds when we get a 2xx HTTP response. The result
|
|
|
|
will be the decoded JSON body.
|
2014-08-12 10:10:52 -04:00
|
|
|
"""
|
|
|
|
|
|
|
|
def get_json(self, url, args=None):
|
2021-02-16 17:32:34 -05:00
|
|
|
"""Gets some json from the given host homeserver and path
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
Args:
|
|
|
|
url (str): The URL to GET data from.
|
|
|
|
args (dict): A dictionary used to create query strings, defaults to
|
|
|
|
None.
|
|
|
|
**Note**: The value of each key is assumed to be an iterable
|
|
|
|
and *not* a string.
|
|
|
|
|
|
|
|
Returns:
|
2017-04-20 19:46:54 -04:00
|
|
|
Deferred: Succeeds when we get a 2xx HTTP response. The result
|
|
|
|
will be the decoded JSON body.
|
2014-08-12 10:10:52 -04:00
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
class TwistedHttpClient(HttpClient):
|
2021-02-16 17:32:34 -05:00
|
|
|
"""Wrapper around the twisted HTTP client api.
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
Attributes:
|
|
|
|
agent (twisted.web.client.Agent): The twisted Agent used to send the
|
|
|
|
requests.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
self.agent = Agent(reactor)
|
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def put_json(self, url, data):
|
|
|
|
response = yield self._create_put_request(
|
2019-06-20 05:32:02 -04:00
|
|
|
url, data, headers_dict={"Content-Type": ["application/json"]}
|
2014-08-12 10:10:52 -04:00
|
|
|
)
|
|
|
|
body = yield readBody(response)
|
|
|
|
defer.returnValue((response.code, body))
|
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def get_json(self, url, args=None):
|
|
|
|
if args:
|
|
|
|
# generates a list of strings of form "k=v".
|
|
|
|
qs = urllib.urlencode(args, True)
|
|
|
|
url = "%s?%s" % (url, qs)
|
|
|
|
response = yield self._create_get_request(url)
|
|
|
|
body = yield readBody(response)
|
|
|
|
defer.returnValue(json.loads(body))
|
|
|
|
|
2021-04-08 17:38:54 -04:00
|
|
|
def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
|
2021-02-16 17:32:34 -05:00
|
|
|
"""Wrapper of _create_request to issue a PUT request"""
|
2021-04-08 17:38:54 -04:00
|
|
|
headers_dict = headers_dict or {}
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
if "Content-Type" not in headers_dict:
|
2019-06-20 05:32:02 -04:00
|
|
|
raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
return self._create_request(
|
2019-06-20 05:32:02 -04:00
|
|
|
"PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
|
2014-08-12 10:10:52 -04:00
|
|
|
)
|
|
|
|
|
2021-04-08 17:38:54 -04:00
|
|
|
def _create_get_request(self, url, headers_dict: Optional[dict] = None):
|
2021-02-16 17:32:34 -05:00
|
|
|
"""Wrapper of _create_request to issue a GET request"""
|
2021-04-08 17:38:54 -04:00
|
|
|
return self._create_request("GET", url, headers_dict=headers_dict or {})
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
2019-06-20 05:32:02 -04:00
|
|
|
def do_request(
|
2021-04-08 17:38:54 -04:00
|
|
|
self,
|
|
|
|
method,
|
|
|
|
url,
|
|
|
|
data=None,
|
|
|
|
qparams=None,
|
|
|
|
jsonreq=True,
|
|
|
|
headers: Optional[dict] = None,
|
2019-06-20 05:32:02 -04:00
|
|
|
):
|
2021-04-08 17:38:54 -04:00
|
|
|
headers = headers or {}
|
|
|
|
|
2014-08-12 10:10:52 -04:00
|
|
|
if qparams:
|
|
|
|
url = "%s?%s" % (url, urllib.urlencode(qparams, True))
|
|
|
|
|
|
|
|
if jsonreq:
|
|
|
|
prod = _JsonProducer(data)
|
2019-06-20 05:32:02 -04:00
|
|
|
headers["Content-Type"] = ["application/json"]
|
2014-08-12 10:10:52 -04:00
|
|
|
else:
|
|
|
|
prod = _RawProducer(data)
|
|
|
|
|
|
|
|
if method in ["POST", "PUT"]:
|
2019-06-20 05:32:02 -04:00
|
|
|
response = yield self._create_request(
|
|
|
|
method, url, producer=prod, headers_dict=headers
|
|
|
|
)
|
2014-08-12 10:10:52 -04:00
|
|
|
else:
|
|
|
|
response = yield self._create_request(method, url)
|
|
|
|
|
|
|
|
body = yield readBody(response)
|
|
|
|
defer.returnValue(json.loads(body))
|
|
|
|
|
|
|
|
@defer.inlineCallbacks
|
2021-04-08 17:38:54 -04:00
|
|
|
def _create_request(
|
|
|
|
self, method, url, producer=None, headers_dict: Optional[dict] = None
|
|
|
|
):
|
2021-02-16 17:32:34 -05:00
|
|
|
"""Creates and sends a request to the given url"""
|
2021-04-08 17:38:54 -04:00
|
|
|
headers_dict = headers_dict or {}
|
|
|
|
|
2014-08-12 10:10:52 -04:00
|
|
|
headers_dict["User-Agent"] = ["Synapse Cmd Client"]
|
|
|
|
|
|
|
|
retries_left = 5
|
2019-06-17 13:21:30 -04:00
|
|
|
print("%s to %s with headers %s" % (method, url, headers_dict))
|
2014-08-12 10:10:52 -04:00
|
|
|
if self.verbose and producer:
|
|
|
|
if "password" in producer.data:
|
|
|
|
temp = producer.data["password"]
|
|
|
|
producer.data["password"] = "[REDACTED]"
|
2019-06-17 13:21:30 -04:00
|
|
|
print(json.dumps(producer.data, indent=4))
|
2014-08-12 10:10:52 -04:00
|
|
|
producer.data["password"] = temp
|
|
|
|
else:
|
2019-06-17 13:21:30 -04:00
|
|
|
print(json.dumps(producer.data, indent=4))
|
2014-08-12 10:10:52 -04:00
|
|
|
|
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
response = yield self.agent.request(
|
2019-06-20 05:32:02 -04:00
|
|
|
method, url.encode("UTF8"), Headers(headers_dict), producer
|
2014-08-12 10:10:52 -04:00
|
|
|
)
|
|
|
|
break
|
|
|
|
except Exception as e:
|
2019-06-17 13:21:30 -04:00
|
|
|
print("uh oh: %s" % e)
|
2014-08-12 10:10:52 -04:00
|
|
|
if retries_left:
|
|
|
|
yield self.sleep(2 ** (5 - retries_left))
|
|
|
|
retries_left -= 1
|
|
|
|
else:
|
|
|
|
raise e
|
|
|
|
|
|
|
|
if self.verbose:
|
2019-06-17 13:21:30 -04:00
|
|
|
print("Status %s %s" % (response.code, response.phrase))
|
|
|
|
print(pformat(list(response.headers.getAllRawHeaders())))
|
2014-08-12 10:10:52 -04:00
|
|
|
defer.returnValue(response)
|
|
|
|
|
|
|
|
def sleep(self, seconds):
|
|
|
|
d = defer.Deferred()
|
|
|
|
reactor.callLater(seconds, d.callback, seconds)
|
|
|
|
return d
|
|
|
|
|
2019-06-20 05:32:02 -04:00
|
|
|
|
2020-09-04 06:54:56 -04:00
|
|
|
class _RawProducer:
|
2014-08-12 10:10:52 -04:00
|
|
|
def __init__(self, data):
|
|
|
|
self.data = data
|
|
|
|
self.body = data
|
|
|
|
self.length = len(self.body)
|
|
|
|
|
|
|
|
def startProducing(self, consumer):
|
|
|
|
consumer.write(self.body)
|
|
|
|
return defer.succeed(None)
|
|
|
|
|
|
|
|
def pauseProducing(self):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def stopProducing(self):
|
|
|
|
pass
|
|
|
|
|
2019-06-20 05:32:02 -04:00
|
|
|
|
2020-09-04 06:54:56 -04:00
|
|
|
class _JsonProducer:
|
2021-02-16 17:32:34 -05:00
|
|
|
"""Used by the twisted http client to create the HTTP body from json"""
|
2019-06-20 05:32:02 -04:00
|
|
|
|
2014-08-12 10:10:52 -04:00
|
|
|
def __init__(self, jsn):
|
|
|
|
self.data = jsn
|
|
|
|
self.body = json.dumps(jsn).encode("utf8")
|
|
|
|
self.length = len(self.body)
|
|
|
|
|
|
|
|
def startProducing(self, consumer):
|
|
|
|
consumer.write(self.body)
|
|
|
|
return defer.succeed(None)
|
|
|
|
|
|
|
|
def pauseProducing(self):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def stopProducing(self):
|
2017-04-20 19:46:54 -04:00
|
|
|
pass
|