Matrix-Mirrors/anonymousland-synapse
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse.git synced 2025-12-15 23:13:50 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/develop' into send_sni_for_federation_requests

Browse source 
# Conflicts:
#	synapse/crypto/context_factory.py
This commit is contained in:
Jeroen 2018-07-14 20:24:46 +02:00
commit 505530f36a
347 changed files with 2249 additions and 2123 deletions
Download patch file Download diff file Expand all files Collapse all files

3
synapse/http/additional_resource.py
View file

@ -13,10 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse.http.server import wrap_json_request_handler
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET
from synapse.http.server import wrap_json_request_handler
class AdditionalResource(Resource):
"""Resource wrapper for additional_resources

62
synapse/http/client.py
View file

@ -13,39 +13,41 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from OpenSSL import SSL
from OpenSSL.SSL import VERIFY_NONE
from synapse.api.errors import (
CodeMessageException, MatrixCodeMessageException, SynapseError, Codes,
)
from synapse.http import cancelled_to_request_timed_out_error, redact_uri
from synapse.util.async import add_timeout_to_deferred
from synapse.util.caches import CACHE_SIZE_FACTOR
from synapse.util.logcontext import make_deferred_yieldable
from synapse.http.endpoint import SpiderEndpoint
from canonicaljson import encode_canonical_json
from twisted.internet import defer, reactor, ssl, protocol, task
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
from twisted.web.client import (
BrowserLikeRedirectAgent, ContentDecoderAgent, GzipDecoder, Agent,
readBody, PartialDownloadError,
HTTPConnectionPool,
)
from twisted.web.client import FileBodyProducer as TwistedFileBodyProducer
from twisted.web.http import PotentialDataLoss
from twisted.web.http_headers import Headers
from twisted.web._newclient import ResponseDone
from six import StringIO
from prometheus_client import Counter
from canonicaljson import json
import logging
import urllib
from six import StringIO
from canonicaljson import encode_canonical_json, json
from prometheus_client import Counter
from OpenSSL import SSL
from OpenSSL.SSL import VERIFY_NONE
from twisted.internet import defer, protocol, reactor, ssl, task
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
from twisted.web._newclient import ResponseDone
from twisted.web.client import Agent, BrowserLikeRedirectAgent, ContentDecoderAgent
from twisted.web.client import FileBodyProducer as TwistedFileBodyProducer
from twisted.web.client import (
GzipDecoder,
HTTPConnectionPool,
PartialDownloadError,
readBody,
)
from twisted.web.http import PotentialDataLoss
from twisted.web.http_headers import Headers
from synapse.api.errors import (
CodeMessageException,
Codes,
MatrixCodeMessageException,
SynapseError,
)
from synapse.http import cancelled_to_request_timed_out_error, redact_uri
from synapse.http.endpoint import SpiderEndpoint
from synapse.util.async import add_timeout_to_deferred
from synapse.util.caches import CACHE_SIZE_FACTOR
from synapse.util.logcontext import make_deferred_yieldable
logger = logging.getLogger(__name__)

14
synapse/http/endpoint.py
View file

@ -12,19 +12,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
from twisted.internet import defer
from twisted.internet.error import ConnectError
from twisted.names import client, dns
from twisted.names.error import DNSNameError, DomainError
import collections
import logging
import random
import re
import time
from twisted.internet import defer
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
from twisted.internet.error import ConnectError
from twisted.names import client, dns
from twisted.names.error import DNSNameError, DomainError
logger = logging.getLogger(__name__)

48
synapse/http/matrixfederationclient.py
View file

@ -13,38 +13,38 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from twisted.internet import defer, reactor, protocol
from twisted.internet.error import DNSLookupError
from twisted.web.client import readBody, HTTPConnectionPool, Agent
from twisted.web.http_headers import Headers
from twisted.web._newclient import ResponseDone
from synapse.http import cancelled_to_request_timed_out_error
from synapse.http.endpoint import matrix_federation_endpoint
import synapse.metrics
from synapse.util.async import add_timeout_to_deferred
from synapse.util import logcontext
from synapse.util.logcontext import make_deferred_yieldable
import synapse.util.retryutils
from canonicaljson import encode_canonical_json, json
from synapse.api.errors import (
SynapseError, Codes, HttpResponseException, FederationDeniedError,
)
from signedjson.sign import sign_json
import cgi
import logging
import random
import sys
import urllib
from six.moves.urllib import parse as urlparse
from six import string_types
from six.moves.urllib import parse as urlparse
from canonicaljson import encode_canonical_json, json
from prometheus_client import Counter
from signedjson.sign import sign_json
from twisted.internet import defer, protocol, reactor
from twisted.internet.error import DNSLookupError
from twisted.web._newclient import ResponseDone
from twisted.web.client import Agent, HTTPConnectionPool, readBody
from twisted.web.http_headers import Headers
import synapse.metrics
import synapse.util.retryutils
from synapse.api.errors import (
Codes,
FederationDeniedError,
HttpResponseException,
SynapseError,
)
from synapse.http import cancelled_to_request_timed_out_error
from synapse.http.endpoint import matrix_federation_endpoint
from synapse.util import logcontext
from synapse.util.async import add_timeout_to_deferred
from synapse.util.logcontext import make_deferred_yieldable
logger = logging.getLogger(__name__)
outbound_logger = logging.getLogger("synapse.http.outbound")

85
synapse/http/request_metrics.py
View file

@ -17,8 +17,8 @@
import logging
from prometheus_client.core import Counter, Histogram
from synapse.metrics import LaterGauge
from synapse.metrics import LaterGauge
from synapse.util.logcontext import LoggingContext
logger = logging.getLogger(__name__)
@ -149,7 +149,9 @@ class RequestMetrics(object):
self.name = name
self.method = method
self._request_stats = _RequestStats.from_context(self.start_context)
# _request_stats records resource usage that we have already added
# to the "in flight" metrics.
self._request_stats = self.start_context.get_resource_usage()
_in_flight_requests.add(self)
@ -177,18 +179,22 @@ class RequestMetrics(object):
time_sec - self.start
)
ru_utime, ru_stime = context.get_resource_usage()
resource_usage = context.get_resource_usage()
response_ru_utime.labels(request.method, self.name, tag).inc(ru_utime)
response_ru_stime.labels(request.method, self.name, tag).inc(ru_stime)
response_ru_utime.labels(request.method, self.name, tag).inc(
resource_usage.ru_utime,
)
response_ru_stime.labels(request.method, self.name, tag).inc(
resource_usage.ru_stime,
)
response_db_txn_count.labels(request.method, self.name, tag).inc(
context.db_txn_count
resource_usage.db_txn_count
)
response_db_txn_duration.labels(request.method, self.name, tag).inc(
context.db_txn_duration_sec
resource_usage.db_txn_duration_sec
)
response_db_sched_duration.labels(request.method, self.name, tag).inc(
context.db_sched_duration_sec
resource_usage.db_sched_duration_sec
)
response_size.labels(request.method, self.name, tag).inc(request.sentLength)
@ -201,7 +207,10 @@ class RequestMetrics(object):
def update_metrics(self):
"""Updates the in flight metrics with values from this request.
"""
diff = self._request_stats.update(self.start_context)
new_stats = self.start_context.get_resource_usage()
diff = new_stats - self._request_stats
self._request_stats = new_stats
in_flight_requests_ru_utime.labels(self.method, self.name).inc(diff.ru_utime)
in_flight_requests_ru_stime.labels(self.method, self.name).inc(diff.ru_stime)
@ -217,61 +226,3 @@ class RequestMetrics(object):
in_flight_requests_db_sched_duration.labels(self.method, self.name).inc(
diff.db_sched_duration_sec
)
class _RequestStats(object):
"""Keeps tracks of various metrics for an in flight request.
"""
__slots__ = [
"ru_utime",
"ru_stime",
"db_txn_count",
"db_txn_duration_sec",
"db_sched_duration_sec",
]
def __init__(
self, ru_utime, ru_stime, db_txn_count, db_txn_duration_sec, db_sched_duration_sec
):
self.ru_utime = ru_utime
self.ru_stime = ru_stime
self.db_txn_count = db_txn_count
self.db_txn_duration_sec = db_txn_duration_sec
self.db_sched_duration_sec = db_sched_duration_sec
@staticmethod
def from_context(context):
ru_utime, ru_stime = context.get_resource_usage()
return _RequestStats(
ru_utime, ru_stime,
context.db_txn_count,
context.db_txn_duration_sec,
context.db_sched_duration_sec,
)
def update(self, context):
"""Updates the current values and returns the difference between the
old and new values.
Returns:
_RequestStats: The difference between the old and new values
"""
new = _RequestStats.from_context(context)
diff = _RequestStats(
new.ru_utime - self.ru_utime,
new.ru_stime - self.ru_stime,
new.db_txn_count - self.db_txn_count,
new.db_txn_duration_sec - self.db_txn_duration_sec,
new.db_sched_duration_sec - self.db_sched_duration_sec,
)
self.ru_utime = new.ru_utime
self.ru_stime = new.ru_stime
self.db_txn_count = new.db_txn_count
self.db_txn_duration_sec = new.db_txn_duration_sec
self.db_sched_duration_sec = new.db_sched_duration_sec
return diff

48
synapse/http/server.py
View file

@ -14,34 +14,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import cgi
from six.moves import http_client
from synapse.api.errors import (
cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes
)
from synapse.http.request_metrics import (
requests_counter,
)
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
from synapse.util.caches import intern_dict
from synapse.util.metrics import Measure
import synapse.metrics
import synapse.events
from canonicaljson import (
encode_canonical_json, encode_pretty_printed_json, json
)
from twisted.internet import defer
from twisted.python import failure
from twisted.web import server, resource
from twisted.web.server import NOT_DONE_YET
from twisted.web.util import redirectTo
import collections
import logging
import urllib
from six.moves import http_client
from canonicaljson import encode_canonical_json, encode_pretty_printed_json, json
from twisted.internet import defer
from twisted.python import failure
from twisted.web import resource, server
from twisted.web.server import NOT_DONE_YET
from twisted.web.util import redirectTo
import synapse.events
import synapse.metrics
from synapse.api.errors import (
CodeMessageException,
Codes,
SynapseError,
UnrecognizedRequestError,
cs_exception,
)
from synapse.http.request_metrics import requests_counter
from synapse.util.caches import intern_dict
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
from synapse.util.metrics import Measure
logger = logging.getLogger(__name__)
HTML_ERROR_TEMPLATE = """<!DOCTYPE html>

5
synapse/http/servlet.py
View file

@ -15,12 +15,11 @@
""" This module contains base REST classes for constructing REST servlets. """
from synapse.api.errors import SynapseError, Codes
import logging
from canonicaljson import json
from synapse.api.errors import Codes, SynapseError
logger = logging.getLogger(__name__)
@ -207,7 +206,7 @@ def parse_json_object_from_request(request, allow_empty_body=False):
return content
def assert_params_in_request(body, required):
def assert_params_in_dict(body, required):
absent = []
for k in required:
if k not in body:

26
synapse/http/site.py
View file

@ -16,11 +16,11 @@ import contextlib
import logging
import time
from twisted.web.server import Site, Request
from twisted.web.server import Request, Site
from synapse.http import redact_uri
from synapse.http.request_metrics import RequestMetrics
from synapse.util.logcontext import LoggingContext
from synapse.util.logcontext import LoggingContext, ContextResourceUsage
logger = logging.getLogger(__name__)
@ -95,15 +95,9 @@ class SynapseRequest(Request):
def _finished_processing(self):
try:
context = LoggingContext.current_context()
ru_utime, ru_stime = context.get_resource_usage()
db_txn_count = context.db_txn_count
db_txn_duration_sec = context.db_txn_duration_sec
db_sched_duration_sec = context.db_sched_duration_sec
evt_db_fetch_count = context.evt_db_fetch_count
usage = context.get_resource_usage()
except Exception:
ru_utime, ru_stime = (0, 0)
db_txn_count, db_txn_duration_sec = (0, 0)
evt_db_fetch_count = 0
usage = ContextResourceUsage()
end_time = time.time()
@ -130,18 +124,18 @@ class SynapseRequest(Request):
self.site.site_tag,
authenticated_entity,
end_time - self.start_time,
ru_utime,
ru_stime,
db_sched_duration_sec,
db_txn_duration_sec,
int(db_txn_count),
usage.ru_utime,
usage.ru_stime,
usage.db_sched_duration_sec,
usage.db_txn_duration_sec,
int(usage.db_txn_count),
self.sentLength,
self.code,
self.method,
self.get_redacted_uri(),
self.clientproto,
user_agent,
evt_db_fetch_count,
usage.evt_db_fetch_count,
)
try: