From d94590ed4855691e54655396e45e3e275f9c1860 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 29 May 2015 12:11:40 +0100 Subject: [PATCH 1/6] Add config for setting the recaptcha verify api endpoint, so we can test it in sytest --- synapse/config/captcha.py | 4 ++++ synapse/handlers/auth.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index d8fe577e3..ba221121c 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -26,6 +26,7 @@ class CaptchaConfig(Config): config["captcha_ip_origin_is_x_forwarded"] ) self.captcha_bypass_secret = config.get("captcha_bypass_secret") + self.recaptcha_siteverify_api = config["recaptcha_siteverify_api"] def default_config(self, config_dir_path, server_name): return """\ @@ -48,4 +49,7 @@ class CaptchaConfig(Config): # A secret key used to bypass the captcha test entirely. #captcha_bypass_secret: "YOUR_SECRET_HERE" + + # The API endpoint to use for verifying m.login.recaptcha responses. + recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" """ diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 4e2e50345..4b442a835 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -187,8 +187,8 @@ class AuthHandler(BaseHandler): # each request try: client = SimpleHttpClient(self.hs) - data = yield client.post_urlencoded_get_json( - "https://www.google.com/recaptcha/api/siteverify", + resp_body = yield client.post_urlencoded_get_json( + self.hs.config.recaptcha_siteverify_api, args={ 'secret': self.hs.config.recaptcha_private_key, 'response': user_response, @@ -198,7 +198,7 @@ class AuthHandler(BaseHandler): except PartialDownloadError as pde: # Twisted is silly data = pde.response - resp_body = simplejson.loads(data) + resp_body = simplejson.loads(data) if 'success' in resp_body and resp_body['success']: defer.returnValue(True) raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) From a7b65bdedf512f646a3ca2478fb96a914856de35 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 May 2015 12:17:33 +0100 Subject: [PATCH 2/6] Add config option to turn off freezing events. Use new encode_json api and ujson.loads --- synapse/app/homeserver.py | 4 ++++ synapse/config/server.py | 1 + synapse/events/__init__.py | 11 ++++++++++- synapse/http/server.py | 17 +++++++++++++---- synapse/python_dependencies.py | 1 + synapse/rest/client/v1/__init__.py | 2 +- synapse/rest/client/v2_alpha/__init__.py | 2 +- synapse/storage/events.py | 23 +++++++++++++---------- 8 files changed, 44 insertions(+), 17 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f3513abb5..d93afdc1c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -54,6 +54,8 @@ from synapse.rest.client.v1 import ClientV1RestResource from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource from synapse.metrics.resource import MetricsResource, METRICS_PREFIX +from synapse import events + from daemonize import Daemonize import twisted.manhole.telnet @@ -415,6 +417,8 @@ def setup(config_options): logger.info("Server hostname: %s", config.server_name) logger.info("Server version: %s", version_string) + events.USE_FROZEN_DICTS = config.use_frozen_dicts + if re.search(":[0-9]+$", config.server_name): domain_with_port = config.server_name else: diff --git a/synapse/config/server.py b/synapse/config/server.py index 78195b3a4..48a26c65d 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -28,6 +28,7 @@ class ServerConfig(Config): self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") + self.use_frozen_dicts = config.get("use_frozen_dicts", True) # Attempt to guess the content_addr for the v0 content repostitory content_addr = config.get("content_addr") diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index e4495ccf1..b2a904b40 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -16,6 +16,12 @@ from synapse.util.frozenutils import freeze +# Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents +# bugs where we accidentally share e.g. signature dicts. However, converting +# a dict to frozen_dicts is expensive. +USE_FROZEN_DICTS = True + + class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): self.__dict__ = dict(internal_metadata_dict) @@ -122,7 +128,10 @@ class FrozenEvent(EventBase): unsigned = dict(event_dict.pop("unsigned", {})) - frozen_dict = freeze(event_dict) + if USE_FROZEN_DICTS: + frozen_dict = freeze(event_dict) + else: + frozen_dict = dict(event_dict) super(FrozenEvent, self).__init__( frozen_dict, diff --git a/synapse/http/server.py b/synapse/http/server.py index 73efbff4f..cdb9f9cef 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -19,9 +19,10 @@ from synapse.api.errors import ( ) from synapse.util.logcontext import LoggingContext, PreserveLoggingContext import synapse.metrics +import synapse.events from syutil.jsonutil import ( - encode_canonical_json, encode_pretty_printed_json + encode_canonical_json, encode_pretty_printed_json, encode_json ) from twisted.internet import defer @@ -168,9 +169,10 @@ class JsonResource(HttpServer, resource.Resource): _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"]) - def __init__(self, hs): + def __init__(self, hs, canonical_json=True): resource.Resource.__init__(self) + self.canonical_json = canonical_json self.clock = hs.get_clock() self.path_regexs = {} self.version_string = hs.version_string @@ -256,6 +258,7 @@ class JsonResource(HttpServer, resource.Resource): response_code_message=response_code_message, pretty_print=_request_user_agent_is_curl(request), version_string=self.version_string, + canonical_json=self.canonical_json, ) @@ -277,11 +280,17 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, response_code_message=None, pretty_print=False, - version_string=""): + version_string="", canonical_json=True): if pretty_print: json_bytes = encode_pretty_printed_json(json_object) + "\n" else: - json_bytes = encode_canonical_json(json_object) + if canonical_json: + json_bytes = encode_canonical_json(json_object) + else: + json_bytes = encode_json( + json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS + ) + logger.debug("json_bytes: %r %r", json_object, json_bytes) return respond_with_json_bytes( request, code, json_bytes, diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a45dd3c93..27b740d00 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -30,6 +30,7 @@ REQUIREMENTS = { "frozendict>=0.4": ["frozendict"], "pillow": ["PIL"], "pydenticon": ["pydenticon"], + "ujson": ["ujson"], } CONDITIONAL_REQUIREMENTS = { "web_client": { diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py index 21876b348..cc9b49d53 100644 --- a/synapse/rest/client/v1/__init__.py +++ b/synapse/rest/client/v1/__init__.py @@ -25,7 +25,7 @@ class ClientV1RestResource(JsonResource): """A resource for version 1 of the matrix client API.""" def __init__(self, hs): - JsonResource.__init__(self, hs) + JsonResource.__init__(self, hs, canonical_json=False) self.register_servlets(self, hs) @staticmethod diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 28d95b272..7d1aff430 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -28,7 +28,7 @@ class ClientV2AlphaRestResource(JsonResource): """A resource for version 2 alpha of the matrix client API.""" def __init__(self, hs): - JsonResource.__init__(self, hs) + JsonResource.__init__(self, hs, canonical_json=False) self.register_servlets(self, hs) @staticmethod diff --git a/synapse/storage/events.py b/synapse/storage/events.py index d2a010bd8..20a8d8179 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -17,7 +17,7 @@ from _base import SQLBaseStore, _RollbackButIsFineException from twisted.internet import defer, reactor -from synapse.events import FrozenEvent +from synapse.events import FrozenEvent, USE_FROZEN_DICTS from synapse.events.utils import prune_event from synapse.util.logcontext import preserve_context_over_deferred @@ -26,11 +26,11 @@ from synapse.api.constants import EventTypes from synapse.crypto.event_signing import compute_event_reference_hash from syutil.base64util import decode_base64 -from syutil.jsonutil import encode_canonical_json +from syutil.jsonutil import encode_json from contextlib import contextmanager import logging -import simplejson as json +import ujson as json logger = logging.getLogger(__name__) @@ -166,8 +166,9 @@ class EventsStore(SQLBaseStore): allow_none=True, ) - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() + metadata_json = encode_json( + event.internal_metadata.get_dict(), + using_frozen_dicts=USE_FROZEN_DICTS ).decode("UTF-8") # If we have already persisted this event, we don't need to do any @@ -235,12 +236,14 @@ class EventsStore(SQLBaseStore): "event_id": event.event_id, "room_id": event.room_id, "internal_metadata": metadata_json, - "json": encode_canonical_json(event_dict).decode("UTF-8"), + "json": encode_json( + event_dict, using_frozen_dicts=USE_FROZEN_DICTS + ).decode("UTF-8"), }, ) - content = encode_canonical_json( - event.content + content = encode_json( + event.content, using_frozen_dicts=USE_FROZEN_DICTS ).decode("UTF-8") vals = { @@ -266,8 +269,8 @@ class EventsStore(SQLBaseStore): ] } - vals["unrecognized_keys"] = encode_canonical_json( - unrec + vals["unrecognized_keys"] = encode_json( + unrec, using_frozen_dicts=USE_FROZEN_DICTS ).decode("UTF-8") sql = ( From 8355b4d0740b49318f66a94b226ea9acb2cf2b38 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 May 2015 13:08:43 +0100 Subject: [PATCH 3/6] Bump syutil version --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 27b740d00..f9e59dd91 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,7 +18,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.6": ["syutil>=0.0.6"], + "syutil>=0.0.7": ["syutil>=0.0.7"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -53,8 +53,8 @@ def github_link(project, version, egg): DEPENDENCY_LINKS = [ github_link( project="matrix-org/syutil", - version="v0.0.6", - egg="syutil-0.0.6", + version="v0.0.7", + egg="syutil-0.0.7", ), github_link( project="matrix-org/matrix-angular-sdk", From ca580ef862fe1c21bf191f8066df90f78171d11f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 May 2015 15:02:55 +0100 Subject: [PATCH 4/6] Don't copy twice --- synapse/events/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index b2a904b40..39ce4f7c4 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -131,7 +131,7 @@ class FrozenEvent(EventBase): if USE_FROZEN_DICTS: frozen_dict = freeze(event_dict) else: - frozen_dict = dict(event_dict) + frozen_dict = event_dict super(FrozenEvent, self).__init__( frozen_dict, From d15f166093c121d93b8d0bcb53e55e1369b4ea84 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 May 2015 15:03:24 +0100 Subject: [PATCH 5/6] Remove log line --- synapse/http/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index cdb9f9cef..ae8f3b397 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -290,7 +290,6 @@ def respond_with_json(request, code, json_object, send_cors=False, json_bytes = encode_json( json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS ) - logger.debug("json_bytes: %r %r", json_object, json_bytes) return respond_with_json_bytes( request, code, json_bytes, From 53ef3a0bfebae941cf320ceee56499faafd3f31c Mon Sep 17 00:00:00 2001 From: Ivan Shapovalov Date: Sun, 31 May 2015 19:21:55 +0300 Subject: [PATCH 6/6] contrib/systemd: log_config.yaml: do not disable existing loggers It turned out that merely configuring the root logger is not enough for "catch-all" semantics. The logging subsystem also needs to be told not to disable existing loggers (so that their messages will get propagated to handlers up the logging hierarchy, not just silently discarded). Signed-off-by: Ivan Shapovalov --- contrib/systemd/log_config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/systemd/log_config.yaml b/contrib/systemd/log_config.yaml index e16fb5456..d85bdd120 100644 --- a/contrib/systemd/log_config.yaml +++ b/contrib/systemd/log_config.yaml @@ -21,3 +21,5 @@ handlers: root: level: INFO handlers: [journal] + +disable_existing_loggers: False