From ed836386680acfeebe0ad2eb26985e5a88ccc3ab Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 14:26:33 +0100 Subject: [PATCH 1/9] Make one-to-one rule an underride otherwise bings don't work in one-to-one wrooms. Likewise a couple of other rules. --- synapse/push/baserules.py | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index f8408d659..f3d1cf5c5 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -126,7 +126,25 @@ def make_base_prepend_override_rules(): def make_base_append_override_rules(): return [ { - 'rule_id': 'global/override/.m.rule.call', + 'rule_id': 'global/override/.m.rule.suppress_notices', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.msgtype', + 'pattern': 'm.notice', + } + ], + 'actions': [ + 'dont_notify', + ] + } + ] + + +def make_base_append_underride_rules(user): + return [ + { + 'rule_id': 'global/underride/.m.rule.call', 'conditions': [ { 'kind': 'event_match', @@ -145,19 +163,6 @@ def make_base_append_override_rules(): } ] }, - { - 'rule_id': 'global/override/.m.rule.suppress_notices', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'content.msgtype', - 'pattern': 'm.notice', - } - ], - 'actions': [ - 'dont_notify', - ] - }, { 'rule_id': 'global/override/.m.rule.contains_display_name', 'conditions': [ @@ -176,7 +181,7 @@ def make_base_append_override_rules(): ] }, { - 'rule_id': 'global/override/.m.rule.room_one_to_one', + 'rule_id': 'global/underride/.m.rule.room_one_to_one', 'conditions': [ { 'kind': 'room_member_count', @@ -193,12 +198,7 @@ def make_base_append_override_rules(): 'value': False } ] - } - ] - - -def make_base_append_underride_rules(user): - return [ + }, { 'rule_id': 'global/underride/.m.rule.invite_for_me', 'conditions': [ From 04d1725752aae0f4d4e9eb6d97a352dee1d8ef77 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 15:01:14 +0100 Subject: [PATCH 2/9] Pedant: OS X has a space --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 14ef6c5ac..249d08c48 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ Homeserver Installation ======================= System requirements: -- POSIX-compliant system (tested on Linux & OSX) +- POSIX-compliant system (tested on Linux & OS X) - Python 2.7 Synapse is written in python but some of the libraries is uses are written in From 44ccfa6258e2a5d0b0b5fe0f7b9e87bc106a1f07 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 15:05:56 +0100 Subject: [PATCH 3/9] Remove ancient history --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index 249d08c48..e5edc0c76 100644 --- a/README.rst +++ b/README.rst @@ -367,10 +367,6 @@ SRV record, as that is the name other machines will expect it to have:: You may additionally want to pass one or more "-v" options, in order to increase the verbosity of logging output; at least for initial testing. -For the initial alpha release, the homeserver is not speaking TLS for -either client-server or server-server traffic for ease of debugging. We have -also not spent any time yet getting the homeserver to run behind loadbalancers. - Running a Demo Federation of Homeservers ---------------------------------------- From a654f3fe4988a624df0355879e1d31c1b12a6f13 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 15:07:24 +0100 Subject: [PATCH 4/9] Matrix ID server is now HTTPS --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e5edc0c76..da0102bfa 100644 --- a/README.rst +++ b/README.rst @@ -429,7 +429,7 @@ track 3PID logins and publish end-user public keys. It's currently early days for identity servers as Matrix is not yet using 3PIDs as the primary means of identity and E2E encryption is not complete. As such, -we are running a single identity server (http://matrix.org:8090) at the current +we are running a single identity server (https://matrix.org) at the current time. From f46eee838ae041ae19b4f7dc1bf6b44fe961e7b5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 15:25:28 +0100 Subject: [PATCH 5/9] Add note about updating your signing keys (ie. "the auto thing") --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index da0102bfa..714ad6443 100644 --- a/README.rst +++ b/README.rst @@ -128,6 +128,15 @@ To set up your homeserver, run (in your virtualenv, as before):: Substituting your host and domain name as appropriate. +This will generate you a config file that you can then customise, but it will +also generate a set of keys for you. These keys will allow your Home Server to +identify itself to other Home Servers, so don't lose or delete them. It would be +wise to back them up somewhere safe. If, for whatever reason, you do need to +change your Home Server's keys, you may find that other Home Servers have the +old key cached. If you update the signing key, you should change the name of the +key in the .signing.key file (the second word, which by default is +, 'auto') to something different. + By default, registration of new users is disabled. You can either enable registration in the config by specifying ``enable_registration: true`` (it is then recommended to also set up CAPTCHA), or From 1c82fbd2eb99d689d8fe835eca9f394518e25316 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 13:59:37 +0100 Subject: [PATCH 6/9] Implement create_observer. `create_observer` takes a deferred and create a new deferred that *observers* the original deferred. Any callbacks added to the observing deferred will *not* affect the origin deferred. --- synapse/util/async.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/synapse/util/async.py b/synapse/util/async.py index c4fe5d522..d8febdb90 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,3 +32,22 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) + + +def create_observer(deferred): + """Creates a deferred that observes the result or failure of the given + deferred *without* affecting the given deferred. + """ + d = defer.Deferred() + + def callback(r): + d.callback(r) + return r + + def errback(f): + d.errback(f) + return f + + deferred.addCallbacks(callback, errback) + + return d From e701aec2d1e9a565d29bc27d2bde61032cba5fd1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:20:26 +0100 Subject: [PATCH 7/9] Implement locks using create_observer for fetching media and server keys --- synapse/crypto/keyring.py | 148 ++++++++++++++----------- synapse/rest/media/v1/base_resource.py | 4 +- 2 files changed, 87 insertions(+), 65 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index f4db7b8a0..d98341f5c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -24,6 +24,8 @@ from synapse.api.errors import SynapseError, Codes from synapse.util.retryutils import get_retry_limiter +from synapse.util.async import create_observer + from OpenSSL import crypto import logging @@ -38,6 +40,8 @@ class Keyring(object): self.clock = hs.get_clock() self.hs = hs + self.key_downloads = {} + @defer.inlineCallbacks def verify_json_for_server(self, server_name, json_object): logger.debug("Verifying for %s", server_name) @@ -97,76 +101,92 @@ class Keyring(object): defer.returnValue(cached[0]) return - # Try to fetch the key from the remote server. + @defer.inlineCallbacks + def fetch_keys(): + # Try to fetch the key from the remote server. - limiter = yield get_retry_limiter( - server_name, - self.clock, - self.store, - ) - - with limiter: - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, ) - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] + with limiter: + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory ) - # Cache the result in the datastore. + # Check the response. - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate ) - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") - raise ValueError("No verification key found for given key ids") + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") + + download = self.key_downloads.get(server_name) + + if download is None: + download = fetch_keys() + self.key_downloads[server_name] = download + + @download.addBoth + def callback(ret): + del self.key_downloads[server_name] + return ret + + r = yield create_observer(download) + defer.returnValue(r) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index edd4f7802..08c8d75af 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -25,6 +25,8 @@ from twisted.internet import defer from twisted.web.resource import Resource from twisted.protocols.basic import FileSender +from synapse.util.async import create_observer + import os import logging @@ -87,7 +89,7 @@ class BaseMediaResource(Resource): def callback(media_info): del self.downloads[key] return media_info - return download + return create_observer(download) @defer.inlineCallbacks def _get_remote_media_impl(self, server_name, media_id): From 0a016b0525c918927c8134d5cb11d9be520a9efc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:37:24 +0100 Subject: [PATCH 8/9] Pull inner function out. --- synapse/crypto/keyring.py | 155 +++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 77 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index d98341f5c..14f8f536e 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -101,86 +101,10 @@ class Keyring(object): defer.returnValue(cached[0]) return - @defer.inlineCallbacks - def fetch_keys(): - # Try to fetch the key from the remote server. - - limiter = yield get_retry_limiter( - server_name, - self.clock, - self.store, - ) - - with limiter: - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory - ) - - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] - ) - - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key - ) - - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return - - raise ValueError("No verification key found for given key ids") - download = self.key_downloads.get(server_name) if download is None: - download = fetch_keys() + download = self._get_server_verify_key_impl(server_name, key_ids) self.key_downloads[server_name] = download @download.addBoth @@ -190,3 +114,80 @@ class Keyring(object): r = yield create_observer(download) defer.returnValue(r) + + + @defer.inlineCallbacks + def _get_server_verify_key_impl(self, server_name, key_ids): + # Try to fetch the key from the remote server. + + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, + ) + + with limiter: + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory + ) + + # Check the response. + + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) + + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") \ No newline at end of file From 2c70849dc32a52157217d75298c99c4cfccce639 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:38:29 +0100 Subject: [PATCH 9/9] Fix newlines --- synapse/crypto/keyring.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 14f8f536e..2b4faee4c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -115,7 +115,6 @@ class Keyring(object): r = yield create_observer(download) defer.returnValue(r) - @defer.inlineCallbacks def _get_server_verify_key_impl(self, server_name, key_ids): # Try to fetch the key from the remote server. @@ -190,4 +189,4 @@ class Keyring(object): defer.returnValue(verify_keys[key_id]) return - raise ValueError("No verification key found for given key ids") \ No newline at end of file + raise ValueError("No verification key found for given key ids")