From 1c82fbd2eb99d689d8fe835eca9f394518e25316 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 13:59:37 +0100 Subject: [PATCH 1/4] Implement create_observer. `create_observer` takes a deferred and create a new deferred that *observers* the original deferred. Any callbacks added to the observing deferred will *not* affect the origin deferred. --- synapse/util/async.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/synapse/util/async.py b/synapse/util/async.py index c4fe5d522..d8febdb90 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,3 +32,22 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) + + +def create_observer(deferred): + """Creates a deferred that observes the result or failure of the given + deferred *without* affecting the given deferred. + """ + d = defer.Deferred() + + def callback(r): + d.callback(r) + return r + + def errback(f): + d.errback(f) + return f + + deferred.addCallbacks(callback, errback) + + return d From e701aec2d1e9a565d29bc27d2bde61032cba5fd1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:20:26 +0100 Subject: [PATCH 2/4] Implement locks using create_observer for fetching media and server keys --- synapse/crypto/keyring.py | 148 ++++++++++++++----------- synapse/rest/media/v1/base_resource.py | 4 +- 2 files changed, 87 insertions(+), 65 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index f4db7b8a0..d98341f5c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -24,6 +24,8 @@ from synapse.api.errors import SynapseError, Codes from synapse.util.retryutils import get_retry_limiter +from synapse.util.async import create_observer + from OpenSSL import crypto import logging @@ -38,6 +40,8 @@ class Keyring(object): self.clock = hs.get_clock() self.hs = hs + self.key_downloads = {} + @defer.inlineCallbacks def verify_json_for_server(self, server_name, json_object): logger.debug("Verifying for %s", server_name) @@ -97,76 +101,92 @@ class Keyring(object): defer.returnValue(cached[0]) return - # Try to fetch the key from the remote server. + @defer.inlineCallbacks + def fetch_keys(): + # Try to fetch the key from the remote server. - limiter = yield get_retry_limiter( - server_name, - self.clock, - self.store, - ) - - with limiter: - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, ) - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] + with limiter: + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory ) - # Cache the result in the datastore. + # Check the response. - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate ) - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") - raise ValueError("No verification key found for given key ids") + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") + + download = self.key_downloads.get(server_name) + + if download is None: + download = fetch_keys() + self.key_downloads[server_name] = download + + @download.addBoth + def callback(ret): + del self.key_downloads[server_name] + return ret + + r = yield create_observer(download) + defer.returnValue(r) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index edd4f7802..08c8d75af 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -25,6 +25,8 @@ from twisted.internet import defer from twisted.web.resource import Resource from twisted.protocols.basic import FileSender +from synapse.util.async import create_observer + import os import logging @@ -87,7 +89,7 @@ class BaseMediaResource(Resource): def callback(media_info): del self.downloads[key] return media_info - return download + return create_observer(download) @defer.inlineCallbacks def _get_remote_media_impl(self, server_name, media_id): From 0a016b0525c918927c8134d5cb11d9be520a9efc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:37:24 +0100 Subject: [PATCH 3/4] Pull inner function out. --- synapse/crypto/keyring.py | 155 +++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 77 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index d98341f5c..14f8f536e 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -101,86 +101,10 @@ class Keyring(object): defer.returnValue(cached[0]) return - @defer.inlineCallbacks - def fetch_keys(): - # Try to fetch the key from the remote server. - - limiter = yield get_retry_limiter( - server_name, - self.clock, - self.store, - ) - - with limiter: - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory - ) - - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] - ) - - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key - ) - - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return - - raise ValueError("No verification key found for given key ids") - download = self.key_downloads.get(server_name) if download is None: - download = fetch_keys() + download = self._get_server_verify_key_impl(server_name, key_ids) self.key_downloads[server_name] = download @download.addBoth @@ -190,3 +114,80 @@ class Keyring(object): r = yield create_observer(download) defer.returnValue(r) + + + @defer.inlineCallbacks + def _get_server_verify_key_impl(self, server_name, key_ids): + # Try to fetch the key from the remote server. + + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, + ) + + with limiter: + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory + ) + + # Check the response. + + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) + + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") \ No newline at end of file From 2c70849dc32a52157217d75298c99c4cfccce639 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:38:29 +0100 Subject: [PATCH 4/4] Fix newlines --- synapse/crypto/keyring.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 14f8f536e..2b4faee4c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -115,7 +115,6 @@ class Keyring(object): r = yield create_observer(download) defer.returnValue(r) - @defer.inlineCallbacks def _get_server_verify_key_impl(self, server_name, key_ids): # Try to fetch the key from the remote server. @@ -190,4 +189,4 @@ class Keyring(object): defer.returnValue(verify_keys[key_id]) return - raise ValueError("No verification key found for given key ids") \ No newline at end of file + raise ValueError("No verification key found for given key ids")