From 78323ccdb359404109bfcdd8b5bf6f641ba3ff9b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 16:17:38 +0100 Subject: [PATCH 1/4] Remove syutil dependency in favour of smaller single-purpose libraries --- synapse/config/key.py | 35 ++++++++++------------ synapse/crypto/event_signing.py | 9 +++--- synapse/crypto/keyring.py | 20 ++++++------- synapse/http/client.py | 3 +- synapse/http/matrixfederationclient.py | 4 +-- synapse/http/server.py | 12 ++++---- synapse/python_dependencies.py | 9 ++---- synapse/rest/client/v1/voip.py | 2 +- synapse/rest/client/v2_alpha/keys.py | 3 +- synapse/rest/key/v1/server_key_resource.py | 6 ++-- synapse/rest/key/v2/local_key_resource.py | 6 ++-- synapse/storage/event_federation.py | 2 +- synapse/storage/events.py | 23 +++++++------- synapse/storage/keys.py | 2 +- synapse/storage/pusher.py | 2 +- synapse/storage/signatures.py | 2 +- synapse/storage/transactions.py | 2 +- 17 files changed, 70 insertions(+), 72 deletions(-) diff --git a/synapse/config/key.py b/synapse/config/key.py index 0494c0cb7..0f90bce04 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -13,14 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from ._base import Config, ConfigError -import syutil.crypto.signing_key -from syutil.crypto.signing_key import ( - is_signing_algorithm_supported, decode_verify_key_bytes -) -from syutil.base64util import decode_base64 + from synapse.util.stringutils import random_string +from signedjson.key import ( + generate_signing_key, is_signing_algorithm_supported, + decode_signing_key_base64, decode_verify_key_bytes, + read_signing_keys, write_signing_keys, NACL_ED25519 +) +from unpadded_base64 import decode_base64 + +import os class KeyConfig(Config): @@ -83,9 +86,7 @@ class KeyConfig(Config): def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") try: - return syutil.crypto.signing_key.read_signing_keys( - signing_keys.splitlines(True) - ) + return read_signing_keys(signing_keys.splitlines(True)) except Exception: raise ConfigError( "Error reading signing_key." @@ -112,22 +113,18 @@ class KeyConfig(Config): if not os.path.exists(signing_key_path): with open(signing_key_path, "w") as signing_key_file: key_id = "a_" + random_string(4) - syutil.crypto.signing_key.write_signing_keys( - signing_key_file, - (syutil.crypto.signing_key.generate_signing_key(key_id),), + write_signing_keys( + signing_key_file, (generate_signing_key(key_id),), ) else: signing_keys = self.read_file(signing_key_path, "signing_key") if len(signing_keys.split("\n")[0].split()) == 1: # handle keys in the old format. key_id = "a_" + random_string(4) - key = syutil.crypto.signing_key.decode_signing_key_base64( - syutil.crypto.signing_key.NACL_ED25519, - key_id, - signing_keys.split("\n")[0] + key = decode_signing_key_base64( + NACL_ED25519, key_id, signing_keys.split("\n")[0] ) with open(signing_key_path, "w") as signing_key_file: - syutil.crypto.signing_key.write_signing_keys( - signing_key_file, - (key,), + write_signing_keys( + signing_key_file, (key,), ) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 6633b1956..64e40864a 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -15,11 +15,12 @@ # limitations under the License. -from synapse.events.utils import prune_event -from syutil.jsonutil import encode_canonical_json -from syutil.base64util import encode_base64, decode_base64 -from syutil.crypto.jsonsign import sign_json from synapse.api.errors import SynapseError, Codes +from synapse.events.utils import prune_event + +from canonicaljson import encode_canonical_json +from unpaddedbase64 import encode_base64, decode_base64 +from signedjson.sign import sign_json import hashlib import logging diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index aa74d4d0c..a692cdbe5 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -14,21 +14,21 @@ # limitations under the License. from synapse.crypto.keyclient import fetch_server_key -from twisted.internet import defer -from syutil.crypto.jsonsign import ( - verify_signed_json, signature_ids, sign_json, encode_canonical_json -) -from syutil.crypto.signing_key import ( - is_signing_algorithm_supported, decode_verify_key_bytes -) -from syutil.base64util import decode_base64, encode_base64 from synapse.api.errors import SynapseError, Codes - from synapse.util.retryutils import get_retry_limiter from synapse.util import unwrapFirstError - from synapse.util.async import ObservableDeferred +from twisted.internet import defer + +from signedjson.sign import ( + verify_signed_json, signature_ids, sign_json, encode_canonical_json +) +from signedjson.key import ( + is_signing_algorithm_supported, decode_verify_key_bytes +) +from unpaddedbase64 import decode_base64, encode_base64 + from OpenSSL import crypto from collections import namedtuple diff --git a/synapse/http/client.py b/synapse/http/client.py index 49737d55d..4b8fd3d3a 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -15,9 +15,10 @@ from synapse.api.errors import CodeMessageException from synapse.util.logcontext import preserve_context_over_fn -from syutil.jsonutil import encode_canonical_json import synapse.metrics +from canonicaljson import encode_canonical_json + from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 854e17a47..1c9e55278 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -25,13 +25,13 @@ from synapse.util.async import sleep from synapse.util.logcontext import preserve_context_over_fn import synapse.metrics -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json from synapse.api.errors import ( SynapseError, Codes, HttpResponseException, ) -from syutil.crypto.jsonsign import sign_json +from signedjson.sign import sign_json import simplejson as json import logging diff --git a/synapse/http/server.py b/synapse/http/server.py index b60e905a6..50feea6f1 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -21,8 +21,8 @@ from synapse.util.logcontext import LoggingContext, PreserveLoggingContext import synapse.metrics import synapse.events -from syutil.jsonutil import ( - encode_canonical_json, encode_pretty_printed_json, encode_json +from canonicaljson import ( + encode_canonical_json, encode_pretty_printed_json ) from twisted.internet import defer @@ -33,6 +33,7 @@ from twisted.web.util import redirectTo import collections import logging import urllib +import ujson logger = logging.getLogger(__name__) @@ -270,12 +271,11 @@ def respond_with_json(request, code, json_object, send_cors=False, if pretty_print: json_bytes = encode_pretty_printed_json(json_object) + "\n" else: - if canonical_json: + if canonical_json or synapse.events.USE_FROZEN_DICTS: json_bytes = encode_canonical_json(json_object) else: - json_bytes = encode_json( - json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS - ) + # ujson doesn't like frozen_dicts. + json_bytes = ujson.dumps(json_object, ensure_ascii=False) return respond_with_json_bytes( request, code, json_bytes, diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a87fdeb2a..ced7ff0d7 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,7 +18,9 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.7": ["syutil>=0.0.7"], + "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"], + "canonicaljson>=1.0.0": ["canconicaljson>=1.0.0"], + "signedjson>=1.0.0": ["signedjson>=1.0.0"], "Twisted>=15.1.0": ["twisted>=15.1.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -54,11 +56,6 @@ def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) DEPENDENCY_LINKS = [ - github_link( - project="matrix-org/syutil", - version="v0.0.7", - egg="syutil-0.0.7", - ), github_link( project="matrix-org/matrix-angular-sdk", version="v0.6.6", diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py index 11d08fbce..45ed5ccea 100644 --- a/synapse/rest/client/v1/voip.py +++ b/synapse/rest/client/v1/voip.py @@ -40,7 +40,7 @@ class VoipRestServlet(ClientV1RestServlet): username = "%d:%s" % (expiry, auth_user.to_string()) mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) - # We need to use standard base64 encoding here, *not* syutil's + # We need to use standard padded base64 encoding here # encode_base64 because we need to add the standard padding to get the # same result as the TURN server. password = base64.b64encode(mac.digest()) diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index 718928eed..21654fa2d 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -18,7 +18,8 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.http.servlet import RestServlet from synapse.types import UserID -from syutil.jsonutil import encode_canonical_json + +from canonicaljson import encode_canonical_json from ._base import client_v2_pattern diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py index 71e9a51f5..6df46969c 100644 --- a/synapse/rest/key/v1/server_key_resource.py +++ b/synapse/rest/key/v1/server_key_resource.py @@ -16,9 +16,9 @@ from twisted.web.resource import Resource from synapse.http.server import respond_with_json_bytes -from syutil.crypto.jsonsign import sign_json -from syutil.base64util import encode_base64 -from syutil.jsonutil import encode_canonical_json +from signedjson.sign import sign_json +from unpaddedbase64 import encode_base64 +from canonicaljson import encode_canonical_json from OpenSSL import crypto import logging diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index 33cbd7cf8..ef7699d59 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -16,9 +16,9 @@ from twisted.web.resource import Resource from synapse.http.server import respond_with_json_bytes -from syutil.crypto.jsonsign import sign_json -from syutil.base64util import encode_base64 -from syutil.jsonutil import encode_canonical_json +from signedjson.sign import sign_json +from unpaddedbase64 import encode_base64 +from canonicaljson import encode_canonical_json from hashlib import sha256 from OpenSSL import crypto import logging diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 25cc84eb9..bc90e17c6 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached -from syutil.base64util import encode_base64 +from unpaddedbase64 import encode_base64 import logging from Queue import PriorityQueue, Empty diff --git a/synapse/storage/events.py b/synapse/storage/events.py index e3eabab13..e7439321b 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -24,7 +24,7 @@ from synapse.util.logcontext import preserve_context_over_deferred from synapse.util.logutils import log_function from synapse.api.constants import EventTypes -from syutil.jsonutil import encode_json +from canonicaljson import encode_canonical_json from contextlib import contextmanager import logging @@ -33,6 +33,13 @@ import ujson as json logger = logging.getLogger(__name__) +def encode_json(json_object): + if USE_FROZEN_DICTS: + # ujson doesn't like frozen_dicts + return encode_canonical_json(json_object) + else: + return json.dumps(json_object, ensure_ascii=False) + # These values are used in the `enqueus_event` and `_do_fetch` methods to # control how we batch/bulk fetch events from the database. # The values are plucked out of thing air to make initial sync run faster @@ -253,8 +260,7 @@ class EventsStore(SQLBaseStore): ) metadata_json = encode_json( - event.internal_metadata.get_dict(), - using_frozen_dicts=USE_FROZEN_DICTS + event.internal_metadata.get_dict() ).decode("UTF-8") sql = ( @@ -329,12 +335,9 @@ class EventsStore(SQLBaseStore): "event_id": event.event_id, "room_id": event.room_id, "internal_metadata": encode_json( - event.internal_metadata.get_dict(), - using_frozen_dicts=USE_FROZEN_DICTS - ).decode("UTF-8"), - "json": encode_json( - event_dict(event), using_frozen_dicts=USE_FROZEN_DICTS + event.internal_metadata.get_dict() ).decode("UTF-8"), + "json": encode_json(event_dict(event)).decode("UTF-8"), } for event, _ in events_and_contexts ], @@ -353,9 +356,7 @@ class EventsStore(SQLBaseStore): "type": event.type, "processed": True, "outlier": event.internal_metadata.is_outlier(), - "content": encode_json( - event.content, using_frozen_dicts=USE_FROZEN_DICTS - ).decode("UTF-8"), + "content": encode_json(event.content).decode("UTF-8"), } for event, _ in events_and_contexts ], diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index ffd6daa88..344cacdc7 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -19,7 +19,7 @@ from synapse.util.caches.descriptors import cachedInlineCallbacks from twisted.internet import defer import OpenSSL -from syutil.crypto.signing_key import decode_verify_key_bytes +from signedjson.key import decode_verify_key_bytes import hashlib diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 08ea62681..00b748f13 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json import logging import simplejson as json diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 4f15e534b..ab57b9217 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -17,7 +17,7 @@ from twisted.internet import defer from _base import SQLBaseStore -from syutil.base64util import encode_base64 +from unpaddedbase64 import encode_base64 from synapse.crypto.event_signing import compute_event_reference_hash diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index c8c7e6591..15695e983 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -18,7 +18,7 @@ from synapse.util.caches.descriptors import cached from collections import namedtuple -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json import logging logger = logging.getLogger(__name__) From f093873d697f99a59cade944bb68850b017afd09 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 16:30:35 +0100 Subject: [PATCH 2/4] Replace syutil references in scripts --- scripts-dev/check_event_hash.py | 2 +- scripts-dev/check_signature.py | 8 +++----- scripts-dev/convert_server_keys.py | 8 ++++---- scripts-dev/hash_history.py | 4 ++-- scripts/upgrade_db_to_v0.6.0.py | 15 +++++---------- 5 files changed, 15 insertions(+), 22 deletions(-) diff --git a/scripts-dev/check_event_hash.py b/scripts-dev/check_event_hash.py index 679afbd26..7ccae34d4 100644 --- a/scripts-dev/check_event_hash.py +++ b/scripts-dev/check_event_hash.py @@ -1,5 +1,5 @@ from synapse.crypto.event_signing import * -from syutil.base64util import encode_base64 +from unpaddedbase64 import encode_base64 import argparse import hashlib diff --git a/scripts-dev/check_signature.py b/scripts-dev/check_signature.py index 59e3d603a..079577908 100644 --- a/scripts-dev/check_signature.py +++ b/scripts-dev/check_signature.py @@ -1,9 +1,7 @@ -from syutil.crypto.jsonsign import verify_signed_json -from syutil.crypto.signing_key import ( - decode_verify_key_bytes, write_signing_keys -) -from syutil.base64util import decode_base64 +from signedjson.sign import verify_signed_json +from signedjson.key import decode_verify_key_bytes, write_signing_keys +from unpaddedbase64 import decode_base64 import urllib2 import json diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py index a1ee39059..151551f22 100644 --- a/scripts-dev/convert_server_keys.py +++ b/scripts-dev/convert_server_keys.py @@ -4,10 +4,10 @@ import sys import json import time import hashlib -from syutil.base64util import encode_base64 -from syutil.crypto.signing_key import read_signing_keys -from syutil.crypto.jsonsign import sign_json -from syutil.jsonutil import encode_canonical_json +from unpaddedbase64 import encode_base64 +from signedjson.key import read_signing_keys +from signedjson.sign import sign_json +from canonicaljson import encode_canonical_json def select_v1_keys(connection): diff --git a/scripts-dev/hash_history.py b/scripts-dev/hash_history.py index bdad530af..616d6a10e 100644 --- a/scripts-dev/hash_history.py +++ b/scripts-dev/hash_history.py @@ -6,8 +6,8 @@ from synapse.crypto.event_signing import ( add_event_pdu_content_hash, compute_pdu_event_reference_hash ) from synapse.api.events.utils import prune_pdu -from syutil.base64util import encode_base64, decode_base64 -from syutil.jsonutil import encode_canonical_json +from unpaddedbase64 import encode_base64, decode_base64 +from canonicaljson import encode_canonical_json import sqlite3 import sys diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py index f466ca510..cd4be28b8 100755 --- a/scripts/upgrade_db_to_v0.6.0.py +++ b/scripts/upgrade_db_to_v0.6.0.py @@ -4,7 +4,7 @@ from synapse.storage._base import SQLBaseStore from synapse.storage.signatures import SignatureStore from synapse.storage.event_federation import EventFederationStore -from syutil.base64util import encode_base64, decode_base64 +from unpaddedbase64 import encode_base64, decode_base64 from synapse.crypto.event_signing import compute_event_signature @@ -13,12 +13,10 @@ from synapse.events.utils import prune_event from synapse.crypto.event_signing import check_event_content_hash -from syutil.crypto.jsonsign import ( - verify_signed_json, SignatureVerifyException, -) -from syutil.crypto.signing_key import decode_verify_key_bytes +from signedjson.sign import verify_signed_json, SignatureVerifyException +from signedjson.key import decode_verify_key_bytes -from syutil.jsonutil import encode_canonical_json +from canonicaljson import encode_canonical_json import argparse # import dns.resolver @@ -26,7 +24,6 @@ import hashlib import httplib import json import sqlite3 -import syutil import urllib2 @@ -324,8 +321,6 @@ if __name__ == "__main__": ) args = parser.parse_args() - signing_key = syutil.crypto.signing_key.read_signing_keys( - args.signing_key - ) + signing_key = signedjson.key.read_signing_keys(args.signing_key) main(args.database, args.server_name, signing_key[0]) From 01fc3943f1aa409766f8fc54037af9102c55c658 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 17:18:58 +0100 Subject: [PATCH 3/4] Fix indent --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index e7439321b..fba837f46 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -35,7 +35,7 @@ logger = logging.getLogger(__name__) def encode_json(json_object): if USE_FROZEN_DICTS: - # ujson doesn't like frozen_dicts + # ujson doesn't like frozen_dicts return encode_canonical_json(json_object) else: return json.dumps(json_object, ensure_ascii=False) From cf8c04948f9a8e8f50349918622164abd7590a13 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 25 Aug 2015 10:42:59 +0100 Subject: [PATCH 4/4] Fix typo in module imports and package dependencies --- synapse/config/key.py | 2 +- synapse/python_dependencies.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/key.py b/synapse/config/key.py index 0f90bce04..23ac8a3fc 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -21,7 +21,7 @@ from signedjson.key import ( decode_signing_key_base64, decode_verify_key_bytes, read_signing_keys, write_signing_keys, NACL_ED25519 ) -from unpadded_base64 import decode_base64 +from unpaddedbase64 import decode_base64 import os diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ced7ff0d7..4c07aa07a 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"], - "canonicaljson>=1.0.0": ["canconicaljson>=1.0.0"], + "canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"], "signedjson>=1.0.0": ["signedjson>=1.0.0"], "Twisted>=15.1.0": ["twisted>=15.1.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"],