Merge pull request #243 from matrix-org/markjh/remove_syutil

Replace syutil dependency with smaller, single-purpose libraries
This commit is contained in:
Mark Haines 2015-08-25 10:52:16 +01:00
commit 1c847af28a
22 changed files with 85 additions and 94 deletions

View File

@ -1,5 +1,5 @@
from synapse.crypto.event_signing import * from synapse.crypto.event_signing import *
from syutil.base64util import encode_base64 from unpaddedbase64 import encode_base64
import argparse import argparse
import hashlib import hashlib

View File

@ -1,9 +1,7 @@
from syutil.crypto.jsonsign import verify_signed_json from signedjson.sign import verify_signed_json
from syutil.crypto.signing_key import ( from signedjson.key import decode_verify_key_bytes, write_signing_keys
decode_verify_key_bytes, write_signing_keys from unpaddedbase64 import decode_base64
)
from syutil.base64util import decode_base64
import urllib2 import urllib2
import json import json

View File

@ -4,10 +4,10 @@ import sys
import json import json
import time import time
import hashlib import hashlib
from syutil.base64util import encode_base64 from unpaddedbase64 import encode_base64
from syutil.crypto.signing_key import read_signing_keys from signedjson.key import read_signing_keys
from syutil.crypto.jsonsign import sign_json from signedjson.sign import sign_json
from syutil.jsonutil import encode_canonical_json from canonicaljson import encode_canonical_json
def select_v1_keys(connection): def select_v1_keys(connection):

View File

@ -6,8 +6,8 @@ from synapse.crypto.event_signing import (
add_event_pdu_content_hash, compute_pdu_event_reference_hash add_event_pdu_content_hash, compute_pdu_event_reference_hash
) )
from synapse.api.events.utils import prune_pdu from synapse.api.events.utils import prune_pdu
from syutil.base64util import encode_base64, decode_base64 from unpaddedbase64 import encode_base64, decode_base64
from syutil.jsonutil import encode_canonical_json from canonicaljson import encode_canonical_json
import sqlite3 import sqlite3
import sys import sys

View File

@ -4,7 +4,7 @@ from synapse.storage._base import SQLBaseStore
from synapse.storage.signatures import SignatureStore from synapse.storage.signatures import SignatureStore
from synapse.storage.event_federation import EventFederationStore from synapse.storage.event_federation import EventFederationStore
from syutil.base64util import encode_base64, decode_base64 from unpaddedbase64 import encode_base64, decode_base64
from synapse.crypto.event_signing import compute_event_signature from synapse.crypto.event_signing import compute_event_signature
@ -13,12 +13,10 @@ from synapse.events.utils import prune_event
from synapse.crypto.event_signing import check_event_content_hash from synapse.crypto.event_signing import check_event_content_hash
from syutil.crypto.jsonsign import ( from signedjson.sign import verify_signed_json, SignatureVerifyException
verify_signed_json, SignatureVerifyException, from signedjson.key import decode_verify_key_bytes
)
from syutil.crypto.signing_key import decode_verify_key_bytes
from syutil.jsonutil import encode_canonical_json from canonicaljson import encode_canonical_json
import argparse import argparse
# import dns.resolver # import dns.resolver
@ -26,7 +24,6 @@ import hashlib
import httplib import httplib
import json import json
import sqlite3 import sqlite3
import syutil
import urllib2 import urllib2
@ -324,8 +321,6 @@ if __name__ == "__main__":
) )
args = parser.parse_args() args = parser.parse_args()
signing_key = syutil.crypto.signing_key.read_signing_keys( signing_key = signedjson.key.read_signing_keys(args.signing_key)
args.signing_key
)
main(args.database, args.server_name, signing_key[0]) main(args.database, args.server_name, signing_key[0])

View File

@ -13,14 +13,17 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
from ._base import Config, ConfigError from ._base import Config, ConfigError
import syutil.crypto.signing_key
from syutil.crypto.signing_key import (
is_signing_algorithm_supported, decode_verify_key_bytes
)
from syutil.base64util import decode_base64
from synapse.util.stringutils import random_string from synapse.util.stringutils import random_string
from signedjson.key import (
generate_signing_key, is_signing_algorithm_supported,
decode_signing_key_base64, decode_verify_key_bytes,
read_signing_keys, write_signing_keys, NACL_ED25519
)
from unpaddedbase64 import decode_base64
import os
class KeyConfig(Config): class KeyConfig(Config):
@ -83,9 +86,7 @@ class KeyConfig(Config):
def read_signing_key(self, signing_key_path): def read_signing_key(self, signing_key_path):
signing_keys = self.read_file(signing_key_path, "signing_key") signing_keys = self.read_file(signing_key_path, "signing_key")
try: try:
return syutil.crypto.signing_key.read_signing_keys( return read_signing_keys(signing_keys.splitlines(True))
signing_keys.splitlines(True)
)
except Exception: except Exception:
raise ConfigError( raise ConfigError(
"Error reading signing_key." "Error reading signing_key."
@ -112,22 +113,18 @@ class KeyConfig(Config):
if not os.path.exists(signing_key_path): if not os.path.exists(signing_key_path):
with open(signing_key_path, "w") as signing_key_file: with open(signing_key_path, "w") as signing_key_file:
key_id = "a_" + random_string(4) key_id = "a_" + random_string(4)
syutil.crypto.signing_key.write_signing_keys( write_signing_keys(
signing_key_file, signing_key_file, (generate_signing_key(key_id),),
(syutil.crypto.signing_key.generate_signing_key(key_id),),
) )
else: else:
signing_keys = self.read_file(signing_key_path, "signing_key") signing_keys = self.read_file(signing_key_path, "signing_key")
if len(signing_keys.split("\n")[0].split()) == 1: if len(signing_keys.split("\n")[0].split()) == 1:
# handle keys in the old format. # handle keys in the old format.
key_id = "a_" + random_string(4) key_id = "a_" + random_string(4)
key = syutil.crypto.signing_key.decode_signing_key_base64( key = decode_signing_key_base64(
syutil.crypto.signing_key.NACL_ED25519, NACL_ED25519, key_id, signing_keys.split("\n")[0]
key_id,
signing_keys.split("\n")[0]
) )
with open(signing_key_path, "w") as signing_key_file: with open(signing_key_path, "w") as signing_key_file:
syutil.crypto.signing_key.write_signing_keys( write_signing_keys(
signing_key_file, signing_key_file, (key,),
(key,),
) )

View File

@ -15,11 +15,12 @@
# limitations under the License. # limitations under the License.
from synapse.events.utils import prune_event
from syutil.jsonutil import encode_canonical_json
from syutil.base64util import encode_base64, decode_base64
from syutil.crypto.jsonsign import sign_json
from synapse.api.errors import SynapseError, Codes from synapse.api.errors import SynapseError, Codes
from synapse.events.utils import prune_event
from canonicaljson import encode_canonical_json
from unpaddedbase64 import encode_base64, decode_base64
from signedjson.sign import sign_json
import hashlib import hashlib
import logging import logging

View File

@ -14,21 +14,21 @@
# limitations under the License. # limitations under the License.
from synapse.crypto.keyclient import fetch_server_key from synapse.crypto.keyclient import fetch_server_key
from twisted.internet import defer
from syutil.crypto.jsonsign import (
verify_signed_json, signature_ids, sign_json, encode_canonical_json
)
from syutil.crypto.signing_key import (
is_signing_algorithm_supported, decode_verify_key_bytes
)
from syutil.base64util import decode_base64, encode_base64
from synapse.api.errors import SynapseError, Codes from synapse.api.errors import SynapseError, Codes
from synapse.util.retryutils import get_retry_limiter from synapse.util.retryutils import get_retry_limiter
from synapse.util import unwrapFirstError from synapse.util import unwrapFirstError
from synapse.util.async import ObservableDeferred from synapse.util.async import ObservableDeferred
from twisted.internet import defer
from signedjson.sign import (
verify_signed_json, signature_ids, sign_json, encode_canonical_json
)
from signedjson.key import (
is_signing_algorithm_supported, decode_verify_key_bytes
)
from unpaddedbase64 import decode_base64, encode_base64
from OpenSSL import crypto from OpenSSL import crypto
from collections import namedtuple from collections import namedtuple

View File

@ -15,9 +15,10 @@
from synapse.api.errors import CodeMessageException from synapse.api.errors import CodeMessageException
from synapse.util.logcontext import preserve_context_over_fn from synapse.util.logcontext import preserve_context_over_fn
from syutil.jsonutil import encode_canonical_json
import synapse.metrics import synapse.metrics
from canonicaljson import encode_canonical_json
from twisted.internet import defer, reactor from twisted.internet import defer, reactor
from twisted.web.client import ( from twisted.web.client import (
Agent, readBody, FileBodyProducer, PartialDownloadError, Agent, readBody, FileBodyProducer, PartialDownloadError,

View File

@ -25,13 +25,13 @@ from synapse.util.async import sleep
from synapse.util.logcontext import preserve_context_over_fn from synapse.util.logcontext import preserve_context_over_fn
import synapse.metrics import synapse.metrics
from syutil.jsonutil import encode_canonical_json from canonicaljson import encode_canonical_json
from synapse.api.errors import ( from synapse.api.errors import (
SynapseError, Codes, HttpResponseException, SynapseError, Codes, HttpResponseException,
) )
from syutil.crypto.jsonsign import sign_json from signedjson.sign import sign_json
import simplejson as json import simplejson as json
import logging import logging

View File

@ -21,8 +21,8 @@ from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
import synapse.metrics import synapse.metrics
import synapse.events import synapse.events
from syutil.jsonutil import ( from canonicaljson import (
encode_canonical_json, encode_pretty_printed_json, encode_json encode_canonical_json, encode_pretty_printed_json
) )
from twisted.internet import defer from twisted.internet import defer
@ -33,6 +33,7 @@ from twisted.web.util import redirectTo
import collections import collections
import logging import logging
import urllib import urllib
import ujson
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -270,12 +271,11 @@ def respond_with_json(request, code, json_object, send_cors=False,
if pretty_print: if pretty_print:
json_bytes = encode_pretty_printed_json(json_object) + "\n" json_bytes = encode_pretty_printed_json(json_object) + "\n"
else: else:
if canonical_json: if canonical_json or synapse.events.USE_FROZEN_DICTS:
json_bytes = encode_canonical_json(json_object) json_bytes = encode_canonical_json(json_object)
else: else:
json_bytes = encode_json( # ujson doesn't like frozen_dicts.
json_object, using_frozen_dicts=synapse.events.USE_FROZEN_DICTS json_bytes = ujson.dumps(json_object, ensure_ascii=False)
)
return respond_with_json_bytes( return respond_with_json_bytes(
request, code, json_bytes, request, code, json_bytes,

View File

@ -18,7 +18,9 @@ from distutils.version import LooseVersion
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
REQUIREMENTS = { REQUIREMENTS = {
"syutil>=0.0.7": ["syutil>=0.0.7"], "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"],
"canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"],
"signedjson>=1.0.0": ["signedjson>=1.0.0"],
"Twisted>=15.1.0": ["twisted>=15.1.0"], "Twisted>=15.1.0": ["twisted>=15.1.0"],
"service_identity>=1.0.0": ["service_identity>=1.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"],
"pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyopenssl>=0.14": ["OpenSSL>=0.14"],
@ -54,11 +56,6 @@ def github_link(project, version, egg):
return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)
DEPENDENCY_LINKS = [ DEPENDENCY_LINKS = [
github_link(
project="matrix-org/syutil",
version="v0.0.7",
egg="syutil-0.0.7",
),
github_link( github_link(
project="matrix-org/matrix-angular-sdk", project="matrix-org/matrix-angular-sdk",
version="v0.6.6", version="v0.6.6",

View File

@ -40,7 +40,7 @@ class VoipRestServlet(ClientV1RestServlet):
username = "%d:%s" % (expiry, auth_user.to_string()) username = "%d:%s" % (expiry, auth_user.to_string())
mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1) mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1)
# We need to use standard base64 encoding here, *not* syutil's # We need to use standard padded base64 encoding here
# encode_base64 because we need to add the standard padding to get the # encode_base64 because we need to add the standard padding to get the
# same result as the TURN server. # same result as the TURN server.
password = base64.b64encode(mac.digest()) password = base64.b64encode(mac.digest())

View File

@ -18,7 +18,8 @@ from twisted.internet import defer
from synapse.api.errors import SynapseError from synapse.api.errors import SynapseError
from synapse.http.servlet import RestServlet from synapse.http.servlet import RestServlet
from synapse.types import UserID from synapse.types import UserID
from syutil.jsonutil import encode_canonical_json
from canonicaljson import encode_canonical_json
from ._base import client_v2_pattern from ._base import client_v2_pattern

View File

@ -16,9 +16,9 @@
from twisted.web.resource import Resource from twisted.web.resource import Resource
from synapse.http.server import respond_with_json_bytes from synapse.http.server import respond_with_json_bytes
from syutil.crypto.jsonsign import sign_json from signedjson.sign import sign_json
from syutil.base64util import encode_base64 from unpaddedbase64 import encode_base64
from syutil.jsonutil import encode_canonical_json from canonicaljson import encode_canonical_json
from OpenSSL import crypto from OpenSSL import crypto
import logging import logging

View File

@ -16,9 +16,9 @@
from twisted.web.resource import Resource from twisted.web.resource import Resource
from synapse.http.server import respond_with_json_bytes from synapse.http.server import respond_with_json_bytes
from syutil.crypto.jsonsign import sign_json from signedjson.sign import sign_json
from syutil.base64util import encode_base64 from unpaddedbase64 import encode_base64
from syutil.jsonutil import encode_canonical_json from canonicaljson import encode_canonical_json
from hashlib import sha256 from hashlib import sha256
from OpenSSL import crypto from OpenSSL import crypto
import logging import logging

View File

@ -17,7 +17,7 @@ from twisted.internet import defer
from ._base import SQLBaseStore from ._base import SQLBaseStore
from synapse.util.caches.descriptors import cached from synapse.util.caches.descriptors import cached
from syutil.base64util import encode_base64 from unpaddedbase64 import encode_base64
import logging import logging
from Queue import PriorityQueue, Empty from Queue import PriorityQueue, Empty

View File

@ -24,7 +24,7 @@ from synapse.util.logcontext import preserve_context_over_deferred
from synapse.util.logutils import log_function from synapse.util.logutils import log_function
from synapse.api.constants import EventTypes from synapse.api.constants import EventTypes
from syutil.jsonutil import encode_json from canonicaljson import encode_canonical_json
from contextlib import contextmanager from contextlib import contextmanager
import logging import logging
@ -33,6 +33,13 @@ import ujson as json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def encode_json(json_object):
if USE_FROZEN_DICTS:
# ujson doesn't like frozen_dicts
return encode_canonical_json(json_object)
else:
return json.dumps(json_object, ensure_ascii=False)
# These values are used in the `enqueus_event` and `_do_fetch` methods to # These values are used in the `enqueus_event` and `_do_fetch` methods to
# control how we batch/bulk fetch events from the database. # control how we batch/bulk fetch events from the database.
# The values are plucked out of thing air to make initial sync run faster # The values are plucked out of thing air to make initial sync run faster
@ -253,8 +260,7 @@ class EventsStore(SQLBaseStore):
) )
metadata_json = encode_json( metadata_json = encode_json(
event.internal_metadata.get_dict(), event.internal_metadata.get_dict()
using_frozen_dicts=USE_FROZEN_DICTS
).decode("UTF-8") ).decode("UTF-8")
sql = ( sql = (
@ -329,12 +335,9 @@ class EventsStore(SQLBaseStore):
"event_id": event.event_id, "event_id": event.event_id,
"room_id": event.room_id, "room_id": event.room_id,
"internal_metadata": encode_json( "internal_metadata": encode_json(
event.internal_metadata.get_dict(), event.internal_metadata.get_dict()
using_frozen_dicts=USE_FROZEN_DICTS
).decode("UTF-8"),
"json": encode_json(
event_dict(event), using_frozen_dicts=USE_FROZEN_DICTS
).decode("UTF-8"), ).decode("UTF-8"),
"json": encode_json(event_dict(event)).decode("UTF-8"),
} }
for event, _ in events_and_contexts for event, _ in events_and_contexts
], ],
@ -353,9 +356,7 @@ class EventsStore(SQLBaseStore):
"type": event.type, "type": event.type,
"processed": True, "processed": True,
"outlier": event.internal_metadata.is_outlier(), "outlier": event.internal_metadata.is_outlier(),
"content": encode_json( "content": encode_json(event.content).decode("UTF-8"),
event.content, using_frozen_dicts=USE_FROZEN_DICTS
).decode("UTF-8"),
} }
for event, _ in events_and_contexts for event, _ in events_and_contexts
], ],

View File

@ -19,7 +19,7 @@ from synapse.util.caches.descriptors import cachedInlineCallbacks
from twisted.internet import defer from twisted.internet import defer
import OpenSSL import OpenSSL
from syutil.crypto.signing_key import decode_verify_key_bytes from signedjson.key import decode_verify_key_bytes
import hashlib import hashlib

View File

@ -18,7 +18,7 @@ from twisted.internet import defer
from synapse.api.errors import StoreError from synapse.api.errors import StoreError
from syutil.jsonutil import encode_canonical_json from canonicaljson import encode_canonical_json
import logging import logging
import simplejson as json import simplejson as json

View File

@ -17,7 +17,7 @@ from twisted.internet import defer
from _base import SQLBaseStore from _base import SQLBaseStore
from syutil.base64util import encode_base64 from unpaddedbase64 import encode_base64
from synapse.crypto.event_signing import compute_event_reference_hash from synapse.crypto.event_signing import compute_event_reference_hash

View File

@ -18,7 +18,7 @@ from synapse.util.caches.descriptors import cached
from collections import namedtuple from collections import namedtuple
from syutil.jsonutil import encode_canonical_json from canonicaljson import encode_canonical_json
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)