Attempt to be more performant on PyPy (#3462)

This commit is contained in:
Amber Brown 2018-06-28 14:49:57 +01:00 committed by GitHub
parent 72d2143ea8
commit 6350bf925e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 79 additions and 62 deletions

1
changelog.d/3462.feature Normal file
View File

@ -0,0 +1 @@
Synapse now uses the best performing JSON encoder/decoder according to your runtime (simplejson on CPython, stdlib json on PyPy).

View File

@ -17,7 +17,8 @@
import logging
import simplejson as json
from canonicaljson import json
from six import iteritems
from six.moves import http_client

View File

@ -17,7 +17,8 @@ from synapse.storage.presence import UserPresenceState
from synapse.types import UserID, RoomID
from twisted.internet import defer
import simplejson as json
from canonicaljson import json
import jsonschema
from jsonschema import FormatChecker

View File

@ -18,7 +18,7 @@ from twisted.web.http import HTTPClient
from twisted.internet.protocol import Factory
from twisted.internet import defer, reactor
from synapse.http.endpoint import matrix_federation_endpoint
import simplejson as json
from canonicaljson import json
import logging

View File

@ -15,7 +15,7 @@
# limitations under the License.
import logging
import simplejson as json
from canonicaljson import json
from twisted.internet import defer
from synapse.api.errors import AuthError, FederationError, SynapseError, NotFoundError

View File

@ -16,6 +16,8 @@
from twisted.internet import defer, threads
from canonicaljson import json
from ._base import BaseHandler
from synapse.api.constants import LoginType
from synapse.api.errors import (
@ -32,7 +34,6 @@ from twisted.web.client import PartialDownloadError
import logging
import bcrypt
import pymacaroons
import simplejson
import attr
import synapse.util.stringutils as stringutils
@ -403,7 +404,7 @@ class AuthHandler(BaseHandler):
except PartialDownloadError as pde:
# Twisted is silly
data = pde.response
resp_body = simplejson.loads(data)
resp_body = json.loads(data)
if 'success' in resp_body:
# Note that we do NOT check the hostname here: we explicitly

View File

@ -14,10 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import simplejson as json
import logging
from canonicaljson import encode_canonical_json
from canonicaljson import encode_canonical_json, json
from twisted.internet import defer
from six import iteritems
@ -357,7 +356,7 @@ def _exception_to_failure(e):
# include ConnectionRefused and other errors
#
# Note that some Exceptions (notably twisted's ResponseFailed etc) don't
# give a string for e.message, which simplejson then fails to serialize.
# give a string for e.message, which json then fails to serialize.
return {
"status": 503, "message": str(e.message),
}

View File

@ -19,7 +19,7 @@
import logging
import simplejson as json
from canonicaljson import json
from twisted.internet import defer

View File

@ -14,10 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import simplejson
import sys
from canonicaljson import encode_canonical_json
from canonicaljson import encode_canonical_json, json
import six
from six import string_types, itervalues, iteritems
from twisted.internet import defer
@ -797,7 +796,7 @@ class EventCreationHandler(object):
# Ensure that we can round trip before trying to persist in db
try:
dump = frozendict_json_encoder.encode(event.content)
simplejson.loads(dump)
json.loads(dump)
except Exception:
logger.exception("Failed to encode content: %r", event.content)
raise

View File

@ -42,7 +42,7 @@ from twisted.web._newclient import ResponseDone
from six import StringIO
from prometheus_client import Counter
import simplejson as json
from canonicaljson import json
import logging
import urllib

View File

@ -27,7 +27,7 @@ from synapse.util import logcontext
from synapse.util.logcontext import make_deferred_yieldable
import synapse.util.retryutils
from canonicaljson import encode_canonical_json
from canonicaljson import encode_canonical_json, json
from synapse.api.errors import (
SynapseError, Codes, HttpResponseException, FederationDeniedError,
@ -36,7 +36,6 @@ from synapse.api.errors import (
from signedjson.sign import sign_json
import cgi
import simplejson as json
import logging
import random
import sys

View File

@ -29,7 +29,7 @@ import synapse.metrics
import synapse.events
from canonicaljson import (
encode_canonical_json, encode_pretty_printed_json
encode_canonical_json, encode_pretty_printed_json, json
)
from twisted.internet import defer
@ -41,7 +41,6 @@ from twisted.web.util import redirectTo
import collections
import logging
import urllib
import simplejson
logger = logging.getLogger(__name__)
@ -410,7 +409,7 @@ def respond_with_json(request, code, json_object, send_cors=False,
if canonical_json or synapse.events.USE_FROZEN_DICTS:
json_bytes = encode_canonical_json(json_object)
else:
json_bytes = simplejson.dumps(json_object)
json_bytes = json.dumps(json_object)
return respond_with_json_bytes(
request, code, json_bytes,

View File

@ -18,7 +18,9 @@
from synapse.api.errors import SynapseError, Codes
import logging
import simplejson
from canonicaljson import json
logger = logging.getLogger(__name__)
@ -171,7 +173,7 @@ def parse_json_value_from_request(request, allow_empty_body=False):
return None
try:
content = simplejson.loads(content_bytes)
content = json.loads(content_bytes)
except Exception as e:
logger.warn("Unable to parse JSON: %s", e)
raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON)

View File

@ -147,7 +147,8 @@ class GCCounts(object):
yield cm
REGISTRY.register(GCCounts())
if not running_on_pypy:
REGISTRY.register(GCCounts())
#
# Twisted reactor metrics

View File

@ -19,13 +19,17 @@ allowed to be sent by which side.
"""
import logging
import simplejson
import platform
if platform.python_implementation() == "PyPy":
import json
_json_encoder = json.JSONEncoder()
else:
import simplejson as json
_json_encoder = json.JSONEncoder(namedtuple_as_object=False)
logger = logging.getLogger(__name__)
_json_encoder = simplejson.JSONEncoder(namedtuple_as_object=False)
class Command(object):
"""The base command class.
@ -102,7 +106,7 @@ class RdataCommand(Command):
return cls(
stream_name,
None if token == "batch" else int(token),
simplejson.loads(row_json)
json.loads(row_json)
)
def to_line(self):
@ -300,7 +304,7 @@ class InvalidateCacheCommand(Command):
def from_line(cls, line):
cache_func, keys_json = line.split(" ", 1)
return cls(cache_func, simplejson.loads(keys_json))
return cls(cache_func, json.loads(keys_json))
def to_line(self):
return " ".join((
@ -329,7 +333,7 @@ class UserIpCommand(Command):
def from_line(cls, line):
user_id, jsn = line.split(" ", 1)
access_token, ip, user_agent, device_id, last_seen = simplejson.loads(jsn)
access_token, ip, user_agent, device_id, last_seen = json.loads(jsn)
return cls(
user_id, access_token, ip, user_agent, device_id, last_seen

View File

@ -23,7 +23,8 @@ from synapse.util.msisdn import phone_number_to_msisdn
from .base import ClientV1RestServlet, client_path_patterns
import simplejson as json
from canonicaljson import json
import urllib
from six.moves.urllib import parse as urlparse

View File

@ -31,7 +31,7 @@ from synapse.http.servlet import (
from six.moves.urllib import parse as urlparse
import logging
import simplejson as json
from canonicaljson import json
logger = logging.getLogger(__name__)

View File

@ -33,7 +33,7 @@ from ._base import set_timeline_upper_limit
import itertools
import logging
import simplejson as json
from canonicaljson import json
logger = logging.getLogger(__name__)

View File

@ -22,8 +22,9 @@ from synapse.api.errors import (
from twisted.protocols.basic import FileSender
from twisted.web import server, resource
from canonicaljson import json
import base64
import simplejson as json
import logging
import os
import re

View File

@ -23,7 +23,8 @@ import re
import shutil
import sys
import traceback
import simplejson as json
from canonicaljson import json
from six.moves import urllib_parse as urlparse
from six import string_types

View File

@ -22,8 +22,9 @@ from synapse.storage.util.id_generators import StreamIdGenerator
from synapse.util.caches.stream_change_cache import StreamChangeCache
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
from canonicaljson import json
import abc
import simplejson as json
import logging
logger = logging.getLogger(__name__)

View File

@ -15,8 +15,8 @@
# limitations under the License.
import logging
import re
import simplejson as json
from twisted.internet import defer
from canonicaljson import json
from synapse.appservice import AppServiceTransaction
from synapse.config.appservice import load_appservices

View File

@ -18,7 +18,8 @@ from . import engines
from twisted.internet import defer
import simplejson as json
from canonicaljson import json
import logging
logger = logging.getLogger(__name__)

View File

@ -14,7 +14,8 @@
# limitations under the License.
import logging
import simplejson
from canonicaljson import json
from twisted.internet import defer
@ -85,7 +86,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
)
rows = []
for destination, edu in remote_messages_by_destination.items():
edu_json = simplejson.dumps(edu)
edu_json = json.dumps(edu)
rows.append((destination, stream_id, now_ms, edu_json))
txn.executemany(sql, rows)
@ -177,7 +178,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
" WHERE user_id = ?"
)
txn.execute(sql, (user_id,))
message_json = simplejson.dumps(messages_by_device["*"])
message_json = json.dumps(messages_by_device["*"])
for row in txn:
# Add the message for all devices for this user on this
# server.
@ -199,7 +200,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
# Only insert into the local inbox if the device exists on
# this server
device = row[0]
message_json = simplejson.dumps(messages_by_device[device])
message_json = json.dumps(messages_by_device[device])
messages_json_for_user[device] = message_json
if messages_json_for_user:
@ -253,7 +254,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
messages = []
for row in txn:
stream_pos = row[0]
messages.append(simplejson.loads(row[1]))
messages.append(json.loads(row[1]))
if len(messages) < limit:
stream_pos = current_stream_id
return (messages, stream_pos)
@ -389,7 +390,7 @@ class DeviceInboxStore(BackgroundUpdateStore):
messages = []
for row in txn:
stream_pos = row[0]
messages.append(simplejson.loads(row[1]))
messages.append(json.loads(row[1]))
if len(messages) < limit:
stream_pos = current_stream_id
return (messages, stream_pos)

View File

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import simplejson as json
from twisted.internet import defer
@ -21,6 +20,8 @@ from synapse.api.errors import StoreError
from ._base import SQLBaseStore, Cache
from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks
from canonicaljson import json
from six import itervalues, iteritems
logger = logging.getLogger(__name__)

View File

@ -16,8 +16,7 @@ from twisted.internet import defer
from synapse.util.caches.descriptors import cached
from canonicaljson import encode_canonical_json
import simplejson as json
from canonicaljson import encode_canonical_json, json
from ._base import SQLBaseStore

View File

@ -19,7 +19,8 @@ from twisted.internet import defer
from synapse.util.caches.descriptors import cachedInlineCallbacks
import logging
import simplejson as json
from canonicaljson import json
from six import iteritems

View File

@ -19,7 +19,8 @@ from functools import wraps
import itertools
import logging
import simplejson as json
from canonicaljson import json
from twisted.internet import defer
from synapse.storage.events_worker import EventsWorkerStore

View File

@ -29,7 +29,8 @@ from synapse.api.errors import SynapseError
from collections import namedtuple
import logging
import simplejson as json
from canonicaljson import json
# these are only included to make the type annotations work
from synapse.events import EventBase # noqa: F401

View File

@ -19,8 +19,7 @@ from ._base import SQLBaseStore
from synapse.api.errors import SynapseError, Codes
from synapse.util.caches.descriptors import cachedInlineCallbacks
from canonicaljson import encode_canonical_json
import simplejson as json
from canonicaljson import encode_canonical_json, json
class FilteringStore(SQLBaseStore):

View File

@ -20,7 +20,7 @@ from synapse.api.errors import SynapseError
from ._base import SQLBaseStore
import simplejson as json
from canonicaljson import json
# The category ID for the "default" category. We don't store as null in the

View File

@ -25,9 +25,10 @@ from synapse.push.baserules import list_with_base_rules
from synapse.api.constants import EventTypes
from twisted.internet import defer
from canonicaljson import json
import abc
import logging
import simplejson as json
logger = logging.getLogger(__name__)

View File

@ -17,12 +17,11 @@
from ._base import SQLBaseStore
from twisted.internet import defer
from canonicaljson import encode_canonical_json
from canonicaljson import encode_canonical_json, json
from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList
import logging
import simplejson as json
import types
logger = logging.getLogger(__name__)

View File

@ -21,9 +21,10 @@ from synapse.util.caches.stream_change_cache import StreamChangeCache
from twisted.internet import defer
from canonicaljson import json
import abc
import logging
import simplejson as json
logger = logging.getLogger(__name__)

View File

@ -20,9 +20,10 @@ from synapse.storage._base import SQLBaseStore
from synapse.storage.search import SearchStore
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
from canonicaljson import json
import collections
import logging
import simplejson as json
import re
logger = logging.getLogger(__name__)

View File

@ -28,7 +28,7 @@ from synapse.api.constants import Membership, EventTypes
from synapse.types import get_domain_from_id
import logging
import simplejson as json
from canonicaljson import json
from six import itervalues, iteritems

View File

@ -16,7 +16,7 @@
from collections import namedtuple
import logging
import re
import simplejson as json
from canonicaljson import json
from six import string_types

View File

@ -19,7 +19,8 @@ from synapse.storage.account_data import AccountDataWorkerStore
from synapse.util.caches.descriptors import cached
from twisted.internet import defer
import simplejson as json
from canonicaljson import json
import logging
from six.moves import range

View File

@ -19,12 +19,11 @@ from synapse.util.caches.descriptors import cached
from twisted.internet import defer
import six
from canonicaljson import encode_canonical_json
from canonicaljson import encode_canonical_json, json
from collections import namedtuple
import logging
import simplejson as json
# py2 sqlite has buffer hardcoded as only binary type, so we must use it,
# despite being deprecated and removed in favor of memoryview

View File

@ -14,7 +14,7 @@
# limitations under the License.
from frozendict import frozendict
import simplejson as json
from canonicaljson import json
from six import string_types