Speedup tests by caching HomeServerConfig instances (#15284)

These two lines:

```
config_obj = HomeServerConfig()
config_obj.parse_config_dict(config, "", "")
```

are called many times with the exact same value for `config`.

As the test suite is CPU-bound and non-negligeably time is spent in
`parse_config_dict`, this saves ~5% on the overall runtime of the Trial
test suite (tested with both `-j2` and `-j12` on a 12t CPU).

This is sadly rather limited, as the cache cannot be shared between
processes (it contains at least jinja2.Template and RLock objects which
aren't pickleable), and Trial tends to run close tests in different
processes.
This commit is contained in:
Val Lorentz 2023-04-18 15:50:27 +02:00 committed by GitHub
parent aec639e3e3
commit cb8e274c07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 61 additions and 2 deletions

1
changelog.d/15284.misc Normal file
View File

@ -0,0 +1 @@
Speedup tests by caching HomeServerConfig instances.

View File

@ -16,6 +16,7 @@
import gc import gc
import hashlib import hashlib
import hmac import hmac
import json
import logging import logging
import secrets import secrets
import time import time
@ -53,6 +54,7 @@ from twisted.web.server import Request
from synapse import events from synapse import events
from synapse.api.constants import EventTypes from synapse.api.constants import EventTypes
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
from synapse.config._base import Config, RootConfig
from synapse.config.homeserver import HomeServerConfig from synapse.config.homeserver import HomeServerConfig
from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.config.server import DEFAULT_ROOM_VERSION
from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.crypto.event_signing import add_hashes_and_signatures
@ -124,6 +126,63 @@ def around(target: TV) -> Callable[[Callable[Concatenate[S, P], R]], None]:
return _around return _around
_TConfig = TypeVar("_TConfig", Config, RootConfig)
def deepcopy_config(config: _TConfig) -> _TConfig:
new_config: _TConfig
if isinstance(config, RootConfig):
new_config = config.__class__(config.config_files) # type: ignore[arg-type]
else:
new_config = config.__class__(config.root)
for attr_name in config.__dict__:
if attr_name.startswith("__") or attr_name == "root":
continue
attr = getattr(config, attr_name)
if isinstance(attr, Config):
new_attr = deepcopy_config(attr)
else:
new_attr = attr
setattr(new_config, attr_name, new_attr)
return new_config
_make_homeserver_config_obj_cache: Dict[str, Union[RootConfig, Config]] = {}
def make_homeserver_config_obj(config: Dict[str, Any]) -> RootConfig:
"""Creates a :class:`HomeServerConfig` instance with the given configuration dict.
This is equivalent to::
config_obj = HomeServerConfig()
config_obj.parse_config_dict(config, "", "")
but it keeps a cache of `HomeServerConfig` instances and deepcopies them as needed,
to avoid validating the whole configuration every time.
"""
cache_key = json.dumps(config)
if cache_key in _make_homeserver_config_obj_cache:
# Cache hit: reuse the existing instance
config_obj = _make_homeserver_config_obj_cache[cache_key]
else:
# Cache miss; create the actual instance
config_obj = HomeServerConfig()
config_obj.parse_config_dict(config, "", "")
# Add to the cache
_make_homeserver_config_obj_cache[cache_key] = config_obj
assert isinstance(config_obj, RootConfig)
return deepcopy_config(config_obj)
class TestCase(unittest.TestCase): class TestCase(unittest.TestCase):
"""A subclass of twisted.trial's TestCase which looks for 'loglevel' """A subclass of twisted.trial's TestCase which looks for 'loglevel'
attributes on both itself and its individual test methods, to override the attributes on both itself and its individual test methods, to override the
@ -528,8 +587,7 @@ class HomeserverTestCase(TestCase):
config = kwargs["config"] config = kwargs["config"]
# Parse the config from a config dict into a HomeServerConfig # Parse the config from a config dict into a HomeServerConfig
config_obj = HomeServerConfig() config_obj = make_homeserver_config_obj(config)
config_obj.parse_config_dict(config, "", "")
kwargs["config"] = config_obj kwargs["config"] = config_obj
async def run_bg_updates() -> None: async def run_bg_updates() -> None: