From cc53c96bf813045c08593d4996deb57915fbb0e5 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Mon, 22 May 2023 13:25:39 +0100 Subject: [PATCH] Limit the size of the `HomeServerConfig` cache in trial test runs (#15646) ...to try to control memory usage. `HomeServerConfig`s hold on to many Jinja2 objects, which come out to over 0.5 MiB per config. Over the course of a full test run, the cache grows to ~360 entries. Limit it to 8 entries. Part of #15622. Signed-off-by: Sean Quah --- changelog.d/15646.misc | 1 + tests/unittest.py | 23 +++++++---------------- 2 files changed, 8 insertions(+), 16 deletions(-) create mode 100644 changelog.d/15646.misc diff --git a/changelog.d/15646.misc b/changelog.d/15646.misc new file mode 100644 index 000000000..872afe30b --- /dev/null +++ b/changelog.d/15646.misc @@ -0,0 +1 @@ +Limit the size of the `HomeServerConfig` cache in trial test runs. diff --git a/tests/unittest.py b/tests/unittest.py index 623c5a75a..c73195b32 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import functools import gc import hashlib import hmac @@ -150,7 +151,11 @@ def deepcopy_config(config: _TConfig) -> _TConfig: return new_config -_make_homeserver_config_obj_cache: Dict[str, Union[RootConfig, Config]] = {} +@functools.lru_cache(maxsize=8) +def _parse_config_dict(config: str) -> RootConfig: + config_obj = HomeServerConfig() + config_obj.parse_config_dict(json.loads(config), "", "") + return config_obj def make_homeserver_config_obj(config: Dict[str, Any]) -> RootConfig: @@ -164,21 +169,7 @@ def make_homeserver_config_obj(config: Dict[str, Any]) -> RootConfig: but it keeps a cache of `HomeServerConfig` instances and deepcopies them as needed, to avoid validating the whole configuration every time. """ - cache_key = json.dumps(config) - - if cache_key in _make_homeserver_config_obj_cache: - # Cache hit: reuse the existing instance - config_obj = _make_homeserver_config_obj_cache[cache_key] - else: - # Cache miss; create the actual instance - config_obj = HomeServerConfig() - config_obj.parse_config_dict(config, "", "") - - # Add to the cache - _make_homeserver_config_obj_cache[cache_key] = config_obj - - assert isinstance(config_obj, RootConfig) - + config_obj = _parse_config_dict(json.dumps(config, sort_keys=True)) return deepcopy_config(config_obj)