mirror of
https://mau.dev/maunium/synapse.git
synced 2024-08-25 11:45:30 +00:00
Merge pull request #612 from matrix-org/erikj/cache_size
Add environment variable SYNAPSE_CACHE_FACTOR, default it to 0.1
This commit is contained in:
commit
2a78dac60d
17
README.rst
17
README.rst
@ -566,3 +566,20 @@ Building internal API documentation::
|
|||||||
|
|
||||||
python setup.py build_sphinx
|
python setup.py build_sphinx
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Halp!! Synapse eats all my RAM!
|
||||||
|
===============================
|
||||||
|
|
||||||
|
Synapse's architecture is quite RAM hungry currently - we deliberately
|
||||||
|
cache a lot of recent room data and metadata in RAM in order to speed up
|
||||||
|
common requests. We'll improve this in future, but for now the easiest
|
||||||
|
way to either reduce the RAM usage (at the risk of slowing things down)
|
||||||
|
is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
|
||||||
|
variable. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1.0 will max out
|
||||||
|
at around 3-4GB of resident memory - this is what we currently run the
|
||||||
|
matrix.org on. The default setting is currently 0.1, which is probably
|
||||||
|
around a ~700MB footprint. You can dial it down further to 0.02 if
|
||||||
|
desired, which targets roughly ~512MB. Conversely you can dial it up if
|
||||||
|
you need performance for lots of users and have a box with a lot of RAM.
|
||||||
|
|
||||||
|
@ -3,6 +3,7 @@
|
|||||||
: ${WORKSPACE:="$(pwd)"}
|
: ${WORKSPACE:="$(pwd)"}
|
||||||
|
|
||||||
export PYTHONDONTWRITEBYTECODE=yep
|
export PYTHONDONTWRITEBYTECODE=yep
|
||||||
|
export SYNAPSE_CACHE_FACTOR=1
|
||||||
|
|
||||||
# Output test results as junit xml
|
# Output test results as junit xml
|
||||||
export TRIAL_FLAGS="--reporter=subunit"
|
export TRIAL_FLAGS="--reporter=subunit"
|
||||||
|
@ -130,6 +130,10 @@ class PresenceHandler(BaseHandler):
|
|||||||
for state in active_presence
|
for state in active_presence
|
||||||
}
|
}
|
||||||
|
|
||||||
|
metrics.register_callback(
|
||||||
|
"user_to_current_state_size", lambda: len(self.user_to_current_state)
|
||||||
|
)
|
||||||
|
|
||||||
now = self.clock.time_msec()
|
now = self.clock.time_msec()
|
||||||
for state in active_presence:
|
for state in active_presence:
|
||||||
self.wheel_timer.insert(
|
self.wheel_timer.insert(
|
||||||
|
@ -36,8 +36,6 @@ class ReceiptsHandler(BaseHandler):
|
|||||||
)
|
)
|
||||||
self.clock = self.hs.get_clock()
|
self.clock = self.hs.get_clock()
|
||||||
|
|
||||||
self._receipt_cache = None
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def received_client_receipt(self, room_id, receipt_type, user_id,
|
def received_client_receipt(self, room_id, receipt_type, user_id,
|
||||||
event_id):
|
event_id):
|
||||||
|
@ -28,6 +28,7 @@ from twisted.internet import defer
|
|||||||
|
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
import os
|
||||||
import functools
|
import functools
|
||||||
import inspect
|
import inspect
|
||||||
import threading
|
import threading
|
||||||
@ -38,6 +39,9 @@ logger = logging.getLogger(__name__)
|
|||||||
_CacheSentinel = object()
|
_CacheSentinel = object()
|
||||||
|
|
||||||
|
|
||||||
|
CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1))
|
||||||
|
|
||||||
|
|
||||||
class Cache(object):
|
class Cache(object):
|
||||||
|
|
||||||
def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False):
|
def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False):
|
||||||
@ -140,6 +144,8 @@ class CacheDescriptor(object):
|
|||||||
"""
|
"""
|
||||||
def __init__(self, orig, max_entries=1000, num_args=1, lru=True, tree=False,
|
def __init__(self, orig, max_entries=1000, num_args=1, lru=True, tree=False,
|
||||||
inlineCallbacks=False):
|
inlineCallbacks=False):
|
||||||
|
max_entries = int(max_entries * CACHE_SIZE_FACTOR)
|
||||||
|
|
||||||
self.orig = orig
|
self.orig = orig
|
||||||
|
|
||||||
if inlineCallbacks:
|
if inlineCallbacks:
|
||||||
|
@ -18,11 +18,15 @@ from synapse.util.caches import cache_counter, caches_by_name
|
|||||||
|
|
||||||
from blist import sorteddict
|
from blist import sorteddict
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1))
|
||||||
|
|
||||||
|
|
||||||
class StreamChangeCache(object):
|
class StreamChangeCache(object):
|
||||||
"""Keeps track of the stream positions of the latest change in a set of entities.
|
"""Keeps track of the stream positions of the latest change in a set of entities.
|
||||||
|
|
||||||
@ -33,7 +37,7 @@ class StreamChangeCache(object):
|
|||||||
old then the cache will simply return all given entities.
|
old then the cache will simply return all given entities.
|
||||||
"""
|
"""
|
||||||
def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}):
|
def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}):
|
||||||
self._max_size = max_size
|
self._max_size = int(max_size * CACHE_SIZE_FACTOR)
|
||||||
self._entity_to_key = {}
|
self._entity_to_key = {}
|
||||||
self._cache = sorteddict()
|
self._cache = sorteddict()
|
||||||
self._earliest_known_stream_pos = current_stream_pos
|
self._earliest_known_stream_pos = current_stream_pos
|
||||||
|
@ -35,7 +35,8 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
|
|||||||
def setUp(self):
|
def setUp(self):
|
||||||
self.as_yaml_files = []
|
self.as_yaml_files = []
|
||||||
config = Mock(
|
config = Mock(
|
||||||
app_service_config_files=self.as_yaml_files
|
app_service_config_files=self.as_yaml_files,
|
||||||
|
event_cache_size=1,
|
||||||
)
|
)
|
||||||
hs = yield setup_test_homeserver(config=config)
|
hs = yield setup_test_homeserver(config=config)
|
||||||
|
|
||||||
@ -109,7 +110,8 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
|
|||||||
self.as_yaml_files = []
|
self.as_yaml_files = []
|
||||||
|
|
||||||
config = Mock(
|
config = Mock(
|
||||||
app_service_config_files=self.as_yaml_files
|
app_service_config_files=self.as_yaml_files,
|
||||||
|
event_cache_size=1,
|
||||||
)
|
)
|
||||||
hs = yield setup_test_homeserver(config=config)
|
hs = yield setup_test_homeserver(config=config)
|
||||||
self.db_pool = hs.get_db_pool()
|
self.db_pool = hs.get_db_pool()
|
||||||
@ -438,7 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
|
|||||||
f1 = self._write_config(suffix="1")
|
f1 = self._write_config(suffix="1")
|
||||||
f2 = self._write_config(suffix="2")
|
f2 = self._write_config(suffix="2")
|
||||||
|
|
||||||
config = Mock(app_service_config_files=[f1, f2])
|
config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
|
||||||
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
||||||
|
|
||||||
ApplicationServiceStore(hs)
|
ApplicationServiceStore(hs)
|
||||||
@ -448,7 +450,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
|
|||||||
f1 = self._write_config(id="id", suffix="1")
|
f1 = self._write_config(id="id", suffix="1")
|
||||||
f2 = self._write_config(id="id", suffix="2")
|
f2 = self._write_config(id="id", suffix="2")
|
||||||
|
|
||||||
config = Mock(app_service_config_files=[f1, f2])
|
config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
|
||||||
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
||||||
|
|
||||||
with self.assertRaises(ConfigError) as cm:
|
with self.assertRaises(ConfigError) as cm:
|
||||||
@ -464,7 +466,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
|
|||||||
f1 = self._write_config(as_token="as_token", suffix="1")
|
f1 = self._write_config(as_token="as_token", suffix="1")
|
||||||
f2 = self._write_config(as_token="as_token", suffix="2")
|
f2 = self._write_config(as_token="as_token", suffix="2")
|
||||||
|
|
||||||
config = Mock(app_service_config_files=[f1, f2])
|
config = Mock(app_service_config_files=[f1, f2], event_cache_size=1)
|
||||||
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
hs = yield setup_test_homeserver(config=config, datastore=Mock())
|
||||||
|
|
||||||
with self.assertRaises(ConfigError) as cm:
|
with self.assertRaises(ConfigError) as cm:
|
||||||
|
Loading…
Reference in New Issue
Block a user