Matrix-Mirrors/synapse-product
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse-product.git synced 2024-12-24 08:09:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

Make cachedList go a bit faster

Browse Source
This commit is contained in:
Erik Johnston 2016-06-01 18:01:22 +01:00
parent 065e739d6e
commit 597013caa5
2 changed files with 45 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
synapse/metrics/metric.py
View File

@ -15,6 +15,7 @@
from itertools import chain from itertools import chain
from collections import Counter
# TODO(paul): I can't believe Python doesn't have one of these # TODO(paul): I can't believe Python doesn't have one of these
@ -55,30 +56,29 @@ class CounterMetric(BaseMetric):
"""The simplest kind of metric; one that stores a monotonically-increasing """The simplest kind of metric; one that stores a monotonically-increasing
integer that counts events.""" integer that counts events."""
__slots__ = ("counts")
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(CounterMetric, self).__init__(*args, **kwargs) super(CounterMetric, self).__init__(*args, **kwargs)
self.counts = {} self.counts = Counter()
# Scalar metrics are never empty # Scalar metrics are never empty
if self.is_scalar(): if self.is_scalar():
self.counts[()] = 0 self.counts[()] = 0
def inc_by(self, incr, *values): def inc_by(self, incr, *values):
if len(values) != self.dimension(): # if len(values) != self.dimension():
raise ValueError( # raise ValueError(
"Expected as many values to inc() as labels (%d)" % (self.dimension()) # "Expected as many values to inc() as labels (%d)" % (self.dimension())
) # )
# TODO: should assert that the tag values are all strings # TODO: should assert that the tag values are all strings
if values not in self.counts:
self.counts[values] = incr
else:
self.counts[values] += incr self.counts[values] += incr
def inc(self, *values): def inc(self, *values):
self.inc_by(1, *values) self.counts[values] += 1
def render_item(self, k): def render_item(self, k):
return ["%s%s %d" % (self.name, self._render_key(k), self.counts[k])] return ["%s%s %d" % (self.name, self._render_key(k), self.counts[k])]
@ -132,6 +132,8 @@ class CacheMetric(object):
This metric generates standard metric name pairs, so that monitoring rules This metric generates standard metric name pairs, so that monitoring rules
can easily be applied to measure hit ratio.""" can easily be applied to measure hit ratio."""
__slots__ = ("name", "hits", "total", "size")
def __init__(self, name, size_callback, labels=[]): def __init__(self, name, size_callback, labels=[]):
self.name = name self.name = name

36
synapse/util/caches/descriptors.py
View File

@ -32,6 +32,7 @@ import os
import functools import functools
import inspect import inspect
import threading import threading
import itertools
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -43,6 +44,14 @@ CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1))
class Cache(object): class Cache(object):
__slots__ = (
"cache",
"max_entries",
"name",
"keylen",
"sequence",
"thread",
)
def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False): def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False):
if lru: if lru:
@ -293,16 +302,21 @@ class CacheListDescriptor(object):
# cached is a dict arg -> deferred, where deferred results in a # cached is a dict arg -> deferred, where deferred results in a
# 2-tuple (`arg`, `result`) # 2-tuple (`arg`, `result`)
cached = {} results = {}
cached_defers = {}
missing = [] missing = []
for arg in list_args: for arg in list_args:
key = list(keyargs) key = list(keyargs)
key[self.list_pos] = arg key[self.list_pos] = arg
try: try:
res = cache.get(tuple(key)).observe() res = cache.get(tuple(key))
if not res.called:
res = res.observe()
res.addCallback(lambda r, arg: (arg, r), arg) res.addCallback(lambda r, arg: (arg, r), arg)
cached[arg] = res cached_defers[arg] = res
else:
results[arg] = res.result
except KeyError: except KeyError:
missing.append(arg) missing.append(arg)
@ -340,12 +354,22 @@ class CacheListDescriptor(object):
res = observer.observe() res = observer.observe()
res.addCallback(lambda r, arg: (arg, r), arg) res.addCallback(lambda r, arg: (arg, r), arg)
cached[arg] = res cached_defers[arg] = res
if cached_defers:
return preserve_context_over_deferred(defer.gatherResults( return preserve_context_over_deferred(defer.gatherResults(
cached.values(), cached_defers.values(),
consumeErrors=True, consumeErrors=True,
).addErrback(unwrapFirstError).addCallback(lambda res: dict(res))) ).addCallback(
lambda res: {
k: v
for k, v in itertools.chain(results.items(), res)
}
)).addErrback(
unwrapFirstError
)
else:
return results
obj.__dict__[self.orig.__name__] = wrapped obj.__dict__[self.orig.__name__] = wrapped