2015-02-24 13:03:39 -05:00
|
|
|
# -*- coding: utf-8 -*-
|
2016-01-06 23:26:29 -05:00
|
|
|
# Copyright 2015, 2016 OpenMarket Ltd
|
2015-02-24 13:03:39 -05:00
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2015-03-05 11:15:21 -05:00
|
|
|
# Because otherwise 'resource' collides with synapse.metrics.resource
|
|
|
|
from __future__ import absolute_import
|
|
|
|
|
2015-03-04 12:13:09 -05:00
|
|
|
import logging
|
2015-09-07 11:45:48 -04:00
|
|
|
from resource import getrusage, RUSAGE_SELF
|
2015-08-13 06:38:59 -04:00
|
|
|
import functools
|
2015-04-01 14:04:55 -04:00
|
|
|
import os
|
|
|
|
import stat
|
2015-08-13 06:38:59 -04:00
|
|
|
import time
|
2016-05-09 05:13:25 -04:00
|
|
|
import gc
|
2015-08-13 06:38:59 -04:00
|
|
|
|
|
|
|
from twisted.internet import reactor
|
2015-03-04 12:13:09 -05:00
|
|
|
|
2015-03-10 11:21:03 -04:00
|
|
|
from .metric import (
|
|
|
|
CounterMetric, CallbackMetric, DistributionMetric, CacheMetric
|
|
|
|
)
|
2015-02-24 13:03:39 -05:00
|
|
|
|
|
|
|
|
2015-03-04 12:13:09 -05:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
2015-02-24 13:03:39 -05:00
|
|
|
# We'll keep all the available metrics in a single toplevel dict, one shared
|
|
|
|
# for the entire process. We don't currently support per-HomeServer instances
|
|
|
|
# of metrics, because in practice any one python VM will host only one
|
|
|
|
# HomeServer anyway. This makes a lot of implementation neater
|
|
|
|
all_metrics = {}
|
|
|
|
|
|
|
|
|
|
|
|
class Metrics(object):
|
|
|
|
""" A single Metrics object gives a (mutable) slice view of the all_metrics
|
|
|
|
dict, allowing callers to easily register new metrics that are namespaced
|
|
|
|
nicely."""
|
|
|
|
|
|
|
|
def __init__(self, name):
|
|
|
|
self.name_prefix = name
|
|
|
|
|
2015-03-04 14:43:46 -05:00
|
|
|
def _register(self, metric_class, name, *args, **kwargs):
|
2015-03-06 11:18:21 -05:00
|
|
|
full_name = "%s_%s" % (self.name_prefix, name)
|
2015-03-04 11:46:44 -05:00
|
|
|
|
2015-03-04 14:43:46 -05:00
|
|
|
metric = metric_class(full_name, *args, **kwargs)
|
2015-03-04 11:46:44 -05:00
|
|
|
|
2015-03-04 14:43:46 -05:00
|
|
|
all_metrics[full_name] = metric
|
2015-03-04 11:46:44 -05:00
|
|
|
return metric
|
|
|
|
|
2015-03-04 14:43:46 -05:00
|
|
|
def register_counter(self, *args, **kwargs):
|
|
|
|
return self._register(CounterMetric, *args, **kwargs)
|
2015-03-04 10:47:23 -05:00
|
|
|
|
2015-03-04 14:43:46 -05:00
|
|
|
def register_callback(self, *args, **kwargs):
|
|
|
|
return self._register(CallbackMetric, *args, **kwargs)
|
2015-03-04 10:47:23 -05:00
|
|
|
|
2015-03-10 11:21:03 -04:00
|
|
|
def register_distribution(self, *args, **kwargs):
|
|
|
|
return self._register(DistributionMetric, *args, **kwargs)
|
2015-03-04 10:47:23 -05:00
|
|
|
|
2015-03-04 14:43:46 -05:00
|
|
|
def register_cache(self, *args, **kwargs):
|
|
|
|
return self._register(CacheMetric, *args, **kwargs)
|
2015-03-04 10:47:23 -05:00
|
|
|
|
2015-02-24 13:03:39 -05:00
|
|
|
|
2015-03-06 11:18:21 -05:00
|
|
|
def get_metrics_for(pkg_name):
|
2015-02-24 13:03:39 -05:00
|
|
|
""" Returns a Metrics instance for conveniently creating metrics
|
|
|
|
namespaced with the given name prefix. """
|
2015-03-06 11:18:21 -05:00
|
|
|
|
|
|
|
# Convert a "package.name" to "package_name" because Prometheus doesn't
|
|
|
|
# let us use . in metric names
|
|
|
|
return Metrics(pkg_name.replace(".", "_"))
|
2015-02-24 13:03:39 -05:00
|
|
|
|
|
|
|
|
|
|
|
def render_all():
|
|
|
|
strs = []
|
|
|
|
|
2015-03-05 11:15:21 -05:00
|
|
|
# TODO(paul): Internal hack
|
|
|
|
update_resource_metrics()
|
|
|
|
|
2015-02-24 13:03:39 -05:00
|
|
|
for name in sorted(all_metrics.keys()):
|
2015-03-04 12:13:09 -05:00
|
|
|
try:
|
|
|
|
strs += all_metrics[name].render()
|
2015-03-12 12:45:05 -04:00
|
|
|
except Exception:
|
2015-03-04 12:13:09 -05:00
|
|
|
strs += ["# FAILED to render %s" % name]
|
|
|
|
logger.exception("Failed to render %s metric", name)
|
2015-02-24 13:03:39 -05:00
|
|
|
|
2015-03-12 12:24:38 -04:00
|
|
|
strs.append("") # to generate a final CRLF
|
2015-03-06 14:08:47 -05:00
|
|
|
|
2015-02-24 13:03:39 -05:00
|
|
|
return "\n".join(strs)
|
2015-03-05 11:15:21 -05:00
|
|
|
|
|
|
|
|
|
|
|
# Now register some standard process-wide state metrics, to give indications of
|
|
|
|
# process resource usage
|
|
|
|
|
|
|
|
rusage = None
|
|
|
|
|
2015-03-12 12:24:38 -04:00
|
|
|
|
2015-03-05 11:15:21 -05:00
|
|
|
def update_resource_metrics():
|
|
|
|
global rusage
|
|
|
|
rusage = getrusage(RUSAGE_SELF)
|
|
|
|
|
|
|
|
resource_metrics = get_metrics_for("process.resource")
|
|
|
|
|
|
|
|
# msecs
|
|
|
|
resource_metrics.register_callback("utime", lambda: rusage.ru_utime * 1000)
|
|
|
|
resource_metrics.register_callback("stime", lambda: rusage.ru_stime * 1000)
|
|
|
|
|
2015-09-07 11:45:48 -04:00
|
|
|
# kilobytes
|
|
|
|
resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * 1024)
|
2015-04-01 14:04:55 -04:00
|
|
|
|
|
|
|
TYPES = {
|
|
|
|
stat.S_IFSOCK: "SOCK",
|
|
|
|
stat.S_IFLNK: "LNK",
|
|
|
|
stat.S_IFREG: "REG",
|
|
|
|
stat.S_IFBLK: "BLK",
|
|
|
|
stat.S_IFDIR: "DIR",
|
|
|
|
stat.S_IFCHR: "CHR",
|
|
|
|
stat.S_IFIFO: "FIFO",
|
|
|
|
}
|
|
|
|
|
2015-04-01 14:17:38 -04:00
|
|
|
|
2015-04-01 14:04:55 -04:00
|
|
|
def _process_fds():
|
|
|
|
counts = {(k,): 0 for k in TYPES.values()}
|
|
|
|
counts[("other",)] = 0
|
|
|
|
|
2015-09-07 11:45:55 -04:00
|
|
|
# Not every OS will have a /proc/self/fd directory
|
|
|
|
if not os.path.exists("/proc/self/fd"):
|
|
|
|
return counts
|
|
|
|
|
2015-04-01 14:04:55 -04:00
|
|
|
for fd in os.listdir("/proc/self/fd"):
|
|
|
|
try:
|
|
|
|
s = os.stat("/proc/self/fd/%s" % (fd))
|
|
|
|
fmt = stat.S_IFMT(s.st_mode)
|
|
|
|
if fmt in TYPES:
|
|
|
|
t = TYPES[fmt]
|
|
|
|
else:
|
|
|
|
t = "other"
|
|
|
|
|
|
|
|
counts[(t,)] += 1
|
|
|
|
except OSError:
|
|
|
|
# the dirh itself used by listdir() is usually missing by now
|
|
|
|
pass
|
|
|
|
|
|
|
|
return counts
|
|
|
|
|
|
|
|
get_metrics_for("process").register_callback("fds", _process_fds, labels=["type"])
|
2015-08-13 06:38:59 -04:00
|
|
|
|
|
|
|
reactor_metrics = get_metrics_for("reactor")
|
|
|
|
tick_time = reactor_metrics.register_distribution("tick_time")
|
|
|
|
pending_calls_metric = reactor_metrics.register_distribution("pending_calls")
|
2016-05-09 05:13:25 -04:00
|
|
|
gc_time = reactor_metrics.register_distribution("gc_time")
|
2015-08-13 06:38:59 -04:00
|
|
|
|
|
|
|
|
|
|
|
def runUntilCurrentTimer(func):
|
|
|
|
|
|
|
|
@functools.wraps(func)
|
|
|
|
def f(*args, **kwargs):
|
2015-08-14 10:42:52 -04:00
|
|
|
now = reactor.seconds()
|
|
|
|
num_pending = 0
|
|
|
|
|
|
|
|
# _newTimedCalls is one long list of *all* pending calls. Below loop
|
|
|
|
# is based off of impl of reactor.runUntilCurrent
|
2015-08-18 06:47:00 -04:00
|
|
|
for delayed_call in reactor._newTimedCalls:
|
|
|
|
if delayed_call.time > now:
|
2015-08-14 10:42:52 -04:00
|
|
|
break
|
|
|
|
|
2015-08-18 06:47:00 -04:00
|
|
|
if delayed_call.delayed_time > 0:
|
2015-08-14 10:42:52 -04:00
|
|
|
continue
|
|
|
|
|
|
|
|
num_pending += 1
|
|
|
|
|
|
|
|
num_pending += len(reactor.threadCallQueue)
|
|
|
|
|
2015-08-13 06:41:57 -04:00
|
|
|
start = time.time() * 1000
|
2015-08-13 06:38:59 -04:00
|
|
|
ret = func(*args, **kwargs)
|
|
|
|
end = time.time() * 1000
|
|
|
|
tick_time.inc_by(end - start)
|
2015-08-14 10:42:52 -04:00
|
|
|
pending_calls_metric.inc_by(num_pending)
|
2016-05-09 05:13:25 -04:00
|
|
|
|
2016-05-13 11:31:08 -04:00
|
|
|
# Check if we need to do a manual GC (since its been disabled), and do
|
|
|
|
# one if necessary.
|
2016-05-09 05:13:25 -04:00
|
|
|
threshold = gc.get_threshold()
|
|
|
|
counts = gc.get_count()
|
|
|
|
|
|
|
|
start = time.time() * 1000
|
|
|
|
for i in [2, 1, 0]:
|
|
|
|
if threshold[i] < counts[i]:
|
|
|
|
logger.info("Collecting gc %d", i)
|
|
|
|
gc.collect(i)
|
|
|
|
end = time.time() * 1000
|
|
|
|
gc_time.inc_by(end - start)
|
|
|
|
|
2015-08-13 06:38:59 -04:00
|
|
|
return ret
|
|
|
|
|
|
|
|
return f
|
|
|
|
|
|
|
|
|
2015-08-18 06:51:08 -04:00
|
|
|
try:
|
|
|
|
# Ensure the reactor has all the attributes we expect
|
|
|
|
reactor.runUntilCurrent
|
|
|
|
reactor._newTimedCalls
|
|
|
|
reactor.threadCallQueue
|
|
|
|
|
2015-08-13 06:38:59 -04:00
|
|
|
# runUntilCurrent is called when we have pending calls. It is called once
|
|
|
|
# per iteratation after fd polling.
|
|
|
|
reactor.runUntilCurrent = runUntilCurrentTimer(reactor.runUntilCurrent)
|
2016-05-13 11:31:08 -04:00
|
|
|
|
|
|
|
# We manually run the GC each reactor tick so that we can get some metrics
|
|
|
|
# about time spent doing GC,
|
2016-05-09 05:13:25 -04:00
|
|
|
gc.disable()
|
2015-08-18 06:51:08 -04:00
|
|
|
except AttributeError:
|
|
|
|
pass
|