From c1b077cd194d009470ad081311e2729769c9750a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 3 Nov 2016 16:27:10 +0000 Subject: [PATCH 1/5] Now we have new-style metrics don't bother exporting legacy-named process ones --- synapse/metrics/process_collector.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/synapse/metrics/process_collector.py b/synapse/metrics/process_collector.py index 0e9558236..4df3a2fa2 100644 --- a/synapse/metrics/process_collector.py +++ b/synapse/metrics/process_collector.py @@ -110,22 +110,7 @@ def _process_fds(): def register_process_collector(process_metrics): - # Legacy synapse-invented metric names - - resource_metrics = process_metrics.make_subspace("resource") - - resource_metrics.register_collector(update_resource_metrics) - - # msecs - resource_metrics.register_callback("utime", lambda: rusage.ru_utime * 1000) - resource_metrics.register_callback("stime", lambda: rusage.ru_stime * 1000) - - # kilobytes - resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * 1024) - - process_metrics.register_callback("fds", _process_fds, labels=["type"]) - - # New prometheus-standard metric names + process_metrics.register_collector(update_resource_metrics) if HAVE_PROC_SELF_STAT: process_metrics.register_callback( From 93ebeb2aa8bf32e0c0f63ff4c933a5e2d43cc29a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 3 Nov 2016 16:37:09 +0000 Subject: [PATCH 2/5] Remove now-unused 'resource' import --- synapse/metrics/process_collector.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/metrics/process_collector.py b/synapse/metrics/process_collector.py index 4df3a2fa2..f812a80e9 100644 --- a/synapse/metrics/process_collector.py +++ b/synapse/metrics/process_collector.py @@ -13,12 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Because otherwise 'resource' collides with synapse.metrics.resource -from __future__ import absolute_import - import os import stat -from resource import getrusage, RUSAGE_SELF TICKS_PER_SEC = 100 @@ -49,7 +45,6 @@ STAT_FIELDS = { } -rusage = None stats = {} fd_counts = None @@ -65,9 +60,6 @@ if HAVE_PROC_STAT: def update_resource_metrics(): - global rusage - rusage = getrusage(RUSAGE_SELF) - if HAVE_PROC_SELF_STAT: global stats with open("/proc/self/stat") as s: From 5219f7e06039aa51b5e578917f7f2609b9944fda Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 3 Nov 2016 16:41:32 +0000 Subject: [PATCH 3/5] Since we don't export per-filetype fd counts any more, delete all the code related to that too --- synapse/metrics/process_collector.py | 40 +++------------------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/synapse/metrics/process_collector.py b/synapse/metrics/process_collector.py index f812a80e9..6fec3de39 100644 --- a/synapse/metrics/process_collector.py +++ b/synapse/metrics/process_collector.py @@ -14,7 +14,6 @@ # limitations under the License. import os -import stat TICKS_PER_SEC = 100 @@ -25,16 +24,6 @@ HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") HAVE_PROC_SELF_LIMITS = os.path.exists("/proc/self/limits") HAVE_PROC_SELF_FD = os.path.exists("/proc/self/fd") -TYPES = { - stat.S_IFSOCK: "SOCK", - stat.S_IFLNK: "LNK", - stat.S_IFREG: "REG", - stat.S_IFBLK: "BLK", - stat.S_IFDIR: "DIR", - stat.S_IFCHR: "CHR", - stat.S_IFIFO: "FIFO", -} - # Field indexes from /proc/self/stat, taken from the proc(5) manpage STAT_FIELDS = { "utime": 14, @@ -46,7 +35,6 @@ STAT_FIELDS = { stats = {} -fd_counts = None # In order to report process_start_time_seconds we need to know the # machine's boot time, because the value in /proc/self/stat is relative to @@ -72,33 +60,13 @@ def update_resource_metrics(): # we've lost the first two fields in PID and COMMAND above stats[name] = int(raw_stats[index - 3]) - global fd_counts - fd_counts = _process_fds() - - -def _process_fds(): - counts = {(k,): 0 for k in TYPES.values()} - counts[("other",)] = 0 +def _count_fds(): # Not every OS will have a /proc/self/fd directory if not HAVE_PROC_SELF_FD: - return counts + return 0 - for fd in os.listdir("/proc/self/fd"): - try: - s = os.stat("/proc/self/fd/%s" % (fd)) - fmt = stat.S_IFMT(s.st_mode) - if fmt in TYPES: - t = TYPES[fmt] - else: - t = "other" - - counts[(t,)] += 1 - except OSError: - # the dirh itself used by listdir() is usually missing by now - pass - - return counts + return len(os.listdir("/proc/self/fd")) def register_process_collector(process_metrics): @@ -135,7 +103,7 @@ def register_process_collector(process_metrics): if HAVE_PROC_SELF_FD: process_metrics.register_callback( "open_fds", - lambda: sum(fd_counts.values()) + lambda: _count_fds() ) if HAVE_PROC_SELF_LIMITS: From 2938a00825f34f271130a4c50fdb402f2dff8ead Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 3 Nov 2016 17:03:52 +0000 Subject: [PATCH 4/5] Rename the python-specific metrics now the docs claim that we have done --- synapse/metrics/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 7041da25c..2265e6e8d 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -111,18 +111,20 @@ def render_all(): return "\n".join(strs) -reactor_metrics = get_metrics_for("reactor") -tick_time = reactor_metrics.register_distribution("tick_time") -pending_calls_metric = reactor_metrics.register_distribution("pending_calls") +register_process_collector(get_metrics_for("process")) -gc_time = reactor_metrics.register_distribution("gc_time", labels=["gen"]) -gc_unreachable = reactor_metrics.register_counter("gc_unreachable", labels=["gen"]) -reactor_metrics.register_callback( +python_metrics = get_metrics_for("python") + +gc_time = python_metrics.register_distribution("gc_time", labels=["gen"]) +gc_unreachable = python_metrics.register_counter("gc_unreachable_total", labels=["gen"]) +python_metrics.register_callback( "gc_counts", lambda: {(i,): v for i, v in enumerate(gc.get_count())}, labels=["gen"] ) -register_process_collector(get_metrics_for("process")) +reactor_metrics = get_metrics_for("python.twisted.reactor") +tick_time = reactor_metrics.register_distribution("tick_time") +pending_calls_metric = reactor_metrics.register_distribution("pending_calls") def runUntilCurrentTimer(func): From 89e3e39d52ba6caa58c0a9d9dbb4e6d0461aa1f2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 3 Nov 2016 17:04:13 +0000 Subject: [PATCH 5/5] Fix copypasto error in metric rename table in docs --- docs/metrics-howto.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index 7aa4757a3..ca10799b0 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -51,9 +51,9 @@ python_gc_counts reactor_gc_counts The twisted-specific reactor metrics have been renamed. -==================================== ================= +==================================== ===================== New name Old name ------------------------------------- ----------------- -python_twisted_reactor_pending_calls reactor_tick_time +------------------------------------ --------------------- +python_twisted_reactor_pending_calls reactor_pending_calls python_twisted_reactor_tick_time reactor_tick_time -==================================== ================= +==================================== =====================