src/collectors/python.d.plugin/go_expvar/go_expvar.chart.py
# -*- coding: utf-8 -*-
# Description: go_expvar netdata python.d module
# Author: Jan Kral (kralewitz)
# SPDX-License-Identifier: GPL-3.0-or-later
from __future__ import division
import json
from collections import namedtuple
from bases.FrameworkServices.UrlService import UrlService
MEMSTATS_ORDER = [
'memstats_heap',
'memstats_stack',
'memstats_mspan',
'memstats_mcache',
'memstats_sys',
'memstats_live_objects',
'memstats_gc_pauses',
]
MEMSTATS_CHARTS = {
'memstats_heap': {
'options': ['heap', 'memory: size of heap memory structures', 'KiB', 'memstats',
'expvar.memstats.heap', 'line'],
'lines': [
['memstats_heap_alloc', 'alloc', 'absolute', 1, 1024],
['memstats_heap_inuse', 'inuse', 'absolute', 1, 1024]
]
},
'memstats_stack': {
'options': ['stack', 'memory: size of stack memory structures', 'KiB', 'memstats',
'expvar.memstats.stack', 'line'],
'lines': [
['memstats_stack_inuse', 'inuse', 'absolute', 1, 1024]
]
},
'memstats_mspan': {
'options': ['mspan', 'memory: size of mspan memory structures', 'KiB', 'memstats',
'expvar.memstats.mspan', 'line'],
'lines': [
['memstats_mspan_inuse', 'inuse', 'absolute', 1, 1024]
]
},
'memstats_mcache': {
'options': ['mcache', 'memory: size of mcache memory structures', 'KiB', 'memstats',
'expvar.memstats.mcache', 'line'],
'lines': [
['memstats_mcache_inuse', 'inuse', 'absolute', 1, 1024]
]
},
'memstats_live_objects': {
'options': ['live_objects', 'memory: number of live objects', 'objects', 'memstats',
'expvar.memstats.live_objects', 'line'],
'lines': [
['memstats_live_objects', 'live']
]
},
'memstats_sys': {
'options': ['sys', 'memory: size of reserved virtual address space', 'KiB', 'memstats',
'expvar.memstats.sys', 'line'],
'lines': [
['memstats_sys', 'sys', 'absolute', 1, 1024]
]
},
'memstats_gc_pauses': {
'options': ['gc_pauses', 'memory: average duration of GC pauses', 'ns', 'memstats',
'expvar.memstats.gc_pauses', 'line'],
'lines': [
['memstats_gc_pauses', 'avg']
]
}
}
EXPVAR = namedtuple(
"EXPVAR",
[
"key",
"type",
"id",
]
)
def flatten(d, top='', sep='.'):
items = []
for key, val in d.items():
nkey = top + sep + key if top else key
if isinstance(val, dict):
items.extend(flatten(val, nkey, sep=sep).items())
else:
items.append((nkey, val))
return dict(items)
class Service(UrlService):
def __init__(self, configuration=None, name=None):
UrlService.__init__(self, configuration=configuration, name=name)
# if memstats collection is enabled, add the charts and their order
if self.configuration.get('collect_memstats'):
self.definitions = dict(MEMSTATS_CHARTS)
self.order = list(MEMSTATS_ORDER)
else:
self.definitions = dict()
self.order = list()
# if extra charts are defined, parse their config
extra_charts = self.configuration.get('extra_charts')
if extra_charts:
self._parse_extra_charts_config(extra_charts)
def check(self):
"""
Check if the module can collect data:
1) At least one JOB configuration has to be specified
2) The JOB configuration needs to define the URL and either collect_memstats must be enabled or at least one
extra_chart must be defined.
The configuration and URL check is provided by the UrlService class.
"""
if not (self.configuration.get('extra_charts') or self.configuration.get('collect_memstats')):
self.error('Memstats collection is disabled and no extra_charts are defined, disabling module.')
return False
return UrlService.check(self)
def _parse_extra_charts_config(self, extra_charts_config):
# a place to store the expvar keys and their types
self.expvars = list()
for chart in extra_charts_config:
chart_dict = dict()
chart_id = chart.get('id')
chart_lines = chart.get('lines')
chart_opts = chart.get('options', dict())
if not all([chart_id, chart_lines]):
self.info('Chart {0} has no ID or no lines defined, skipping'.format(chart))
continue
chart_dict['options'] = [
chart_opts.get('name', ''),
chart_opts.get('title', ''),
chart_opts.get('units', ''),
chart_opts.get('family', ''),
chart_opts.get('context', ''),
chart_opts.get('chart_type', 'line')
]
chart_dict['lines'] = list()
# add the lines to the chart
for line in chart_lines:
ev_key = line.get('expvar_key')
ev_type = line.get('expvar_type')
line_id = line.get('id')
if not all([ev_key, ev_type, line_id]):
self.info('Line missing expvar_key, expvar_type, or line_id, skipping: {0}'.format(line))
continue
if ev_type not in ['int', 'float']:
self.info('Unsupported expvar_type "{0}". Must be "int" or "float"'.format(ev_type))
continue
# self.expvars[ev_key] = (ev_type, line_id)
self.expvars.append(EXPVAR(ev_key, ev_type, line_id))
chart_dict['lines'].append(
[
line.get('id', ''),
line.get('name', ''),
line.get('algorithm', ''),
line.get('multiplier', 1),
line.get('divisor', 100 if ev_type == 'float' else 1),
line.get('hidden', False)
]
)
self.order.append(chart_id)
self.definitions[chart_id] = chart_dict
def _get_data(self):
"""
Format data received from http request
:return: dict
"""
raw_data = self._get_raw_data()
if not raw_data:
return None
data = json.loads(raw_data)
expvars = dict()
if self.configuration.get('collect_memstats'):
expvars.update(self._parse_memstats(data))
if self.configuration.get('extra_charts'):
# the memstats part of the data has been already parsed, so we remove it before flattening and checking
# the rest of the data, thus avoiding needless iterating over the multiply nested memstats dict.
del (data['memstats'])
flattened = flatten(data)
for ev in self.expvars:
v = flattened.get(ev.key)
if v is None:
continue
try:
if ev.type == 'int':
expvars[ev.id] = int(v)
elif ev.type == 'float':
expvars[ev.id] = float(v) * 100
except ValueError:
self.info('Failed to parse value for key {0} as {1}, ignoring key.'.format(ev.key, ev.type))
return None
return expvars
@staticmethod
def _parse_memstats(data):
memstats = data['memstats']
# calculate the number of live objects in memory
live_objs = int(memstats['Mallocs']) - int(memstats['Frees'])
# calculate GC pause times average
# the Go runtime keeps the last 256 GC pause durations in a circular buffer,
# so we need to filter out the 0 values before the buffer is filled
gc_pauses = memstats['PauseNs']
try:
gc_pause_avg = sum(gc_pauses) / len([x for x in gc_pauses if x > 0])
# no GC cycles have occurred yet
except ZeroDivisionError:
gc_pause_avg = 0
return {
'memstats_heap_alloc': memstats['HeapAlloc'],
'memstats_heap_inuse': memstats['HeapInuse'],
'memstats_stack_inuse': memstats['StackInuse'],
'memstats_mspan_inuse': memstats['MSpanInuse'],
'memstats_mcache_inuse': memstats['MCacheInuse'],
'memstats_sys': memstats['Sys'],
'memstats_live_objects': live_objs,
'memstats_gc_pauses': gc_pause_avg,
}