src/collectors/proc.plugin/proc_mdstat.c
// SPDX-License-Identifier: GPL-3.0-or-later
#include "plugin_proc.h"
#define PLUGIN_PROC_MODULE_MDSTAT_NAME "/proc/mdstat"
struct raid {
int redundant;
char *name;
uint32_t hash;
char *level;
RRDDIM *rd_health;
unsigned long long failed_disks;
RRDSET *st_disks;
RRDDIM *rd_down;
RRDDIM *rd_inuse;
unsigned long long total_disks;
unsigned long long inuse_disks;
RRDSET *st_operation;
RRDDIM *rd_check;
RRDDIM *rd_resync;
RRDDIM *rd_recovery;
RRDDIM *rd_reshape;
unsigned long long check;
unsigned long long resync;
unsigned long long recovery;
unsigned long long reshape;
RRDSET *st_finish;
RRDDIM *rd_finish_in;
unsigned long long finish_in;
RRDSET *st_speed;
RRDDIM *rd_speed;
unsigned long long speed;
char *mismatch_cnt_filename;
RRDSET *st_mismatch_cnt;
RRDDIM *rd_mismatch_cnt;
unsigned long long mismatch_cnt;
RRDSET *st_nonredundant;
RRDDIM *rd_nonredundant;
};
struct old_raid {
int redundant;
char *name;
uint32_t hash;
int found;
};
static inline char *remove_trailing_chars(char *s, char c)
{
while (*s) {
if (unlikely(*s == c)) {
*s = '\0';
}
s++;
}
return s;
}
static inline void make_chart_obsolete(char *name, const char *id_modifier)
{
char id[50 + 1];
RRDSET *st = NULL;
if (likely(name && id_modifier)) {
snprintfz(id, sizeof(id) - 1, "mdstat.%s_%s", name, id_modifier);
st = rrdset_find_active_byname_localhost(id);
if (likely(st))
rrdset_is_obsolete___safe_from_collector_thread(st);
}
}
static void add_labels_to_mdstat(struct raid *raid, RRDSET *st) {
rrdlabels_add(st->rrdlabels, "device", raid->name, RRDLABEL_SRC_AUTO);
rrdlabels_add(st->rrdlabels, "raid_level", raid->level, RRDLABEL_SRC_AUTO);
}
int do_proc_mdstat(int update_every, usec_t dt)
{
(void)dt;
static procfile *ff = NULL;
static int do_health = -1, do_nonredundant = -1, do_disks = -1, do_operations = -1, do_mismatch = -1,
do_mismatch_config = -1;
static int make_charts_obsolete = -1;
static char *mdstat_filename = NULL, *mismatch_cnt_filename = NULL;
static struct raid *raids = NULL;
static size_t raids_allocated = 0;
size_t raids_num = 0, raid_idx = 0, redundant_num = 0;
static struct old_raid *old_raids = NULL;
static size_t old_raids_allocated = 0;
size_t old_raid_idx = 0;
if (unlikely(do_health == -1)) {
do_health =
config_get_boolean("plugin:proc:/proc/mdstat", "faulty devices", CONFIG_BOOLEAN_YES);
do_nonredundant =
config_get_boolean("plugin:proc:/proc/mdstat", "nonredundant arrays availability", CONFIG_BOOLEAN_YES);
do_mismatch_config =
config_get_boolean_ondemand("plugin:proc:/proc/mdstat", "mismatch count", CONFIG_BOOLEAN_AUTO);
do_disks =
config_get_boolean("plugin:proc:/proc/mdstat", "disk stats", CONFIG_BOOLEAN_YES);
do_operations =
config_get_boolean("plugin:proc:/proc/mdstat", "operation status", CONFIG_BOOLEAN_YES);
make_charts_obsolete =
config_get_boolean("plugin:proc:/proc/mdstat", "make charts obsolete", CONFIG_BOOLEAN_YES);
char filename[FILENAME_MAX + 1];
snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/mdstat");
mdstat_filename = config_get("plugin:proc:/proc/mdstat", "filename to monitor", filename);
snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/block/%s/md/mismatch_cnt");
mismatch_cnt_filename = config_get("plugin:proc:/proc/mdstat", "mismatch_cnt filename to monitor", filename);
}
if (unlikely(!ff)) {
ff = procfile_open(mdstat_filename, " \t:", PROCFILE_FLAG_DEFAULT);
if (unlikely(!ff))
return 1;
}
ff = procfile_readall(ff);
if (unlikely(!ff))
return 0; // we return 0, so that we will retry opening it next time
size_t lines = procfile_lines(ff);
size_t words = 0;
if (unlikely(lines < 2)) {
collector_error("Cannot read /proc/mdstat. Expected 2 or more lines, read %zu.", lines);
return 1;
}
// find how many raids are there
size_t l;
raids_num = 0;
for (l = 1; l < lines - 2; l++) {
if (unlikely(procfile_lineword(ff, l, 1)[0] == 'a')) // check if the raid is active
raids_num++;
}
if (unlikely(!raids_num && !old_raids_allocated))
return 0; // we return 0, so that we will retry searching for raids next time
// allocate the memory we need;
if (unlikely(raids_num != raids_allocated)) {
for (raid_idx = 0; raid_idx < raids_allocated; raid_idx++) {
struct raid *raid = &raids[raid_idx];
freez(raid->name);
freez(raid->level);
freez(raid->mismatch_cnt_filename);
}
if (raids_num) {
raids = (struct raid *)reallocz(raids, raids_num * sizeof(struct raid));
memset(raids, 0, raids_num * sizeof(struct raid));
} else {
freez(raids);
raids = NULL;
}
raids_allocated = raids_num;
}
// loop through all lines except the first and the last ones
for (l = 1, raid_idx = 0; l < (lines - 2) && raid_idx < raids_num; l++) {
struct raid *raid = &raids[raid_idx];
raid->redundant = 0;
words = procfile_linewords(ff, l);
if (unlikely(words < 3))
continue;
if (unlikely(procfile_lineword(ff, l, 1)[0] != 'a'))
continue;
if (unlikely(!raid->name)) {
raid->name = strdupz(procfile_lineword(ff, l, 0));
raid->hash = simple_hash(raid->name);
raid->level = strdupz(procfile_lineword(ff, l, 2));
} else if (unlikely(strcmp(raid->name, procfile_lineword(ff, l, 0)))) {
freez(raid->name);
freez(raid->mismatch_cnt_filename);
freez(raid->level);
memset(raid, 0, sizeof(struct raid));
raid->name = strdupz(procfile_lineword(ff, l, 0));
raid->hash = simple_hash(raid->name);
raid->level = strdupz(procfile_lineword(ff, l, 2));
}
if (unlikely(!raid->name || !raid->name[0]))
continue;
raid_idx++;
// check if raid has disk status
l++;
words = procfile_linewords(ff, l);
if (words < 2 || procfile_lineword(ff, l, words - 1)[0] != '[')
continue;
// split inuse and total number of disks
if (likely(do_health || do_disks)) {
char *s = NULL, *str_total = NULL, *str_inuse = NULL;
s = procfile_lineword(ff, l, words - 2);
if (unlikely(s[0] != '[')) {
collector_error("Cannot read /proc/mdstat raid health status. Unexpected format: missing opening bracket.");
continue;
}
str_total = ++s;
while (*s) {
if (unlikely(*s == '/')) {
*s = '\0';
str_inuse = s + 1;
} else if (unlikely(*s == ']')) {
*s = '\0';
break;
}
s++;
}
if (unlikely(str_total[0] == '\0' || !str_inuse || str_inuse[0] == '\0')) {
collector_error("Cannot read /proc/mdstat raid health status. Unexpected format.");
continue;
}
raid->inuse_disks = str2ull(str_inuse, NULL);
raid->total_disks = str2ull(str_total, NULL);
raid->failed_disks = raid->total_disks - raid->inuse_disks;
}
raid->redundant = 1;
redundant_num++;
l++;
// check if any operation is performed on the raid
if (likely(do_operations)) {
char *s = NULL;
raid->check = 0;
raid->resync = 0;
raid->recovery = 0;
raid->reshape = 0;
raid->finish_in = 0;
raid->speed = 0;
words = procfile_linewords(ff, l);
if (likely(words < 2))
continue;
if (unlikely(procfile_lineword(ff, l, 0)[0] != '['))
continue;
if (unlikely(words < 7)) {
collector_error("Cannot read /proc/mdstat line. Expected 7 params, read %zu.", words);
continue;
}
char *word;
word = procfile_lineword(ff, l, 3);
remove_trailing_chars(word, '%');
unsigned long long percentage = (unsigned long long)(str2ndd(word, NULL) * 100);
// possible operations: check, resync, recovery, reshape
// 4-th character is unique for each operation so it is checked
switch (procfile_lineword(ff, l, 1)[3]) {
case 'c': // check
raid->check = percentage;
break;
case 'y': // resync
raid->resync = percentage;
break;
case 'o': // recovery
raid->recovery = percentage;
break;
case 'h': // reshape
raid->reshape = percentage;
break;
}
word = procfile_lineword(ff, l, 5);
s = remove_trailing_chars(word, 'm'); // remove trailing "min"
word += 7; // skip leading "finish="
if (likely(s > word))
raid->finish_in = (unsigned long long)(str2ndd(word, NULL) * 60);
word = procfile_lineword(ff, l, 6);
s = remove_trailing_chars(word, 'K'); // remove trailing "K/sec"
word += 6; // skip leading "speed="
if (likely(s > word))
raid->speed = str2ull(word, NULL);
}
}
// read mismatch_cnt files
if (do_mismatch == -1) {
if (do_mismatch_config == CONFIG_BOOLEAN_AUTO) {
if (raids_num > 50)
do_mismatch = CONFIG_BOOLEAN_NO;
else
do_mismatch = CONFIG_BOOLEAN_YES;
} else
do_mismatch = do_mismatch_config;
}
if (likely(do_mismatch)) {
for (raid_idx = 0; raid_idx < raids_num; raid_idx++) {
char filename[FILENAME_MAX + 1];
struct raid *raid = &raids[raid_idx];
if (likely(raid->redundant)) {
if (unlikely(!raid->mismatch_cnt_filename)) {
snprintfz(filename, FILENAME_MAX, mismatch_cnt_filename, raid->name);
raid->mismatch_cnt_filename = strdupz(filename);
}
if (unlikely(read_single_number_file(raid->mismatch_cnt_filename, &raid->mismatch_cnt))) {
collector_error("Cannot read file '%s'", raid->mismatch_cnt_filename);
do_mismatch = CONFIG_BOOLEAN_NO;
collector_error("Monitoring for mismatch count has been disabled");
break;
}
}
}
}
// check for disappeared raids
for (old_raid_idx = 0; old_raid_idx < old_raids_allocated; old_raid_idx++) {
struct old_raid *old_raid = &old_raids[old_raid_idx];
int found = 0;
for (raid_idx = 0; raid_idx < raids_num; raid_idx++) {
struct raid *raid = &raids[raid_idx];
if (unlikely(
raid->hash == old_raid->hash && !strcmp(raid->name, old_raid->name) &&
raid->redundant == old_raid->redundant))
found = 1;
}
old_raid->found = found;
}
int raid_disappeared = 0;
for (old_raid_idx = 0; old_raid_idx < old_raids_allocated; old_raid_idx++) {
struct old_raid *old_raid = &old_raids[old_raid_idx];
if (unlikely(!old_raid->found)) {
if (likely(make_charts_obsolete)) {
make_chart_obsolete(old_raid->name, "disks");
make_chart_obsolete(old_raid->name, "mismatch");
make_chart_obsolete(old_raid->name, "operation");
make_chart_obsolete(old_raid->name, "finish");
make_chart_obsolete(old_raid->name, "speed");
make_chart_obsolete(old_raid->name, "availability");
}
raid_disappeared = 1;
}
}
// allocate memory for nonredundant arrays
if (unlikely(raid_disappeared || old_raids_allocated != raids_num)) {
for (old_raid_idx = 0; old_raid_idx < old_raids_allocated; old_raid_idx++) {
freez(old_raids[old_raid_idx].name);
}
if (likely(raids_num)) {
old_raids = reallocz(old_raids, sizeof(struct old_raid) * raids_num);
memset(old_raids, 0, sizeof(struct old_raid) * raids_num);
} else {
freez(old_raids);
old_raids = NULL;
}
old_raids_allocated = raids_num;
for (old_raid_idx = 0; old_raid_idx < old_raids_allocated; old_raid_idx++) {
struct old_raid *old_raid = &old_raids[old_raid_idx];
struct raid *raid = &raids[old_raid_idx];
old_raid->name = strdupz(raid->name);
old_raid->hash = raid->hash;
old_raid->redundant = raid->redundant;
}
}
if (likely(do_health && redundant_num)) {
static RRDSET *st_mdstat_health = NULL;
if (unlikely(!st_mdstat_health)) {
st_mdstat_health = rrdset_create_localhost(
"mdstat",
"mdstat_health",
NULL,
"health",
"md.health",
"Faulty Devices In MD",
"failed disks",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_MDSTAT_NAME,
NETDATA_CHART_PRIO_MDSTAT_HEALTH,
update_every,
RRDSET_TYPE_LINE);
rrdset_isnot_obsolete___safe_from_collector_thread(st_mdstat_health);
}
if (!redundant_num) {
if (likely(make_charts_obsolete))
make_chart_obsolete("mdstat", "health");
} else {
for (raid_idx = 0; raid_idx < raids_num; raid_idx++) {
struct raid *raid = &raids[raid_idx];
if (likely(raid->redundant)) {
if (unlikely(!raid->rd_health && !(raid->rd_health = rrddim_find_active(st_mdstat_health, raid->name))))
raid->rd_health = rrddim_add(st_mdstat_health, raid->name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(st_mdstat_health, raid->rd_health, raid->failed_disks);
}
}
rrdset_done(st_mdstat_health);
}
}
for (raid_idx = 0; raid_idx < raids_num; raid_idx++) {
struct raid *raid = &raids[raid_idx];
char id[50 + 1];
char family[50 + 1];
if (likely(raid->redundant)) {
if (likely(do_disks)) {
snprintfz(id, sizeof(id) - 1, "%s_disks", raid->name);
if (unlikely(!raid->st_disks && !(raid->st_disks = rrdset_find_active_byname_localhost(id)))) {
snprintfz(family, sizeof(family) - 1, "%s (%s)", raid->name, raid->level);
raid->st_disks = rrdset_create_localhost(
"mdstat",
id,
NULL,
family,
"md.disks",
"Disks Stats",
"disks",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_MDSTAT_NAME,
NETDATA_CHART_PRIO_MDSTAT_DISKS + raid_idx * 10,
update_every,
RRDSET_TYPE_STACKED);
rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_disks);
add_labels_to_mdstat(raid, raid->st_disks);
}
if (unlikely(!raid->rd_inuse && !(raid->rd_inuse = rrddim_find_active(raid->st_disks, "inuse"))))
raid->rd_inuse = rrddim_add(raid->st_disks, "inuse", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
if (unlikely(!raid->rd_down && !(raid->rd_down = rrddim_find_active(raid->st_disks, "down"))))
raid->rd_down = rrddim_add(raid->st_disks, "down", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(raid->st_disks, raid->rd_inuse, raid->inuse_disks);
rrddim_set_by_pointer(raid->st_disks, raid->rd_down, raid->failed_disks);
rrdset_done(raid->st_disks);
}
if (likely(do_mismatch)) {
snprintfz(id, sizeof(id) - 1, "%s_mismatch", raid->name);
if (unlikely(!raid->st_mismatch_cnt && !(raid->st_mismatch_cnt = rrdset_find_active_byname_localhost(id)))) {
snprintfz(family, sizeof(family) - 1, "%s (%s)", raid->name, raid->level);
raid->st_mismatch_cnt = rrdset_create_localhost(
"mdstat",
id,
NULL,
family,
"md.mismatch_cnt",
"Mismatch Count",
"unsynchronized blocks",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_MDSTAT_NAME,
NETDATA_CHART_PRIO_MDSTAT_MISMATCH + raid_idx * 10,
update_every,
RRDSET_TYPE_LINE);
rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_mismatch_cnt);
add_labels_to_mdstat(raid, raid->st_mismatch_cnt);
}
if (unlikely(!raid->rd_mismatch_cnt && !(raid->rd_mismatch_cnt = rrddim_find_active(raid->st_mismatch_cnt, "count"))))
raid->rd_mismatch_cnt = rrddim_add(raid->st_mismatch_cnt, "count", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(raid->st_mismatch_cnt, raid->rd_mismatch_cnt, raid->mismatch_cnt);
rrdset_done(raid->st_mismatch_cnt);
}
if (likely(do_operations)) {
snprintfz(id, sizeof(id) - 1, "%s_operation", raid->name);
if (unlikely(!raid->st_operation && !(raid->st_operation = rrdset_find_active_byname_localhost(id)))) {
snprintfz(family, sizeof(family) - 1, "%s (%s)", raid->name, raid->level);
raid->st_operation = rrdset_create_localhost(
"mdstat",
id,
NULL,
family,
"md.status",
"Current Status",
"percent",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_MDSTAT_NAME,
NETDATA_CHART_PRIO_MDSTAT_OPERATION + raid_idx * 10,
update_every,
RRDSET_TYPE_LINE);
rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_operation);
add_labels_to_mdstat(raid, raid->st_operation);
}
if(unlikely(!raid->rd_check && !(raid->rd_check = rrddim_find_active(raid->st_operation, "check"))))
raid->rd_check = rrddim_add(raid->st_operation, "check", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
if(unlikely(!raid->rd_resync && !(raid->rd_resync = rrddim_find_active(raid->st_operation, "resync"))))
raid->rd_resync = rrddim_add(raid->st_operation, "resync", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
if(unlikely(!raid->rd_recovery && !(raid->rd_recovery = rrddim_find_active(raid->st_operation, "recovery"))))
raid->rd_recovery = rrddim_add(raid->st_operation, "recovery", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
if(unlikely(!raid->rd_reshape && !(raid->rd_reshape = rrddim_find_active(raid->st_operation, "reshape"))))
raid->rd_reshape = rrddim_add(raid->st_operation, "reshape", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(raid->st_operation, raid->rd_check, raid->check);
rrddim_set_by_pointer(raid->st_operation, raid->rd_resync, raid->resync);
rrddim_set_by_pointer(raid->st_operation, raid->rd_recovery, raid->recovery);
rrddim_set_by_pointer(raid->st_operation, raid->rd_reshape, raid->reshape);
rrdset_done(raid->st_operation);
snprintfz(id, sizeof(id) - 1, "%s_finish", raid->name);
if (unlikely(!raid->st_finish && !(raid->st_finish = rrdset_find_active_byname_localhost(id)))) {
snprintfz(family, sizeof(family) - 1, "%s (%s)", raid->name, raid->level);
raid->st_finish = rrdset_create_localhost(
"mdstat",
id,
NULL,
family,
"md.expected_time_until_operation_finish",
"Approximate Time Until Finish",
"seconds",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_MDSTAT_NAME,
NETDATA_CHART_PRIO_MDSTAT_FINISH + raid_idx * 10,
update_every, RRDSET_TYPE_LINE);
rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_finish);
add_labels_to_mdstat(raid, raid->st_finish);
}
if(unlikely(!raid->rd_finish_in && !(raid->rd_finish_in = rrddim_find_active(raid->st_finish, "finish_in"))))
raid->rd_finish_in = rrddim_add(raid->st_finish, "finish_in", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(raid->st_finish, raid->rd_finish_in, raid->finish_in);
rrdset_done(raid->st_finish);
snprintfz(id, sizeof(id) - 1, "%s_speed", raid->name);
if (unlikely(!raid->st_speed && !(raid->st_speed = rrdset_find_active_byname_localhost(id)))) {
snprintfz(family, sizeof(family) - 1, "%s (%s)", raid->name, raid->level);
raid->st_speed = rrdset_create_localhost(
"mdstat",
id,
NULL,
family,
"md.operation_speed",
"Operation Speed",
"KiB/s",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_MDSTAT_NAME,
NETDATA_CHART_PRIO_MDSTAT_SPEED + raid_idx * 10,
update_every,
RRDSET_TYPE_LINE);
rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_speed);
add_labels_to_mdstat(raid, raid->st_speed);
}
if (unlikely(!raid->rd_speed && !(raid->rd_speed = rrddim_find_active(raid->st_speed, "speed"))))
raid->rd_speed = rrddim_add(raid->st_speed, "speed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(raid->st_speed, raid->rd_speed, raid->speed);
rrdset_done(raid->st_speed);
}
} else {
if (likely(do_nonredundant)) {
snprintfz(id, sizeof(id) - 1, "%s_availability", raid->name);
if (unlikely(!raid->st_nonredundant && !(raid->st_nonredundant = rrdset_find_active_localhost(id)))) {
snprintfz(family, sizeof(family) - 1, "%s (%s)", raid->name, raid->level);
raid->st_nonredundant = rrdset_create_localhost(
"mdstat",
id,
NULL,
family,
"md.nonredundant",
"Nonredundant Array Availability",
"boolean",
PLUGIN_PROC_NAME,
PLUGIN_PROC_MODULE_MDSTAT_NAME,
NETDATA_CHART_PRIO_MDSTAT_NONREDUNDANT + raid_idx * 10,
update_every,
RRDSET_TYPE_LINE);
rrdset_isnot_obsolete___safe_from_collector_thread(raid->st_nonredundant);
add_labels_to_mdstat(raid, raid->st_nonredundant);
}
if (unlikely(!raid->rd_nonredundant && !(raid->rd_nonredundant = rrddim_find_active(raid->st_nonredundant, "available"))))
raid->rd_nonredundant = rrddim_add(raid->st_nonredundant, "available", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
rrddim_set_by_pointer(raid->st_nonredundant, raid->rd_nonredundant, 1);
rrdset_done(raid->st_nonredundant);
}
}
}
return 0;
}