netdata/netdata

View on GitHub
src/go/collectors/go.d.plugin/modules/consul/charts.go

Summary

Maintainability
D
2 days
Test Coverage
// SPDX-License-Identifier: GPL-3.0-or-later

package consul

import (
    "fmt"

    "github.com/blang/semver/v4"

    "github.com/netdata/netdata/go/go.d.plugin/agent/module"
)

const (
    _ = module.Priority + iota
    prioKVSApplyTime
    prioKVSApplyOperations
    prioTXNApplyTime
    prioTXNApplyOperations
    prioRaftCommitTime
    prioRaftCommitsRate

    prioServerLeadershipStatus
    prioRaftLeaderLastContactTime
    prioRaftFollowerLastContactLeaderTime
    prioRaftLeaderElections
    prioRaftLeadershipTransitions

    prioAutopilotClusterHealthStatus
    prioAutopilotFailureTolerance
    prioAutopilotServerHealthStatus
    prioAutopilotServerStableTime
    prioAutopilotServerSerfStatus
    prioAutopilotServerVoterStatus

    prioNetworkLanRTT

    prioRPCRequests
    prioRPCRequestsExceeded
    prioRPCRequestsFailed

    prioRaftThreadMainSaturation
    prioRaftThreadFSMSaturation

    prioRaftFSMLastRestoreDuration
    prioRaftLeaderOldestLogAge
    prioRaftRPCInstallSnapshotTime

    prioBoltDBFreelistBytes
    prioBoltDBLogsPerBatch
    prioBoltDBStoreLogsTime

    prioMemoryAllocated
    prioMemorySys
    prioGCPauseTime

    prioServiceHealthCheckStatus
    prioNodeHealthCheckStatus

    prioLicenseExpirationTime
)

var (
    clientCharts = module.Charts{
        clientRPCRequestsRateChart.Copy(),
        clientRPCRequestsExceededRateChart.Copy(),
        clientRPCRequestsFailedRateChart.Copy(),

        memoryAllocatedChart.Copy(),
        memorySysChart.Copy(),
        gcPauseTimeChart.Copy(),

        licenseExpirationTimeChart.Copy(),
    }

    serverLeaderCharts = module.Charts{
        raftCommitTimeChart.Copy(),
        raftLeaderLastContactTimeChart.Copy(),
        raftCommitsRateChart.Copy(),
        raftLeaderOldestLogAgeChart.Copy(),
    }
    serverFollowerCharts = module.Charts{
        raftFollowerLastContactLeaderTimeChart.Copy(),
        raftRPCInstallSnapshotTimeChart.Copy(),
    }
    serverAutopilotHealthCharts = module.Charts{
        autopilotServerHealthStatusChart.Copy(),
        autopilotServerStableTimeChart.Copy(),
        autopilotServerSerfStatusChart.Copy(),
        autopilotServerVoterStatusChart.Copy(),
    }
    serverCommonCharts = module.Charts{
        kvsApplyTimeChart.Copy(),
        kvsApplyOperationsRateChart.Copy(),
        txnApplyTimeChart.Copy(),
        txnApplyOperationsRateChart.Copy(),

        autopilotClusterHealthStatusChart.Copy(),
        autopilotFailureTolerance.Copy(),

        raftLeaderElectionsRateChart.Copy(),
        raftLeadershipTransitionsRateChart.Copy(),
        serverLeadershipStatusChart.Copy(),

        networkLanRTTChart.Copy(),

        clientRPCRequestsRateChart.Copy(),
        clientRPCRequestsExceededRateChart.Copy(),
        clientRPCRequestsFailedRateChart.Copy(),

        raftThreadMainSaturationPercChart.Copy(),
        raftThreadFSMSaturationPercChart.Copy(),

        raftFSMLastRestoreDurationChart.Copy(),

        raftBoltDBFreelistBytesChart.Copy(),
        raftBoltDBLogsPerBatchChart.Copy(),
        raftBoltDBStoreLogsTimeChart.Copy(),

        memoryAllocatedChart.Copy(),
        memorySysChart.Copy(),
        gcPauseTimeChart.Copy(),

        licenseExpirationTimeChart.Copy(),
    }

    kvsApplyTimeChart = module.Chart{
        ID:       "kvs_apply_time",
        Title:    "KVS apply time",
        Units:    "ms",
        Fam:      "transaction timing",
        Ctx:      "consul.kvs_apply_time",
        Priority: prioKVSApplyTime,
        Dims: module.Dims{
            {ID: "kvs_apply_quantile=0.5", Name: "quantile_0.5", Div: precision * precision},
            {ID: "kvs_apply_quantile=0.9", Name: "quantile_0.9", Div: precision * precision},
            {ID: "kvs_apply_quantile=0.99", Name: "quantile_0.99", Div: precision * precision},
        },
    }
    kvsApplyOperationsRateChart = module.Chart{
        ID:       "kvs_apply_operations_rate",
        Title:    "KVS apply operations",
        Units:    "ops/s",
        Fam:      "transaction timing",
        Ctx:      "consul.kvs_apply_operations_rate",
        Priority: prioKVSApplyOperations,
        Dims: module.Dims{
            {ID: "kvs_apply_count", Name: "kvs_apply"},
        },
    }
    txnApplyTimeChart = module.Chart{
        ID:       "txn_apply_time",
        Title:    "Transaction apply time",
        Units:    "ms",
        Fam:      "transaction timing",
        Ctx:      "consul.txn_apply_time",
        Priority: prioTXNApplyTime,
        Dims: module.Dims{
            {ID: "txn_apply_quantile=0.5", Name: "quantile_0.5", Div: precision * precision},
            {ID: "txn_apply_quantile=0.9", Name: "quantile_0.9", Div: precision * precision},
            {ID: "txn_apply_quantile=0.99", Name: "quantile_0.99", Div: precision * precision},
        },
    }
    txnApplyOperationsRateChart = module.Chart{
        ID:       "txn_apply_operations_rate",
        Title:    "Transaction apply operations",
        Units:    "ops/s",
        Fam:      "transaction timing",
        Ctx:      "consul.txn_apply_operations_rate",
        Priority: prioTXNApplyOperations,
        Dims: module.Dims{
            {ID: "txn_apply_count", Name: "kvs_apply"},
        },
    }

    raftCommitTimeChart = module.Chart{
        ID:       "raft_commit_time",
        Title:    "Raft commit time",
        Units:    "ms",
        Fam:      "transaction timing",
        Ctx:      "consul.raft_commit_time",
        Priority: prioRaftCommitTime,
        Dims: module.Dims{
            {ID: "raft_commitTime_quantile=0.5", Name: "quantile_0.5", Div: precision * precision},
            {ID: "raft_commitTime_quantile=0.9", Name: "quantile_0.9", Div: precision * precision},
            {ID: "raft_commitTime_quantile=0.99", Name: "quantile_0.99", Div: precision * precision},
        },
    }
    raftCommitsRateChart = module.Chart{
        ID:       "raft_commits_rate",
        Title:    "Raft commits rate",
        Units:    "commits/s",
        Fam:      "transaction timing",
        Ctx:      "consul.raft_commits_rate",
        Priority: prioRaftCommitsRate,
        Dims: module.Dims{
            {ID: "raft_apply", Name: "commits", Div: precision, Algo: module.Incremental},
        },
    }

    autopilotClusterHealthStatusChart = module.Chart{
        ID:       "autopilot_health_status",
        Title:    "Autopilot cluster health status",
        Units:    "status",
        Fam:      "autopilot",
        Ctx:      "consul.autopilot_health_status",
        Priority: prioAutopilotClusterHealthStatus,
        Dims: module.Dims{
            {ID: "autopilot_healthy_yes", Name: "healthy"},
            {ID: "autopilot_healthy_no", Name: "unhealthy"},
        },
    }
    autopilotFailureTolerance = module.Chart{
        ID:       "autopilot_failure_tolerance",
        Title:    "Autopilot cluster failure tolerance",
        Units:    "servers",
        Fam:      "autopilot",
        Ctx:      "consul.autopilot_failure_tolerance",
        Priority: prioAutopilotFailureTolerance,
        Dims: module.Dims{
            {ID: "autopilot_failure_tolerance", Name: "failure_tolerance"},
        },
    }
    autopilotServerHealthStatusChart = module.Chart{
        ID:       "autopilot_server_health_status",
        Title:    "Autopilot server health status",
        Units:    "status",
        Fam:      "autopilot",
        Ctx:      "consul.autopilot_server_health_status",
        Priority: prioAutopilotServerHealthStatus,
        Dims: module.Dims{
            {ID: "autopilot_server_healthy_yes", Name: "healthy"},
            {ID: "autopilot_server_healthy_no", Name: "unhealthy"},
        },
    }
    autopilotServerStableTimeChart = module.Chart{
        ID:       "autopilot_server_stable_time",
        Title:    "Autopilot server stable time",
        Units:    "seconds",
        Fam:      "autopilot",
        Ctx:      "consul.autopilot_server_stable_time",
        Priority: prioAutopilotServerStableTime,
        Dims: module.Dims{
            {ID: "autopilot_server_stable_time", Name: "stable"},
        },
    }
    autopilotServerSerfStatusChart = module.Chart{
        ID:       "autopilot_server_serf_status",
        Title:    "Autopilot server Serf status",
        Units:    "status",
        Fam:      "autopilot",
        Ctx:      "consul.autopilot_server_serf_status",
        Priority: prioAutopilotServerSerfStatus,
        Dims: module.Dims{
            {ID: "autopilot_server_sefStatus_alive", Name: "alive"},
            {ID: "autopilot_server_sefStatus_failed", Name: "failed"},
            {ID: "autopilot_server_sefStatus_left", Name: "left"},
            {ID: "autopilot_server_sefStatus_none", Name: "none"},
        },
    }
    autopilotServerVoterStatusChart = module.Chart{
        ID:       "autopilot_server_voter_status",
        Title:    "Autopilot server Raft voting membership",
        Units:    "status",
        Fam:      "autopilot",
        Ctx:      "consul.autopilot_server_voter_status",
        Priority: prioAutopilotServerVoterStatus,
        Dims: module.Dims{
            {ID: "autopilot_server_voter_yes", Name: "voter"},
            {ID: "autopilot_server_voter_no", Name: "not_voter"},
        },
    }

    raftLeaderLastContactTimeChart = module.Chart{
        ID:       "raft_leader_last_contact_time",
        Title:    "Raft leader last contact time",
        Units:    "ms",
        Fam:      "leadership changes",
        Ctx:      "consul.raft_leader_last_contact_time",
        Priority: prioRaftLeaderLastContactTime,
        Dims: module.Dims{
            {ID: "raft_leader_lastContact_quantile=0.5", Name: "quantile_0.5", Div: precision * precision},
            {ID: "raft_leader_lastContact_quantile=0.9", Name: "quantile_0.9", Div: precision * precision},
            {ID: "raft_leader_lastContact_quantile=0.99", Name: "quantile_0.99", Div: precision * precision},
        },
    }
    raftFollowerLastContactLeaderTimeChart = module.Chart{
        ID:       "raft_follower_last_contact_leader_time",
        Title:    "Raft follower last contact with the leader time",
        Units:    "ms",
        Fam:      "leadership changes",
        Ctx:      "consul.raft_follower_last_contact_leader_time",
        Priority: prioRaftFollowerLastContactLeaderTime,
        Dims: module.Dims{
            {ID: "autopilot_server_lastContact_leader", Name: "leader_last_contact"},
        },
    }
    raftLeaderElectionsRateChart = module.Chart{
        ID:       "raft_leader_elections_rate",
        Title:    "Raft leader elections rate",
        Units:    "elections/s",
        Fam:      "leadership changes",
        Ctx:      "consul.raft_leader_elections_rate",
        Priority: prioRaftLeaderElections,
        Dims: module.Dims{
            {ID: "raft_state_candidate", Name: "leader", Algo: module.Incremental},
        },
    }
    raftLeadershipTransitionsRateChart = module.Chart{
        ID:       "raft_leadership_transitions_rate",
        Title:    "Raft leadership transitions rate",
        Units:    "transitions/s",
        Fam:      "leadership changes",
        Ctx:      "consul.raft_leadership_transitions_rate",
        Priority: prioRaftLeadershipTransitions,
        Dims: module.Dims{
            {ID: "raft_state_leader", Name: "leadership", Algo: module.Incremental},
        },
    }
    serverLeadershipStatusChart = module.Chart{
        ID:       "server_leadership_status",
        Title:    "Server leadership status",
        Units:    "status",
        Fam:      "leadership changes",
        Ctx:      "consul.server_leadership_status",
        Priority: prioServerLeadershipStatus,
        Dims: module.Dims{
            {ID: "server_isLeader_yes", Name: "leader"},
            {ID: "server_isLeader_no", Name: "not_leader"},
        },
    }

    networkLanRTTChart = module.Chart{
        ID:       "network_lan_rtt",
        Title:    "Network lan RTT",
        Units:    "ms",
        Fam:      "network rtt",
        Ctx:      "consul.network_lan_rtt",
        Type:     module.Area,
        Priority: prioNetworkLanRTT,
        Dims: module.Dims{
            {ID: "network_lan_rtt_min", Name: "min", Div: 1e6},
            {ID: "network_lan_rtt_max", Name: "max", Div: 1e6},
            {ID: "network_lan_rtt_avg", Name: "avg", Div: 1e6},
        },
    }

    clientRPCRequestsRateChart = module.Chart{
        ID:       "client_rpc_requests_rate",
        Title:    "Client RPC requests",
        Units:    "requests/s",
        Fam:      "rpc network activity",
        Ctx:      "consul.client_rpc_requests_rate",
        Priority: prioRPCRequests,
        Dims: module.Dims{
            {ID: "client_rpc", Name: "rpc", Algo: module.Incremental},
        },
    }
    clientRPCRequestsExceededRateChart = module.Chart{
        ID:       "client_rpc_requests_exceeded_rate",
        Title:    "Client rate-limited RPC requests",
        Units:    "requests/s",
        Fam:      "rpc network activity",
        Ctx:      "consul.client_rpc_requests_exceeded_rate",
        Priority: prioRPCRequestsExceeded,
        Dims: module.Dims{
            {ID: "client_rpc_exceeded", Name: "exceeded", Algo: module.Incremental},
        },
    }
    clientRPCRequestsFailedRateChart = module.Chart{
        ID:       "client_rpc_requests_failed_rate",
        Title:    "Client failed RPC requests",
        Units:    "requests/s",
        Fam:      "rpc network activity",
        Ctx:      "consul.client_rpc_requests_failed_rate",
        Priority: prioRPCRequestsFailed,
        Dims: module.Dims{
            {ID: "client_rpc_failed", Name: "failed", Algo: module.Incremental},
        },
    }

    raftThreadMainSaturationPercChart = module.Chart{
        ID:       "raft_thread_main_saturation_perc",
        Title:    "Raft main thread saturation",
        Units:    "percentage",
        Fam:      "raft saturation",
        Ctx:      "consul.raft_thread_main_saturation_perc",
        Priority: prioRaftThreadMainSaturation,
        Dims: module.Dims{
            {ID: "raft_thread_main_saturation_quantile=0.5", Name: "quantile_0.5", Div: precision * 10},
            {ID: "raft_thread_main_saturation_quantile=0.9", Name: "quantile_0.9", Div: precision * 10},
            {ID: "raft_thread_main_saturation_quantile=0.99", Name: "quantile_0.99", Div: precision * 10},
        },
    }
    raftThreadFSMSaturationPercChart = module.Chart{
        ID:       "raft_thread_fsm_saturation_perc",
        Title:    "Raft FSM thread saturation",
        Units:    "percentage",
        Fam:      "raft saturation",
        Ctx:      "consul.raft_thread_fsm_saturation_perc",
        Priority: prioRaftThreadFSMSaturation,
        Dims: module.Dims{
            {ID: "raft_thread_fsm_saturation_quantile=0.5", Name: "quantile_0.5", Div: precision * 10},
            {ID: "raft_thread_fsm_saturation_quantile=0.9", Name: "quantile_0.9", Div: precision * 10},
            {ID: "raft_thread_fsm_saturation_quantile=0.99", Name: "quantile_0.99", Div: precision * 10},
        },
    }

    raftFSMLastRestoreDurationChart = module.Chart{
        ID:       "raft_fsm_last_restore_duration",
        Title:    "Raft last restore duration",
        Units:    "ms",
        Fam:      "raft replication capacity",
        Ctx:      "consul.raft_fsm_last_restore_duration",
        Priority: prioRaftFSMLastRestoreDuration,
        Dims: module.Dims{
            {ID: "raft_fsm_lastRestoreDuration", Name: "last_restore_duration"},
        },
    }
    raftLeaderOldestLogAgeChart = module.Chart{
        ID:       "raft_leader_oldest_log_age",
        Title:    "Raft leader oldest log age",
        Units:    "seconds",
        Fam:      "raft replication capacity",
        Ctx:      "consul.raft_leader_oldest_log_age",
        Priority: prioRaftLeaderOldestLogAge,
        Dims: module.Dims{
            {ID: "raft_leader_oldestLogAge", Name: "oldest_log_age", Div: 1000},
        },
    }
    raftRPCInstallSnapshotTimeChart = module.Chart{
        ID:       "raft_rpc_install_snapshot_time",
        Title:    "Raft RPC install snapshot time",
        Units:    "ms",
        Fam:      "raft replication capacity",
        Ctx:      "consul.raft_rpc_install_snapshot_time",
        Priority: prioRaftRPCInstallSnapshotTime,
        Dims: module.Dims{
            {ID: "raft_rpc_installSnapshot_quantile=0.5", Name: "quantile_0.5", Div: precision * precision},
            {ID: "raft_rpc_installSnapshot_quantile=0.9", Name: "quantile_0.9", Div: precision * precision},
            {ID: "raft_rpc_installSnapshot_quantile=0.99", Name: "quantile_0.99", Div: precision * precision},
        },
    }

    raftBoltDBFreelistBytesChart = module.Chart{
        ID:       "raft_boltdb_freelist_bytes",
        Title:    "Raft BoltDB freelist",
        Units:    "bytes",
        Fam:      "boltdb performance",
        Ctx:      "consul.raft_boltdb_freelist_bytes",
        Priority: prioBoltDBFreelistBytes,
        Dims: module.Dims{
            {ID: "raft_boltdb_freelistBytes", Name: "freelist"},
        },
    }
    raftBoltDBLogsPerBatchChart = module.Chart{
        ID:       "raft_boltdb_logs_per_batch_rate",
        Title:    "Raft BoltDB logs written per batch",
        Units:    "logs/s",
        Fam:      "boltdb performance",
        Ctx:      "consul.raft_boltdb_logs_per_batch_rate",
        Priority: prioBoltDBLogsPerBatch,
        Dims: module.Dims{
            {ID: "raft_boltdb_logsPerBatch_sum", Name: "written", Algo: module.Incremental},
        },
    }

    raftBoltDBStoreLogsTimeChart = module.Chart{
        ID:       "raft_boltdb_store_logs_time",
        Title:    "Raft BoltDB store logs time",
        Units:    "ms",
        Fam:      "boltdb performance",
        Ctx:      "consul.raft_boltdb_store_logs_time",
        Priority: prioBoltDBStoreLogsTime,
        Dims: module.Dims{
            {ID: "raft_boltdb_storeLogs_quantile=0.5", Name: "quantile_0.5", Div: precision * precision},
            {ID: "raft_boltdb_storeLogs_quantile=0.9", Name: "quantile_0.9", Div: precision * precision},
            {ID: "raft_boltdb_storeLogs_quantile=0.99", Name: "quantile_0.99", Div: precision * precision},
        },
    }

    memoryAllocatedChart = module.Chart{
        ID:       "memory_allocated",
        Title:    "Memory allocated by the Consul process",
        Units:    "bytes",
        Fam:      "memory",
        Ctx:      "consul.memory_allocated",
        Priority: prioMemoryAllocated,
        Dims: module.Dims{
            {ID: "runtime_alloc_bytes", Name: "allocated"},
        },
    }
    memorySysChart = module.Chart{
        ID:       "memory_sys",
        Title:    "Memory obtained from the OS",
        Units:    "bytes",
        Fam:      "memory",
        Ctx:      "consul.memory_sys",
        Priority: prioMemorySys,
        Dims: module.Dims{
            {ID: "runtime_sys_bytes", Name: "sys"},
        },
    }

    gcPauseTimeChart = module.Chart{
        ID:       "gc_pause_time",
        Title:    "Garbage collection stop-the-world pause time",
        Units:    "seconds",
        Fam:      "garbage collection",
        Ctx:      "consul.gc_pause_time",
        Priority: prioGCPauseTime,
        Dims: module.Dims{
            {ID: "runtime_total_gc_pause_ns", Name: "gc_pause", Algo: module.Incremental, Div: 1e9},
        },
    }

    licenseExpirationTimeChart = module.Chart{
        ID:       "license_expiration_time",
        Title:    "License expiration time",
        Units:    "seconds",
        Fam:      "license",
        Ctx:      "consul.license_expiration_time",
        Priority: prioLicenseExpirationTime,
        Dims: module.Dims{
            {ID: "system_licenseExpiration", Name: "license_expiration"},
        },
    }
)

var (
    serviceHealthCheckStatusChartTmpl = module.Chart{
        ID:       "health_check_%s_status",
        Title:    "Service health check status",
        Units:    "status",
        Fam:      "service health checks",
        Ctx:      "consul.service_health_check_status",
        Priority: prioServiceHealthCheckStatus,
        Dims: module.Dims{
            {ID: "health_check_%s_passing_status", Name: "passing"},
            {ID: "health_check_%s_critical_status", Name: "critical"},
            {ID: "health_check_%s_maintenance_status", Name: "maintenance"},
            {ID: "health_check_%s_warning_status", Name: "warning"},
        },
    }
    nodeHealthCheckStatusChartTmpl = module.Chart{
        ID:       "health_check_%s_status",
        Title:    "Node health check status",
        Units:    "status",
        Fam:      "node health checks",
        Ctx:      "consul.node_health_check_status",
        Priority: prioNodeHealthCheckStatus,
        Dims: module.Dims{
            {ID: "health_check_%s_passing_status", Name: "passing"},
            {ID: "health_check_%s_critical_status", Name: "critical"},
            {ID: "health_check_%s_maintenance_status", Name: "maintenance"},
            {ID: "health_check_%s_warning_status", Name: "warning"},
        },
    }
)

func (c *Consul) addGlobalCharts() {
    if !c.isTelemetryPrometheusEnabled() {
        return
    }

    var charts *module.Charts

    if !c.isServer() {
        charts = clientCharts.Copy()
    } else {
        charts = serverCommonCharts.Copy()

        // can't really rely on checking if a response contains a metric due to retention of some metrics
        // https://github.com/hashicorp/go-metrics/blob/b6d5c860c07ef6eeec89f4a662c7b452dd4d0c93/prometheus/prometheus.go#L75-L76
        if c.version != nil {
            if c.version.LT(semver.Version{Major: 1, Minor: 13, Patch: 0}) {
                _ = charts.Remove(raftThreadMainSaturationPercChart.ID)
                _ = charts.Remove(raftThreadFSMSaturationPercChart.ID)
            }
            if c.version.LT(semver.Version{Major: 1, Minor: 11, Patch: 0}) {
                _ = charts.Remove(kvsApplyTimeChart.ID)
                _ = charts.Remove(kvsApplyOperationsRateChart.ID)
                _ = charts.Remove(txnApplyTimeChart.ID)
                _ = charts.Remove(txnApplyOperationsRateChart.ID)
                _ = charts.Remove(raftBoltDBFreelistBytesChart.ID)
            }
        }
    }

    if !c.hasLicense() {
        _ = charts.Remove(licenseExpirationTimeChart.ID)
    }

    for _, chart := range *charts {
        chart.Labels = []module.Label{
            {Key: "datacenter", Value: c.cfg.Config.Datacenter},
            {Key: "node_name", Value: c.cfg.Config.NodeName},
        }
    }

    if err := c.Charts().Add(*charts.Copy()...); err != nil {
        c.Warning(err)
    }
}

func (c *Consul) addServerAutopilotHealthCharts() {
    charts := serverAutopilotHealthCharts.Copy()

    for _, chart := range *charts {
        chart.Labels = []module.Label{
            {Key: "datacenter", Value: c.cfg.Config.Datacenter},
            {Key: "node_name", Value: c.cfg.Config.NodeName},
        }
    }

    if err := c.Charts().Add(*charts...); err != nil {
        c.Warning(err)
    }
}

func newServiceHealthCheckChart(check *agentCheck) *module.Chart {
    chart := serviceHealthCheckStatusChartTmpl.Copy()
    chart.ID = fmt.Sprintf(chart.ID, check.CheckID)
    chart.Labels = []module.Label{
        {Key: "node_name", Value: check.Node},
        {Key: "check_name", Value: check.Name},
        {Key: "service_name", Value: check.ServiceName},
    }
    for _, d := range chart.Dims {
        d.ID = fmt.Sprintf(d.ID, check.CheckID)
    }
    return chart
}

func newNodeHealthCheckChart(check *agentCheck) *module.Chart {
    chart := nodeHealthCheckStatusChartTmpl.Copy()
    chart.ID = fmt.Sprintf(chart.ID, check.CheckID)
    chart.Labels = []module.Label{
        {Key: "node_name", Value: check.Node},
        {Key: "check_name", Value: check.Name},
    }
    for _, d := range chart.Dims {
        d.ID = fmt.Sprintf(d.ID, check.CheckID)
    }
    return chart
}

func (c *Consul) addHealthCheckCharts(check *agentCheck) {
    var chart *module.Chart

    if check.ServiceName != "" {
        chart = newServiceHealthCheckChart(check)
    } else {
        chart = newNodeHealthCheckChart(check)
    }

    chart.Labels = append(chart.Labels, module.Label{
        Key:   "datacenter",
        Value: c.cfg.Config.Datacenter,
    })

    if err := c.Charts().Add(chart); err != nil {
        c.Warning(err)
    }
}

func (c *Consul) removeHealthCheckCharts(checkID string) {
    id := fmt.Sprintf("health_check_%s_status", checkID)

    chart := c.Charts().Get(id)
    if chart == nil {
        c.Warningf("failed to remove '%s' chart: the chart does not exist", id)
        return
    }

    chart.MarkRemove()
    chart.MarkNotCreated()
}

func (c *Consul) addLeaderCharts() {
    charts := serverLeaderCharts.Copy()

    for _, chart := range *charts {
        chart.Labels = []module.Label{
            {Key: "datacenter", Value: c.cfg.Config.Datacenter},
            {Key: "node_name", Value: c.cfg.Config.NodeName},
        }
    }

    if err := c.Charts().Add(*charts...); err != nil {
        c.Warning(err)
    }
}

func (c *Consul) removeLeaderCharts() {
    s := make(map[string]bool)
    for _, v := range serverLeaderCharts {
        s[v.ID] = true
    }

    for _, v := range *c.Charts() {
        if s[v.ID] {
            v.MarkRemove()
            v.MarkNotCreated()
        }
    }
}

func (c *Consul) addFollowerCharts() {
    charts := serverFollowerCharts.Copy()
    if c.isCloudManaged() {
        // 'autopilot_server_lastContact_leader' comes from 'operator/autopilot/health' which is disabled
        _ = charts.Remove(raftFollowerLastContactLeaderTimeChart.ID)
    }

    for _, chart := range *charts {
        chart.Labels = []module.Label{
            {Key: "datacenter", Value: c.cfg.Config.Datacenter},
            {Key: "node_name", Value: c.cfg.Config.NodeName},
        }
    }

    if err := c.Charts().Add(*charts...); err != nil {
        c.Warning(err)
    }
}

func (c *Consul) removeFollowerCharts() {
    s := make(map[string]bool)
    for _, v := range serverFollowerCharts {
        s[v.ID] = true
    }

    for _, v := range *c.Charts() {
        if s[v.ID] {
            v.MarkRemove()
            v.MarkNotCreated()
        }
    }
}