LibreNMS/Data/Store/Rrd.php
<?php
/**
* Rrd.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* @link https://www.librenms.org
*
* @copyright 2018 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/
namespace LibreNMS\Data\Store;
use App\Polling\Measure\Measurement;
use LibreNMS\Config;
use LibreNMS\Enum\ImageFormat;
use LibreNMS\Exceptions\FileExistsException;
use LibreNMS\Exceptions\RrdGraphException;
use LibreNMS\Proc;
use LibreNMS\Util\Debug;
use LibreNMS\Util\Rewrite;
use Log;
use Symfony\Component\Process\Process;
class Rrd extends BaseDatastore
{
private $disabled = false;
/** @var Proc */
private $sync_process;
/** @var Proc */
private $async_process;
private $rrd_dir;
private $version;
private $rrdcached;
private $rra;
private $step;
public function __construct()
{
parent::__construct();
$this->rrdcached = Config::get('rrdcached', false);
$this->init();
$this->rrd_dir = Config::get('rrd_dir', Config::get('install_dir') . '/rrd');
$this->step = Config::get('rrd.step', 300);
$this->rra = Config::get(
'rrd_rra',
'RRA:AVERAGE:0.5:1:2016 RRA:AVERAGE:0.5:6:1440 RRA:AVERAGE:0.5:24:1440 RRA:AVERAGE:0.5:288:1440 ' .
' RRA:MIN:0.5:1:2016 RRA:MIN:0.5:6:1440 RRA:MIN:0.5:24:1440 RRA:MIN:0.5:288:1440 ' .
' RRA:MAX:0.5:1:2016 RRA:MAX:0.5:6:1440 RRA:MAX:0.5:24:1440 RRA:MAX:0.5:288:1440 ' .
' RRA:LAST:0.5:1:2016 '
);
$this->version = Config::get('rrdtool_version', '1.4');
}
public function getName()
{
return 'RRD';
}
public static function isEnabled()
{
return Config::get('rrd.enable', true);
}
/**
* Opens up a pipe to RRDTool using handles provided
*
* @param bool $dual_process start an additional process that's output should be read after every command
* @return bool the process(s) have been successfully started
*/
public function init($dual_process = true)
{
$command = Config::get('rrdtool', 'rrdtool') . ' -';
$descriptor_spec = [
0 => ['pipe', 'r'], // stdin is a pipe that the child will read from
1 => ['pipe', 'w'], // stdout is a pipe that the child will write to
2 => ['pipe', 'w'], // stderr is a pipe that the child will write to
];
$cwd = Config::get('rrd_dir');
if (! $this->isSyncRunning()) {
$this->sync_process = new Proc($command, $descriptor_spec, $cwd);
}
if ($dual_process && ! $this->isAsyncRunning()) {
$this->async_process = new Proc($command, $descriptor_spec, $cwd);
$this->async_process->setSynchronous(false);
}
return $this->isSyncRunning() && ($dual_process ? $this->isAsyncRunning() : true);
}
public function isSyncRunning()
{
return isset($this->sync_process) && $this->sync_process->isRunning();
}
public function isAsyncRunning()
{
return isset($this->async_process) && $this->async_process->isRunning();
}
/**
* Close all open rrdtool processes.
* This should be done before exiting
*/
public function close()
{
if ($this->isSyncRunning()) {
$this->sync_process->close('quit');
}
if ($this->isAsyncRunning()) {
$this->async_process->close('quit');
}
}
/**
* rrdtool backend implementation of data_update
*
* Tags:
* rrd_def RrdDefinition
* rrd_name array|string: the rrd filename, will be processed with rrd_name()
* rrd_oldname array|string: old rrd filename to rename, will be processed with rrd_name()
* rrd_step int: rrd step, defaults to 300
*
* @param array $device device array
* @param string $measurement the name of this measurement (if no rrd_name tag is given, this will be used to name the file)
* @param array $tags tags to pass additional info to rrdtool
* @param array $fields data values to update
*/
public function put($device, $measurement, $tags, $fields)
{
$rrd_name = isset($tags['rrd_name']) ? $tags['rrd_name'] : $measurement;
$step = isset($tags['rrd_step']) ? $tags['rrd_step'] : $this->step;
if (! empty($tags['rrd_oldname'])) {
self::renameFile($device, $tags['rrd_oldname'], $rrd_name);
}
if (isset($tags['rrd_proxmox_name'])) {
$pmxvars = $tags['rrd_proxmox_name'];
$rrd = self::proxmoxName($pmxvars['pmxcluster'], $pmxvars['vmid'], $pmxvars['vmport']);
} else {
$rrd = self::name($device['hostname'], $rrd_name);
}
if (isset($tags['rrd_def'])) {
$rrd_def = $tags['rrd_def'];
// filter out data not in the definition
$fields = array_filter($fields, function ($key) use ($rrd_def) {
$valid = $rrd_def->isValidDataset($key);
if (! $valid) {
Log::debug("RRD warning: unused data sent $key");
}
return $valid;
}, ARRAY_FILTER_USE_KEY);
if (! $this->checkRrdExists($rrd)) {
$newdef = "--step $step $rrd_def $this->rra";
$this->command('create', $rrd, $newdef);
}
}
$this->update($rrd, $fields);
}
public function lastUpdate(string $filename): ?TimeSeriesPoint
{
$output = $this->command('lastupdate', $filename, '')[0];
if (preg_match('/((?: \w+)+)\n\n(\d+):((?: [\d.-]+)+)\nOK/', $output, $matches)) {
$data = array_combine(
explode(' ', ltrim($matches[1])),
explode(' ', ltrim($matches[3])),
);
return new TimeSeriesPoint((int) $matches[2], $data);
}
return null;
}
/**
* Updates an rrd database at $filename using $options
* Where $options is an array, each entry which is not a number is replaced with "U"
*
* @internal
*
* @param string $filename
* @param array $data
* @return array|string
*/
public function update($filename, $data)
{
$values = [];
// Do some sanitation on the data if passed as an array.
if (is_array($data)) {
$values[] = 'N';
foreach ($data as $v) {
if (! is_numeric($v)) {
$v = 'U';
}
$values[] = $v;
}
$data = implode(':', $values);
return $this->command('update', $filename, $data);
} else {
return 'Bad options passed to rrdtool_update';
}
}
// rrdtool_update
/**
* Modify an rrd file's max value and trim the peaks as defined by rrdtool
*
* @param string $type only 'port' is supported at this time
* @param string $filename the path to the rrd file
* @param int $max the new max value
* @return bool
*/
public function tune($type, $filename, $max)
{
$fields = [];
if ($type === 'port') {
if ($max < 10000000) {
return false;
}
$max = $max / 8;
$fields = [
'INOCTETS',
'OUTOCTETS',
'INERRORS',
'OUTERRORS',
'INUCASTPKTS',
'OUTUCASTPKTS',
'INNUCASTPKTS',
'OUTNUCASTPKTS',
'INDISCARDS',
'OUTDISCARDS',
'INUNKNOWNPROTOS',
'INBROADCASTPKTS',
'OUTBROADCASTPKTS',
'INMULTICASTPKTS',
'OUTMULTICASTPKTS',
];
}
if (count($fields) > 0) {
$options = '--maximum ' . implode(":$max --maximum ", $fields) . ":$max";
$this->command('tune', $filename, $options);
}
return true;
}
// rrdtool_tune
/**
* Generates a filename for a proxmox cluster rrd
*
* @param string $pmxcluster
* @param string $vmid
* @param string $vmport
* @return string full path to the rrd.
*/
public function proxmoxName($pmxcluster, $vmid, $vmport)
{
$pmxcdir = implode('/', [$this->rrd_dir, 'proxmox', self::safeName($pmxcluster)]);
// this is not needed for remote rrdcached
if (! is_dir($pmxcdir)) {
mkdir($pmxcdir, 0775, true);
}
return implode('/', [$pmxcdir, self::safeName($vmid . '_netif_' . $vmport . '.rrd')]);
}
/**
* Get the name of the port rrd file. For alternate rrd, specify the suffix.
*
* @param int $port_id
* @param string $suffix
* @return string
*/
public function portName($port_id, $suffix = null)
{
return "port-id$port_id" . (empty($suffix) ? '' : '-' . $suffix);
}
/**
* rename an rrdfile, can only be done on the LibreNMS server hosting the rrd files
*
* @param array $device Device object
* @param string|array $oldname RRD name array as used with rrd_name()
* @param string|array $newname RRD name array as used with rrd_name()
* @return bool indicating rename success or failure
*/
public function renameFile($device, $oldname, $newname)
{
$oldrrd = self::name($device['hostname'], $oldname);
$newrrd = self::name($device['hostname'], $newname);
if (is_file($oldrrd) && ! is_file($newrrd)) {
if (rename($oldrrd, $newrrd)) {
log_event("Renamed $oldrrd to $newrrd", $device, 'poller', 1);
return true;
} else {
log_event("Failed to rename $oldrrd to $newrrd", $device, 'poller', 5);
return false;
}
} else {
// we don't need to rename the file
return true;
}
}
/**
* Generates a filename based on the hostname (or IP) and some extra items
*
* @param string $host Host name
* @param array|string $extra Components of RRD filename - will be separated with "-", or a pre-formed rrdname
* @param string $extension File extension (default is .rrd)
* @return string the name of the rrd file for $host's $extra component
*/
public function name($host, $extra, $extension = '.rrd')
{
$filename = self::safeName(is_array($extra) ? implode('-', $extra) : $extra);
return implode('/', [$this->dirFromHost($host), $filename . $extension]);
}
/**
* Generates a path based on the hostname (or IP)
*
* @param string $host Host name
* @return string the name of the rrd directory for $host
*/
public function dirFromHost($host)
{
$host = str_replace(':', '_', trim($host, '[]'));
return implode('/', [$this->rrd_dir, $host]);
}
/**
* Generates and pipes a command to rrdtool
*
* @internal
*
* @param string $command create, update, updatev, graph, graphv, dump, restore, fetch, tune, first, last, lastupdate, info, resize, xport, flushcached
* @param string $filename The full patth to the rrd file
* @param string $options rrdtool command options
* @return array the output of stdout and stderr in an array
*
* @throws \Exception thrown when the rrdtool process(s) cannot be started
*/
private function command($command, $filename, $options)
{
$stat = Measurement::start($this->coalesceStatisticType($command));
$output = null;
try {
$cmd = self::buildCommand($command, $filename, $options);
} catch (FileExistsException $e) {
Log::debug("RRD[%g$filename already exists%n]", ['color' => true]);
return [null, null];
}
Log::debug("RRD[%g$cmd%n]", ['color' => true]);
// do not write rrd files, but allow read-only commands
$ro_commands = ['graph', 'graphv', 'dump', 'fetch', 'first', 'last', 'lastupdate', 'info', 'xport'];
if ($this->disabled && ! in_array($command, $ro_commands)) {
if (! Config::get('hide_rrd_disabled')) {
Log::debug('[%rRRD Disabled%n]', ['color' => true]);
}
return [null, null];
}
// send the command!
if (in_array($command, ['last', 'list', 'lastupdate']) && $this->init(false)) {
// send this to our synchronous process so output is guaranteed
$output = $this->sync_process->sendCommand($cmd);
} elseif ($this->init()) {
// don't care about the return of other commands, so send them to the faster async process
$output = $this->async_process->sendCommand($cmd);
} else {
Log::error('rrdtool could not start');
}
if (Debug::isVerbose()) {
echo 'RRDtool Output: ';
echo $output[0];
echo $output[1];
}
$this->recordStatistic($stat->end());
return $output;
}
/**
* Build a command for rrdtool
* Shortens the filename as needed
* Determines if --daemon and -O should be used
*
* @internal
*
* @param string $command The base rrdtool command. Usually create, update, last.
* @param string $filename The full path to the rrd file
* @param string $options Options for the command possibly including the rrd definition
* @return string returns a full command ready to be piped to rrdtool
*
* @throws FileExistsException if rrdtool <1.4.3 and the rrd file exists locally
*/
public function buildCommand($command, $filename, $options)
{
if ($command == 'create') {
// <1.4.3 doesn't support -O, so make sure the file doesn't exist
if (version_compare($this->version, '1.4.3', '<')) {
if (is_file($filename)) {
throw new FileExistsException();
}
} else {
$options .= ' -O';
}
}
// no remote for create < 1.5.5 and tune < 1.5
if ($this->rrdcached &&
! ($command == 'create' && version_compare($this->version, '1.5.5', '<')) &&
! ($command == 'tune' && $this->rrdcached && version_compare($this->version, '1.5', '<'))
) {
// only relative paths if using rrdcached
$filename = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $filename);
$options = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $options);
return "$command $filename $options --daemon " . $this->rrdcached;
}
return "$command $filename $options";
}
/**
* Get array of all rrd files for a device,
* via rrdached or localdisk.
*
* @param array $device device for which we get the rrd's
* @return array array of rrd files for this host
*/
public function getRrdFiles($device)
{
if ($this->rrdcached) {
$filename = sprintf('/%s', self::safeName($device['hostname']));
$rrd_files = $this->command('list', $filename, '');
// Command output is an array, create new array with each filename as a item in array.
$rrd_files_array = explode("\n", trim($rrd_files[0]));
// Remove status line from response
array_pop($rrd_files_array);
} else {
$rrddir = $this->dirFromHost($device['hostname']);
$pattern = sprintf('%s/*.rrd', $rrddir);
$rrd_files_array = glob($pattern);
}
sort($rrd_files_array);
return $rrd_files_array;
}
/**
* Get array of rrd files for specific application.
*
* @param array $device device for which we get the rrd's
* @param int $app_id application id on the device
* @param string $app_name name of app to be searched
* @param string $category which category of graphs are searched
* @return array array of rrd files for this host
*/
public function getRrdApplicationArrays($device, $app_id, $app_name, $category = null)
{
$entries = [];
$separator = '-';
$rrdfile_array = $this->getRrdFiles($device);
if ($category) {
$pattern = sprintf('%s-%s-%s-%s', 'app', $app_name, $app_id, $category);
} else {
$pattern = sprintf('%s-%s-%s', 'app', $app_name, $app_id);
}
// app_name contains a separator character? consider it
$offset = substr_count($app_name, $separator);
foreach ($rrdfile_array as $rrd) {
if (str_contains($rrd, $pattern)) {
$filename = basename($rrd, '.rrd');
$entry = explode($separator, $filename, 4 + $offset)[3 + $offset];
if ($entry) {
array_push($entries, $entry);
}
}
}
return $entries;
}
/**
* Checks if the rrd file exists on the server
* This will perform a remote check if using rrdcached and rrdtool >= 1.5
*
* @param string $filename full path to the rrd file
* @return bool whether or not the passed rrd file exists
*/
public function checkRrdExists($filename)
{
if ($this->rrdcached && version_compare($this->version, '1.5', '>=')) {
$check_output = implode($this->command('last', $filename, ''));
$filename = str_replace([$this->rrd_dir . '/', $this->rrd_dir], '', $filename);
return ! (str_contains($check_output, $filename) && str_contains($check_output, 'No such file or directory'));
} else {
return is_file($filename);
}
}
/**
* Remove RRD file(s). Use with care as this permanently deletes rrd data.
*
* @param string $hostname rrd subfolder (hostname)
* @param string $prefix start of rrd file name all files matching will be deleted
*/
public function purge($hostname, $prefix)
{
if (empty($hostname)) {
Log::error("Could not purge rrd $prefix, empty hostname");
return;
}
foreach (glob($this->name($hostname, $prefix, '*.rrd')) as $rrd) {
unlink($rrd);
}
}
/**
* Generates a graph file at $graph_file using $options
* Graphs are a single command per run, so this just runs rrdtool
*
* @param string $options
* @return string
*
* @throws RrdGraphException
*/
public function graph(string $options, array $env = null): string
{
$process = new Process([Config::get('rrdtool', 'rrdtool'), '-'], $this->rrd_dir, $env);
$process->setTimeout(300);
$process->setIdleTimeout(300);
try {
$command = $this->buildCommand('graph', '-', $options);
$process->setInput($command . "\nquit");
$process->run();
} catch (FileExistsException $e) {
throw new RrdGraphException($e->getMessage(), 'File Exists');
}
$feedback_position = strrpos($process->getOutput(), 'OK ');
if ($feedback_position !== false) {
return substr($process->getOutput(), 0, $feedback_position);
}
// if valid image is returned with error, extract image and feedback
// rrdtool defaults to png if imgformat not specified
$graph_type = preg_match('/--imgformat=([^\s]+)/', $options, $matches) ? strtolower($matches[1]) : 'png';
$imageFormat = ImageFormat::forGraph($graph_type);
$search = $imageFormat->getImageEnd();
if (($position = strrpos($process->getOutput(), $search)) !== false) {
$position += strlen($search);
throw new RrdGraphException(
substr($process->getOutput(), $position),
null,
null,
null,
$process->getExitCode(),
substr($process->getOutput(), 0, $position)
);
}
// only error text was returned
$error = trim($process->getOutput() . PHP_EOL . $process->getErrorOutput());
throw new RrdGraphException($error, null, null, null, $process->getExitCode());
}
public function __destruct()
{
$this->close();
}
/**
* Remove invalid characters from the rrd file name
*
* @param string $name
* @return string
*/
public static function safeName($name)
{
return (string) preg_replace('/[^a-zA-Z0-9,._\-]/', '_', $name);
}
/**
* Remove invalid characters from the rrd description
*
* @param string $descr
* @return string
*/
public static function safeDescr($descr)
{
return (string) preg_replace('/[^a-zA-Z0-9,._\-\/\ ]/', ' ', $descr);
}
/**
* Escapes strings and sets them to a fixed length for use with RRDtool
*
* @param string $descr the string to escape
* @param int $length if passed, string will be padded and trimmed to exactly this length (after rrdtool unescapes it)
* @return string
*/
public static function fixedSafeDescr($descr, $length)
{
$result = Rewrite::shortenIfType($descr);
$result = str_replace("'", '', $result); // remove quotes
if (is_numeric($length)) {
// preserve original $length for str_pad()
// determine correct strlen() for substr_count()
$substr_count_length = $length <= 0 ? null : min(strlen($descr), $length);
$extra = substr_count($descr, ':', 0, $substr_count_length);
$result = substr(str_pad($result, $length), 0, $length + $extra);
if ($extra > 0) {
$result = substr($result, 0, -1 * $extra);
}
}
$result = str_replace(':', '\:', $result); // escape colons
return $result . ' ';
}
/**
* Run rrdtool and parse the version from the output.
*
* @return string
*/
public static function version(): ?string
{
$proc = new Process([Config::get('rrdtool', 'rrdtool'), '--version']);
$proc->run();
$parts = explode(' ', $proc->getOutput(), 3);
return $proc->isSuccessful() && isset($parts[1]) ? str_replace('1.7.01.7.0', '1.7.0', $parts[1]) : null;
}
/**
* Only track update and create primarily, just put all others in an "other" bin
*
* @param string $type
* @return string
*/
private function coalesceStatisticType($type)
{
return ($type == 'update' || $type == 'create') ? $type : 'other';
}
}