salt/engines/junos_syslog.py
# -*- coding: utf-8 -*-
'''
Junos Syslog Engine
==========================
.. versionadded:: 2017.7.0
:depends: pyparsing, twisted
An engine that listens to syslog message from Junos devices,
extract event information and generate message on SaltStack bus.
The event topic sent to salt is dynamically generated according to the topic title
specified by the user. The incoming event data (from the junos device) consists
of the following fields:
1. hostname
2. hostip
3. daemon
4. event
5. severity
6. priority
7. timestamp
8. message
9. pid
10. raw (the raw event data forwarded from the device)
The topic title can consist of any of the combination of above fields, but the
topic has to start with ``jnpr/syslog``. Here are a couple example
combinations:
- jnpr/syslog/hostip/daemon/event
- jnpr/syslog/daemon/severity
The corresponding dynamic topic sent on salt event bus would look something like:
- jnpr/syslog/1.1.1.1/mgd/UI_COMMIT_COMPLETED
- jnpr/syslog/sshd/7
The default topic title is ``jnpr/syslog/hostname/event``.
One can choose the type of data they want from the event bus. For example, if
one wants only events pertaining to a particular daemon, this can be specified
in the configuration file:
.. code-block:: yaml
daemon: mgd
One can even have a list of daemons:
.. code-block:: yaml
daemon:
- mgd
- sshd
Example configuration (to be written in master config file)
.. code-block:: yaml
engines:
- junos_syslog:
port: 9999
topic: jnpr/syslog/hostip/daemon/event
daemon:
- mgd
- sshd
For junos_syslog engine to receive events, syslog must be set on the junos device.
This can be done via following configuration:
.. code-block:: text
set system syslog host <ip-of-the-salt-device> port 516 any any
Below is a sample syslog event which is received from the junos device:
.. code-block:: text
<30>May 29 05:18:12 bng-ui-vm-9 mspd[1492]: No chassis configuration found
The source for parsing the syslog messages is taken from:
https://gist.github.com/leandrosilva/3651640#file-xlog-py
'''
from __future__ import absolute_import, print_function, unicode_literals
import re
import logging
import time
try:
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor, threads
from pyparsing import Word, alphas, Suppress, Combine, nums, string, \
Optional, Regex, LineEnd, StringEnd, delimitedList
HAS_TWISTED_AND_PYPARSING = True
except ImportError:
HAS_TWISTED_AND_PYPARSING = False
# Fallback class
class DatagramProtocol(object):
pass
import salt.utils.event as event
# Import 3rd-party libs
from salt.ext import six
from salt.ext.six.moves import range # pylint: disable=redefined-builtin
# logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
__virtualname__ = 'junos_syslog'
def __virtual__():
'''
Load only if twisted and pyparsing libs are present.
'''
if not HAS_TWISTED_AND_PYPARSING:
return (False, 'junos_syslog could not be loaded.'
' Make sure you have twisted and pyparsing python libraries.')
return True
class _Parser(object):
def __init__(self):
ints = Word(nums)
EOL = LineEnd().suppress()
# ip address of device
ipAddress = Optional(
delimitedList(
ints,
".",
combine=True) + Suppress(
":"))
# priority
priority = Suppress("<") + ints + Suppress(">")
# timestamp
month = Word(string.uppercase, string.lowercase, exact=3)
day = ints
hour = Combine(ints + ":" + ints + ":" + ints)
timestamp = month + day + hour
# hostname
hostname = Word(alphas + nums + "_" + "-" + ".")
# daemon
daemon = Word(alphas + nums + "/" + "-" + "_" + ".") + Optional(
Suppress("[") + ints + Suppress("]")) + Suppress(":")
# message
message = Regex(".*")
# pattern build
self.__pattern = ipAddress + priority + timestamp + \
hostname + daemon + message + StringEnd() | EOL
self.__pattern_without_daemon = ipAddress + priority + \
timestamp + hostname + message + StringEnd() | EOL
def parse(self, line):
try:
parsed = self.__pattern.parseString(line)
except Exception:
try:
parsed = self.__pattern_without_daemon.parseString(line)
except Exception:
return
if len(parsed) == 6:
payload = {}
payload["priority"] = int(parsed[0])
payload["severity"] = payload["priority"] & 0x07
payload["facility"] = payload["priority"] >> 3
payload["timestamp"] = time.strftime("%Y-%m-%d %H:%M:%S")
payload["hostname"] = parsed[4]
payload["daemon"] = 'unknown'
payload["message"] = parsed[5]
payload["event"] = 'SYSTEM'
payload['raw'] = line
return payload
elif len(parsed) == 7:
payload = {}
payload["priority"] = int(parsed[0])
payload["severity"] = payload["priority"] & 0x07
payload["facility"] = payload["priority"] >> 3
payload["timestamp"] = time.strftime("%Y-%m-%d %H:%M:%S")
payload["hostname"] = parsed[4]
payload["daemon"] = parsed[5]
payload["message"] = parsed[6]
payload["event"] = 'SYSTEM'
obj = re.match(r'(\w+): (.*)', payload["message"])
if obj:
payload["message"] = obj.group(2)
payload["raw"] = line
return payload
elif len(parsed) == 8:
payload = {}
payload["priority"] = int(parsed[0])
payload["severity"] = payload["priority"] & 0x07
payload["facility"] = payload["priority"] >> 3
payload["timestamp"] = time.strftime("%Y-%m-%d %H:%M:%S")
payload["hostname"] = parsed[4]
payload["daemon"] = parsed[5]
payload["pid"] = parsed[6]
payload["message"] = parsed[7]
payload["event"] = 'SYSTEM'
obj = re.match(r'(\w+): (.*)', payload["message"])
if obj:
payload["event"] = obj.group(1)
payload["message"] = obj.group(2)
payload["raw"] = line
return payload
elif len(parsed) == 9:
payload = {}
payload["hostip"] = parsed[0]
payload["priority"] = int(parsed[1])
payload["severity"] = payload["priority"] & 0x07
payload["facility"] = payload["priority"] >> 3
payload["timestamp"] = time.strftime("%Y-%m-%d %H:%M:%S")
payload["hostname"] = parsed[5]
payload["daemon"] = parsed[6]
payload["pid"] = parsed[7]
payload["message"] = parsed[8]
payload["event"] = 'SYSTEM'
obj = re.match(r'(\w+): (.*)', payload["message"])
if obj:
payload["event"] = obj.group(1)
payload["message"] = obj.group(2)
payload["raw"] = line
return payload
class _SyslogServerFactory(DatagramProtocol):
def __init__(self, options):
self.options = options
self.obj = _Parser()
data = [
"hostip",
"priority",
"severity",
"facility",
"timestamp",
"hostname",
"daemon",
"pid",
"message",
"event"]
if 'topic' in self.options:
# self.title = 'jnpr/syslog'
# To remove the stray '/', if not removed splitting the topic
# won't work properly. Eg: '/jnpr/syslog/event' won't be split
# properly if the starting '/' is not stripped
self.options['topic'] = options['topic'].strip('/')
topics = options['topic'].split("/")
self.title = topics
if len(topics) < 2 or topics[0] != 'jnpr' or topics[1] != 'syslog':
log.debug(
'The topic specified in configuration should start with \
"jnpr/syslog". Using the default topic.')
self.title = ['jnpr', 'syslog', 'hostname', 'event']
else:
for i in range(2, len(topics)):
if topics[i] not in data:
log.debug(
'Please check the topic specified. \
Only the following keywords can be specified \
in the topic: hostip, priority, severity, \
facility, timestamp, hostname, daemon, pid, \
message, event. Using the default topic.')
self.title = ['jnpr', 'syslog', 'hostname', 'event']
break
# We are done processing the topic. All other arguments are the
# filters given by the user. While processing the filters we don't
# explicitly ignore the 'topic', but delete it here itself.
del self.options['topic']
else:
self.title = ['jnpr', 'syslog', 'hostname', 'event']
def parseData(self, data, host, port, options):
'''
This function will parse the raw syslog data, dynamically create the
topic according to the topic specified by the user (if specified) and
decide whether to send the syslog data as an event on the master bus,
based on the constraints given by the user.
:param data: The raw syslog event data which is to be parsed.
:param host: The IP of the host from where syslog is forwarded.
:param port: Port of the junos device from which the data is sent
:param options: kwargs provided by the user in the configuration file.
:return: The result dictionary which contains the data and the topic,
if the event is to be sent on the bus.
'''
data = self.obj.parse(data)
data['hostip'] = host
log.debug(
'Junos Syslog - received %s from %s, sent from port %s',
data, host, port
)
send_this_event = True
for key in options:
if key in data:
if isinstance(options[key], (six.string_types, int)):
if six.text_type(options[key]) != six.text_type(data[key]):
send_this_event = False
break
elif isinstance(options[key], list):
for opt in options[key]:
if six.text_type(opt) == six.text_type(data[key]):
break
else:
send_this_event = False
break
else:
raise Exception(
'Arguments in config not specified properly')
else:
raise Exception(
'Please check the arguments given to junos engine in the\
configuration file')
if send_this_event:
if 'event' in data:
topic = 'jnpr/syslog'
for i in range(2, len(self.title)):
topic += '/' + six.text_type(data[self.title[i]])
log.debug(
'Junos Syslog - sending this event on the bus: %s from %s',
data, host
)
result = {'send': True, 'data': data, 'topic': topic}
return result
else:
raise Exception(
'The incoming event data could not be parsed properly.')
else:
result = {'send': False}
return result
def send_event_to_salt(self, result):
'''
This function identifies whether the engine is running on the master
or the minion and sends the data to the master event bus accordingly.
:param result: It's a dictionary which has the final data and topic.
'''
if result['send']:
data = result['data']
topic = result['topic']
# If the engine is run on master, get the event bus and send the
# parsed event.
if __opts__['__role'] == 'master':
event.get_master_event(__opts__,
__opts__['sock_dir']
).fire_event(data, topic)
# If the engine is run on minion, use the fire_master execution
# module to send event on the master bus.
else:
__salt__['event.fire_master'](data=data, tag=topic)
def handle_error(self, err_msg):
'''
Log the error messages.
'''
log.error(err_msg.getErrorMessage)
def datagramReceived(self, data, connection_details):
(host, port) = connection_details
d = threads.deferToThread(
self.parseData,
data,
host,
port,
self.options)
d.addCallbacks(self.send_event_to_salt, self.handle_error)
def start(port=516, **kwargs):
log.info('Starting junos syslog engine (port %s)', port)
reactor.listenUDP(port, _SyslogServerFactory(kwargs))
reactor.run()