Merge 9b8d174 into d13e316

Dockerfile

@@ -7,9 +7,9 @@ COPY ./ /var/cache/napalm-logs/
 RUN apk add --no-cache \
     libffi \
     libffi-dev \
-    python-dev \
+    python3-dev \
     build-base \
-    && pip --no-cache-dir install cffi /var/cache/napalm-logs/ \
+    && pip --no-cache-dir install cffi pyzmq==19.0.2 /var/cache/napalm-logs/ \
     && rm -rf /var/cache/napalm-logs/
 
 CMD napalm-logs --config-file /etc/napalm/logs

docs/developers/index.rst

@@ -10,3 +10,4 @@ Here we you will find out how to add new functionality to ``napalm-logs``.
    architecture
    pluggable_modules
    device_profiles
+   local_testing

docs/developers/local_testing.rst

@@ -0,0 +1,32 @@
+.. _local_testing:
+
+=============
+Local Testing
+=============
+
+When adding new functionality to ``napalm-logs``, in particular new parsing templates, it can be helpful to run an end-to-end simulation.
+
+Testing Installation
+++++++++++++++++++++
+
+As referenced in the :ref:`installation <installation>` documentation, it's advisable to use a Python Virtual Environment. This also applies for the testing installation.
+Once you have the environment setup, you can install your local copy (with modifications) of ``napalm-logs`` in it:
+
+.. code-block:: bash
+
+    pip install -e <local_path/url>
+
+Suggested startup options for ``napalm-logs`` once it's installed:
+
+.. code-block:: bash
+
+    napalm-logs -l debug --disable-security --publisher cli  --listener udp --port 5514
+
+This will run ``napalm-logs`` in a ``debug`` log-level, without certificate-based authentication, using the ``cli`` publisher, while listening for ``syslog`` messages on UDP port 5514.
+You can then proceed to send a sample ``syslog`` message to ``napalm-logs`` using ``netcat``:
+
+.. code-block:: bash
+
+    nc -w0 -u 127.0.0.1 5514 <<< "<32>Oct 24 20:21:27  vmx01 jlaunchd: System reaching processes ceiling high watermark: Contact to system administrator to clean up unnecessary processes or increase maxproc ceiling. Further process fork request may be denied."
+
+The debug output will go to ``/var/log/napalm/logs/`` by default, and it will output the yang message on the CLI.

docs/listener/tcp.rst

@@ -72,3 +72,45 @@ Example:
   listener:
     tcp:
       max_clients: 100
+
+.. _listener-tcp-framing:
+
+``framing``: ``traditional``
+----------------------------
+
+.. versionadded:: 0.10.0
+
+Framing mode used when receiving messages. Available options: ``traditional`` 
+or ``octet-counted``.
+
+In protocol engineering, *framing* means how multiple messages over the same
+connection are separated. Usually, this is transparent to users. Unfortunately,
+the early syslog protocol evolved and so there are cases where users need to
+specify the framing. The ``traditional`` framing is nontransparent. With it,
+messages end when an LF (i.e., line break / return) is encountered, and the next
+message starts immediately after the LF. If multi-line messages are received,
+these are essentially broken up into multiple message, usually with all but the
+first message segment being incorrectly formatted. The ``octet-counted`` framing
+solves this issue. With it, each message is prefixed with the actual message
+length, so that a receiver knows exactly where the message ends. Multi-line
+messages cause no problem here. This mode is very close to the method described
+in RFC5425 for TLS-enabled syslog. Unfortunately, only few syslogd
+implementations support ``octet-counted`` framing. As such, the ``traditional``
+framing is set as default, even though it has defects. If it is known that the
+transmitter supports ``octet-counted`` framing, it is suggested to use that
+framing mode.
+
+(Detailed documentation notes based on `<rsyslog 
+omfwd> https://www.rsyslog.com/doc/v8-stable/configuration/modules/omfwd.html#tcp-framing`__)
+
+.. _listener-tcp-frame-delimiter:
+
+``frame_delimiter``: ``\n``
+---------------------------
+
+.. versionadded:: 0.10.0
+
+Sets a custom frame delimiter for TCP transmission when ``framing`` is
+configured in ``traditional`` mode. The delimiter has to be a number between
+0 and 255 (representing the ASCII-code of said character). The default value for
+this parameter is 10, representing a ``\n``.

docs/metrics/index.rst

@@ -29,11 +29,11 @@ Listener Process(es)
 --------------------
 
 napalm_logs_listener_logs_ingested
-  Count of ingested log messages. Labels are used to seperate metrics for each Listener process.
+  Count of ingested log messages. Labels are used to separate metrics for each Listener process.
 
 napalm_logs_listener_messages_published
   Count of published messages. These are messages published to the message queue for processing by the Server Process.
-  Labels are used to seperate metrics for each Listener process.
+  Labels are used to separate metrics for each Listener process.
 
 Server Process
 --------------
@@ -42,53 +42,62 @@ napalm_logs_server_messages_received
   Count of messages received from Listener processes.
 
 napalm_logs_server_messages_with_identified_os
-  Count of messages with positive OS identification. Labels are used to seperate metrics for each Device OS.
+  Count of messages with positive OS identification. Labels are used to separate metrics for each Device OS.
 
 napalm_logs_server_messages_without_identified_os
   Count of messages which fail OS identification.
 
 napalm_logs_server_messages_failed_device_queuing
   Count of messages per device OS that fail to be queued to a proper Device process. Note these are messages that
-  pass OS identification and we know how to route them but fail to be queued. Labels are used to seperate metrics
+  pass OS identification and we know how to route them but fail to be queued. Labels are used to separate metrics
   for each Device OS.
 
 napalm_logs_server_messages_device_queued
-  Count of messages sucessfully queued to Device processes. Labels are used to seperate metrics for each Device OS process.
+  Count of messages successfully queued to Device processes. Labels are used to separate metrics for each Device OS process.
 
 napalm_logs_server_messages_unknown_queued
-  Count of messages which fail OS indentification and thus we don't know how to route them, but the user has instructed
+  Count of messages which fail OS identification and thus we don't know how to route them, but the user has instructed
   the system to queue them "as-is."
 
 Device Process(es)
 ------------------
 
 napalm_logs_device_messages_received
-  Count of messages received from the Server process. Labels are used to seperate metrics for each Device OS process.
+  Count of messages received from the Server process. Labels are used to separate metrics for each Device OS process.
 
 napalm_logs_device_raw_published_messages
   Count of raw type published messages. In this case, the message did not match a configured message type but the
-  user has instructed the system to publish the message in a raw format. Labels are used to seperate metrics for
+  user has instructed the system to publish the message in a raw format. Labels are used to separate metrics for
   each Device OS process.
 
 napalm_logs_device_published_messages
-  Count of published messages. These are messages which are sucessfully converted to an OpenConfig format. Labels
-  are used to seperate metrics for each Device OS process.
+  Count of published messages. These are messages which are successfully converted to an OpenConfig format. Labels
+  are used to separate metrics for each Device OS process.
 
-napalm_logs_device_oc_object_failed 
+napalm_logs_device_oc_object_failed
   Counter of failed OpenConfig object generations. These are messages for which the system attempts to map to a
-  known OpenConfig object model but fails. Labels are used to seperate metrics for each Device OS process.
+  known OpenConfig object model but fails. Labels are used to separate metrics for each Device OS process.
+
+napalm_logs_device_published_messages_attrs
+  Count of published messages. This metrics supersedes 
+  ``napalm_logs_device_published_messages`` as it provides a more granular 
+  selection, using two additional labels (besides ``device_os``): ``error`` and 
+  ``host`` for the *napalm-logs* error / message type and the host, 
+  respectively. As this metric has a potential to generate a large amount of 
+  metrics, you are able to disable it by configuring 
+  ``metrics_include_attrbutes: false`` in the napalm-logs configuration file.
 
 Publisher Process(es)
 ---------------------
 
 napalm_logs_publisher_received_messages
-  Count of messages received by the Publisher from Device Process(es). Labels are used to seperate metrics for
+  Count of messages received by the Publisher from Device Process(es). Labels are used to separate metrics for
   each Publisher process.
 
 napalm_logs_publisher_whitelist_blacklist_check_fail
-  Count of messages which fail the whitelist/blacklist check. Labels are used to seperate metrics for each
+  Count of messages which fail the whitelist/blacklist check. Labels are used to separate metrics for each
   Publisher process.
 
 napalm_logs_publisher_messages_published
   Count of published messages. These are messages which are published for clients to receive (i.e. output of the
-  system). Labels are used to seperate metrics for each Publisher process.
+  system). Labels are used to separate metrics for each Publisher process.

docs/options/index.rst

@@ -171,6 +171,23 @@ Configuration file example:
 
   metrics_dir: /tmp/a_new_dir_for_metrics
 
+.. _configuration-options-metrics-attrs:
+
+``metrics_include_attributes``
+------------------------------
+
+.. versionadded:: 0.10.0
+
+Disable detailed metrics with attributes per published device OS, hostname, and 
+napalm-logs error type. Default: ``True`` (the metrics will include detailed 
+attributes).
+
+Configuration file example:
+
+.. code-block:: yaml
+
+  metrics_include_attributes: false
+
 .. _configuration-options-certificate:
 
 ``certificate``

docs/publisher/alerta.rst

@@ -50,8 +50,29 @@ Configuration examples:
 Available options
 ^^^^^^^^^^^^^^^^^
 
-The options are generally inherited from the :ref:`publisher-http` Publisher, 
-with the following notes:
+.. _publisher-alerta-pairs:
+
+``pairs``
+---------
+
+.. versionadded:: 0.10.0
+
+Hash that defines the remapping of a specific *napalm-logs* notification to a
+pair notification that will close the previous alert. For example, 
+an ``INTERFACE_UP`` alert would close an existing ``INTERFACE_DOWN`` alert, 
+instead of creating an alert for ``INTERFACE_UP``, and so on.
+
+``pair`` defaults to:
+
+.. code-block:: yaml
+
+    pairs:
+      INTERFACE_UP: INTERFACE_DOWN
+      OSPF_NEIGHBOR_UP: OSPF_NEIGHBOR_DOWN
+      ISIS_NEIGHBOR_UP: ISIS_NEIGHBOR_DOWN
+
+The next options are generally inherited from the :ref:`publisher-http`
+Publisher, with the following notes:
 
 .. _publisher-alerta-address:
 

docs/publisher/index.rst

@@ -74,6 +74,7 @@ Available publishers and their options
    kafka
    log
    zmq
+   prometheus
 
 Globally available options
 --------------------------

docs/publisher/prometheus.rst

@@ -0,0 +1,59 @@
+.. _publisher-prometheus:
+
+==========
+Prometheus
+==========
+
+.. versionadded:: 0.10.0
+
+Expose *napalm-logs* notifications as Prometheus metrics.
+
+.. important::
+
+    In order to make use of this Publisher, you'll need to enable the 
+    :ref:`configuration-options-enable-metrics` option (and eventually 
+    customise the other related settings).
+
+The metrics start with ``napalm_logs``, plus the *napalm-logs* notification 
+name. For example, the metric corresponding to :ref:`ISIS_NEIGHBOR_DOWN` is 
+named ``napalm_logs_isis_neighbor_down``.
+
+All the metrics have at least the ``host`` label. Some of them have additional
+labels, whenever it makes sense to include additional information that can be 
+used for selection and/or alerting.
+
+By default, it will expose metrics for all kinds of messages, so you might want
+to narrow down the selection using the :ref:`publisher-opts-error-whitelist` or
+:ref:`publisher-opts-error-blacklist` options.
+
+CLI usage example:
+
+.. code-block:: bash
+
+  $ sudo napalm-logs --publisher prometheus --enable-metrics
+
+Starting through the command above, you'll find the metrics at 
+http://localhost:9443/metrics.
+
+Metrics examples:
+
+.. code-block:: text
+
+  # HELP napalm_logs_interface_up_total Multiprocess metric
+  # TYPE napalm_logs_interface_up_total counter
+  napalm_logs_interface_up_total{host="veos01",interface="Ethernet28"} 1.0
+  # HELP napalm_logs_interface_down_total Multiprocess metric
+  # TYPE napalm_logs_interface_down_total counter
+  napalm_logs_interface_down_total{host="veos01",interface="Ethernet28"} 1.0
+  # HELP napalm_logs_isis_neighbor_down_total Multiprocess metric
+  # TYPE napalm_logs_isis_neighbor_down_total counter
+  napalm_logs_isis_neighbor_down_total{host="HOSTNAME",interface="et7",level="L1",neighbor="1920.0000.2006"} 1.0
+  # HELP napalm_logs_bgp_neighbor_state_changed_total Multiprocess metric
+  # TYPE napalm_logs_bgp_neighbor_state_changed_total counter
+  napalm_logs_bgp_neighbor_state_changed_total{current_state="IDLE",host="HOSTNAME",neighbor="192.0.2.2",peer_as="12345",previous_state="ESTABLISHED"} 1.0
+  # HELP napalm_logs_agent_initialized_total Multiprocess metric
+  # TYPE napalm_logs_agent_initialized_total counter
+  napalm_logs_agent_initialized_total{host="edge01.bru01"} 1.0
+  # HELP napalm_logs_interface_mac_limit_reached Multiprocess metric
+  # TYPE napalm_logs_interface_mac_limit_reached gauge
+  napalm_logs_interface_mac_limit_reached{host="vmx01",interface="ge-1/0/23.0",pid="15711"} 3.0

docs/requirements.txt

@@ -1,9 +1,6 @@
+-r ../requirements.txt
 jinja2
-pyzmq
-pyyaml
-pynacl
 u-msgpack-python
 sphinx
 sphinx-rtd-theme
 sphinxcontrib-napoleon
-prometheus_client

napalm_logs/base.py

@@ -68,7 +68,8 @@ def __init__(self,
                  hwm=None,
                  device_worker_processes=1,
                  serializer='msgpack',
-                 buffer=None):
+                 buffer=None,
+                 opts=None):
         '''
         Init the napalm-logs engine.
 
@@ -107,7 +108,7 @@ def __init__(self,
         self.hwm = hwm
         self._buffer_cfg = buffer
         self._buffer = None
-        self.opts = {}
+        self.opts = opts if opts else {}
         # Setup the environment
         self._setup_log()
         self._build_config()
@@ -308,7 +309,7 @@ def _load_config(self, path):
                     try:
                         log.debug('Loading %s as YAML', file_)
                         with open(filepath, 'r') as fstream:
-                            cfg = yaml.load(fstream)
+                            cfg = yaml.load(fstream, Loader=yaml.FullLoader)
                             # Reposition at the top and read the comments.
                             if file_name not in CONFIG.OS_INIT_FILENAMES:
                                 # If the file name is not a profile init.

napalm_logs/config/eos/init.yml

@@ -12,4 +12,11 @@ prefixes:
       processName: (\w+)
       tag: ([\w-]+)
     line: '{date} {time} {host} {processName}: %{tag}'
-
+  # ISO8601 date-time format
+  - values:
+      date: (\d{4}-\d{2}-\d{2})
+      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
+      host: ([^ ]+)
+      processName: (\w+)
+      tag: ([\w-]+)
+    line: '{date}T{time} {host} {processName}: %{tag}'