Merge c3f7352 into 19175ad

VERSION

@@ -1 +1 @@
-0.6.0
+0.7.0

docs/buffer/index.rst

@@ -0,0 +1,75 @@
+.. _buffer:
+
+================
+Buffer Interface
+================
+
+.. versionadded:: 0.7.0
+
+By design, syslog messages can be fired spuriously, and the same information 
+may be sent multiple times within a short period of time. For example, when
+an NTP server is unreachable, Junos would send the following message several
+times from the same device, within just a couple of seconds:
+
+.. code-block:: text
+
+  Aug 27 12:03:46  router1 xntpd[4308]: NTP Server 10.10.10.1 is Unreachable
+  Aug 27 12:03:47  router1 xntpd[4308]: NTP Server 10.10.10.1 is Unreachable
+  Aug 27 12:03:49  router1 xntpd[4308]: NTP Server 10.10.10.1 is Unreachable
+
+To ensure ``napalm-logs`` is sending notifications only for one copy of these
+messages, you can enable the buffer interface to cache the first occurrence of
+the message, and the next ones would not be published.
+
+.. warning::
+
+    Before enabling this feature please consider the caveats and the 
+    limitations added by the interface of choice.
+    One of the most important is the fact that lookups are time consuming and 
+    might decrease the performances, and/or require a higher memory 
+    consumption.
+
+The Buffer interface is pluggable subsystem and can be enabled from the 
+configuration file, under the ``buffer`` block, for example:
+
+.. code-block:: yaml
+
+  buffer:
+    redis:
+      host: 127.0.0.1
+      db: 4
+
+.. _buffer-drivers:
+
+Available Buffer drivers and their options
+------------------------------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   memory
+   redis
+
+Globally available options
+--------------------------
+
+In addition to the particular options for each individual Buffer driver, you
+can fine tune ``napalm-logs`` using the following options:
+
+.. _buffer-opts-expire-time:
+
+``expire_time``: ``5``
+-----------------------
+
+The time in seconds to cache the message and filter out duplicates. The first 
+occurrence of the message is recorded and hashed into the Buffer interface of 
+choice. The next messages with the same content and looked up and discarded if 
+they have been fired within ``expire_time`` seconds.
+
+Configuration example:
+
+.. code-block:: yaml
+
+  buffer:
+    mem:
+      expire_time: 30

docs/buffer/memory.rst

@@ -0,0 +1,16 @@
+.. _buffer-memory:
+
+======
+Memory
+======
+
+This :ref:`buffer` interface caches the messages into memory.
+
+.. warning::
+
+    When running ``napalm-logs`` in environment with very hig volume of syslog
+    messages, using this interface may lead to high memory consumption. To 
+    control this, please check the :ref:`buffer-opts-expire-time` option to 
+    fine tune this behaviour.
+
+This interface doesn't have any specific options.

docs/buffer/redis.rst

@@ -0,0 +1,53 @@
+.. _buffer-redis:
+
+=====
+Redis
+=====
+
+This :ref:`buffer` interface caches the messages into
+`Redis <https://redis.io/>`_.
+
+.. note::
+
+    This driver is available only when the underlying library ``redis``, 
+    available as system package, or
+    `Python library <https://pypi.org/project/redis/>`_.
+
+.. _buffer-redis-opts:
+
+Options
+-------
+
+This interface supports all the options of the
+`StrictRedis class <https://redis-py.readthedocs.io/en/latest/#redis.StrictRedis>`_
+
+Configuration example:
+
+.. code-block:: yaml
+
+    buffer:
+      redis:
+        host: redis-srv.example.com
+        port: 16379
+        socket_keepalive: true
+
+In addition to the ``StrictRedis`` specific options, this interface also allows 
+to configure:
+
+.. _buffer-redis-opts-key-prefix:
+
+``key_prefix``
+~~~~~~~~~~~~~~
+
+The prefix to be added to the Redis keys. This option defaults to empty string 
+(it doesn't add any prefix).
+
+.. _buffer-redis-opts-keys-set-name:
+
+``keys_set_name``: ``__napalm_logs_keys_set``
+---------------------------------------------
+
+The name of the `Redis Set <https://redis.io/commands#set>`_ having the list of 
+keys managed by ``napalm-logs`` to keep the cache. This is due to the fact that
+``keys *`` operation is not optimal in Redis, therefore the bank of the keys is 
+saved into a Redis Set for quick lookups.

docs/device_config/netiron.rst

@@ -5,3 +5,9 @@ Brocade NetIron
 ===============
 
 .. versionadded:: 0.6.0
+
+The following will configure NetIron to send syslog messages, over UDP to a syslog server on IP address ``10.0.0.9``
+
+.. code-block:: text
+
+    logging host 10.0.0.9

docs/index.rst

@@ -316,6 +316,7 @@ daemon):
    publisher/index
    serializer/index
    logger/index
+   buffer/index
    syslog/index
    developers/index
    releases/index

docs/publisher/index.rst

@@ -325,3 +325,20 @@ You can specify a separate serialize per publisher, e.g.:
         serializer: json
     - cli:
         serializer: pprint
+
+.. _publisher-opts-strip-message-details:
+
+``strip_message_details``: ``False``
+------------------------------------
+
+.. versionadded:: 0.7.0
+
+Strip the ``message_details`` key before publishing the object.
+
+Configuration example:
+
+.. code-block:: yaml
+
+  publisher:
+    - kafka:
+        strip_message_details: true

docs/releases/0.7.0.rst

@@ -0,0 +1,51 @@
+.. _release-0.7.0:
+
+=============================
+Release 0.7.0 - Codename Froe
+=============================
+
+A new pluggable interface: buffer
+---------------------------------
+
+By design, syslog messages can be fired spuriously, and the same information 
+may be sent multiple times within a short period of time.
+
+The new :ref:`buffer` can cache the messages for a number of seconds and 
+prevent firing the same messages redundantly.
+
+The Buffer interface is pluggable subsystem and can be enabled from the 
+configuration file, under the buffer block, for example:
+
+.. code-block:: yaml
+
+  buffer:
+    redis:
+      host: 127.0.0.1
+      db: 4
+
+For further details and configuration options, please check :ref:`buffer`.
+
+New features
+------------
+
+Added :ref:`publisher-opts-strip-message-details` Publisher configuration
+option to remove the ``message_details`` key from the object to be published.
+``message_details`` contains the raw chunks of data resulted from the
+napalm-logs parsing, before mapping to the associated YANG model.
+
+
+New Structured Messages
+-----------------------
+
+- :ref:`AGENT_INITIALIZED` when a system process is (re-)initialized.
+- :ref:`USER_WRITE_CONFIG` added by George Pickering (@bigpick) which is a
+  notification sent when the user saves the startup config (though not
+  necessarily updating the running config).
+
+The following messages were already defined, now extending the list of 
+supported platforms:
+
+- :ref:`OSPF_NEIGHBOR_UP` and :ref: `OSPF_NEIGHBOR_DOWN` added for 
+  :ref:`device-configuration-netiron` thanks to Johan van den Dorpe (@johanek).
+- :ref:`USER_ENTER_CONFIG_MODE` and :ref:`USER_EXIT_CONFIG_MODE` added for
+  :ref:`device-configuration-eos` thanks to George Pickering (@bigpick).

docs/releases/index.rst

@@ -8,11 +8,12 @@ Release Notes
 Latest Release
 ^^^^^^^^^^^^^^
 
-- :ref:`release-0.6.0`
+- :ref:`release-0.7.0`
 
 Previous Releases
 ^^^^^^^^^^^^^^^^^
 
+- :ref:`release-0.6.0`
 - :ref:`release-0.5.0`
 - :ref:`release-0.4.2`
 - :ref:`release-0.4.1`

napalm_logs/base.py

@@ -26,6 +26,7 @@
 # Import napalm-logs pkgs
 import napalm_logs.utils
 import napalm_logs.config as CONFIG
+import napalm_logs.buffer
 # processes
 from napalm_logs.auth import NapalmLogsAuthProc
 from napalm_logs.device import NapalmLogsDeviceProc
@@ -66,7 +67,8 @@ def __init__(self,
                  device_whitelist=[],
                  hwm=None,
                  device_worker_processes=1,
-                 serializer='msgpack'):
+                 serializer='msgpack',
+                 buffer=None):
         '''
         Init the napalm-logs engine.
 
@@ -103,6 +105,8 @@ def __init__(self,
         self.serializer = serializer
         self.device_worker_processes = device_worker_processes
         self.hwm = hwm
+        self._buffer_cfg = buffer
+        self._buffer = None
         self.opts = {}
         # Setup the environment
         self._setup_log()
@@ -111,6 +115,7 @@ def __init__(self,
         self._post_preparation()
         # Start the Prometheus metrics server
         self._setup_metrics()
+        self._setup_buffer()
         # Private vars
         self.__priv_key = None
         self.__signing_key = None
@@ -126,6 +131,19 @@ def __exit__(self, exc_type, exc_value, exc_traceback):
             log.error('Exiting due to unhandled exception', exc_info=True)
             self.__raise_clean_exception(exc_type, exc_value, exc_traceback)
 
+    def _setup_buffer(self):
+        '''
+        Setup the buffer subsystem.
+        '''
+        if not self._buffer_cfg or not isinstance(self._buffer_cfg, dict):
+            return
+        buffer_name = list(self._buffer_cfg.keys())[0]
+        buffer_class = napalm_logs.buffer.get_interface(buffer_name)
+        log.debug('Setting up buffer interface "%s"', buffer_name)
+        if 'expire_time' not in self._buffer_cfg[buffer_name]:
+            self._buffer_cfg[buffer_name]['expire_time'] = CONFIG.BUFFER_EXPIRE_TIME
+        self._buffer = buffer_class(**self._buffer_cfg[buffer_name])
+
     def _setup_metrics(self):
         """
         Start metric exposition
@@ -534,7 +552,8 @@ def _start_srv_proc(self,
         log.debug('Starting the server process')
         server = NapalmLogsServerProc(self.opts,
                                       self.config_dict,
-                                      started_os_proc)
+                                      started_os_proc,
+                                      buffer=self._buffer)
         proc = Process(target=server.start)
         proc.start()
         proc.description = 'Server process'

napalm_logs/buffer/__init__.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+'''
+napalm-logs pluggable buffer interface.
+'''
+from __future__ import absolute_import, unicode_literals
+
+# Import python std lib
+import logging
+
+# Import napalm-logs pkgs
+# Exceptions
+from napalm_logs.exceptions import InvalidBufferException
+
+# Import buffer classes
+from napalm_logs.buffer.memory import MemoryBuffer
+from napalm_logs.buffer.redisbuf import RedisBuffer
+
+log = logging.getLogger(__file__)
+
+BUFFER_LOOKUP = {
+    'mem': MemoryBuffer,
+    'memory': MemoryBuffer,
+    'cache': MemoryBuffer,
+    'redis': RedisBuffer
+}
+
+
+def get_interface(name):
+    '''
+    Return the serialize function.
+    '''
+    try:
+        log.debug('Using %s as buffer interface', name)
+        return BUFFER_LOOKUP[name]
+    except KeyError:
+        msg = 'Buffer interface {} is not available'.format(name)
+        log.error(msg, exc_info=True)
+        raise InvalidBufferException(msg)
+
+
+__all__ = (
+    'get_interface',
+)