Merge 9ef1b4d into 1e44ce5

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.venv
+*.yml
+*.md
+*.png
+*.ini
+docs
+examples
+tests

.gitignore

@@ -84,6 +84,7 @@ celerybeat-schedule
 .env
 
 # virtualenv
+.venv/
 venv/
 ENV/
 
@@ -94,3 +95,5 @@ ENV/
 .ropeproject
 report.json
 
+# PyCharm stuff
+.idea

.travis.yml

@@ -1,18 +1,11 @@
 language: python
-sudo: required
-services:
-- docker
 python:
 - 2.7
 - 3.5
 - 3.6
 install:
   - pip install tox-travis
   - pip install coveralls
-env:
-  global:
-    - secure: "aayKlfqIU/usydWSBz9FQnDWkzH6k4Uoz/aQmEDDB6ziSB9Y7AkszZKJ112aCGN2Zxqde0JTRIPQkAtniH8r1ZPvcoG+4ew31/cz8wolBE9OZs+wSsoWis57EwC/N/xjngvji5ENd3scRULOZciS8KUlqchRzAfcCM+sxPROJ38a3n2oA63IvSVMzbrSGyE2MukxE+bOyXeKtM9upyOC6nGEjcyb7nOdTQz8UcRzxnIIed4W9WpZqhK5gUbmZMlZSpXhu8upgXdrkooDgVLcpH65OaDfRMtExuTEV2CRxWPZLghxNaDgGVui9VbX9nUxOIe8nLwT0FpDr/fiDGn1StBBoy/6XbAbkWsy3PhLRNvXN9dvhKunCmjGs6W2qT6Ems0HmePrHJx2mtI/dGPQAxknDHY6YjMSqtMfbrOJFOerqQ6UI6YmYiBOMfgC36thIa11Pr6VFtGFtn12XTFw3ZmbBIrQ/EB7YtHAvK6w8oX1+Q//UJjTR9Vs8ZPZPcq376WxaNkc15EvHbQ28ZZOjtRyCDz8+0gNcBgx+EY9nt/1lWcLZ/wHLSFjGJtCMZhnfdE7IsJHnG31SpnumweSdQG4/Ngeo1fQYIXTX6eOt7hOJvkRCuzRJTxDQbD9KYXYs3ZmJYuAxVVlV25fmTaniHj6O5llbNwX5vvaYO6/YRY="
-    - secure: "T5Kbh4QRvETRSaRKVjeu6Tap8TLvtePHaxerJM9ZQn0mH5c+LLjbQvWhtMfZ+Te0GiTy4CZZFybPjt5qpNrQrk8bi2BOjwYNwL7gJyMHKC7hPLfx6zYn+6aZAIR82egz+wIh/LS8cHWbHFNKT1Yj34sUaqt+7PTtLdI8o/rk98R/yd0vQEHFiNy//PLvzHp5OJ40LuT+pYrD9swpQAMBKVHr5pFZEInZga66CT4FdHKs1fpzOWpYm+wDUJ1bhgZAESz+ ZaMNG+wGIRwJeete7A3Qglo2i07wGfpqi6m0pEt4H07uw+4q5EEqRVJq+V/uPC18CcTvjrHq7zcawPhP+Q57bIWd2tqMkEv+RqOIDG7xKU2USlex5AeJ8vUDQ3wq7FpcWXb0h9X8afUMRWOck9R3X0QJnXRtQ3M+bRwncmJHvk0Nb5VhTQ2vlewFrUISZEXdeambirsj1x1AMksIbQCnzQoALB8pzjgmfde5sgw5oBWbIv1EKYxmPa+kOayiXxNrKTgfUTaPMfAbN27uF y1CqrpcRUAMLurTitnbavZlFvyCO+wcdP/tB8E0jIQ2juFAp/8RKMI6Mf0LsVOWzsFrjYwhq6Ehc0QvJI8adE0QuBlV1dMjFwg1KIHrvFeFlKPuSB2w1SIrN51wcohMfETbpj7NAYTLVKEzQo/3Wxc="
 deploy:
   - provider: pypi
     user: mirucha
@@ -21,11 +14,6 @@ deploy:
     on:
       tags: true
       branch: master
-  - provider: script
-    script: bash docker_push.sh
-    on:
-      tags: true
-      branch: master
 script:
 - tox
 after_success:

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.6-alpine
+
+COPY docker/config.txt /etc/napalm/logs
+COPY ./ /var/cache/napalm-logs/
+
+# Install napalm-logs and pre-requisites
+RUN apk add --no-cache \
+    libffi \
+    libffi-dev \
+    python-dev \
+    build-base \
+    && pip --no-cache-dir install cffi /var/cache/napalm-logs/ \
+    && rm -rf /var/cache/napalm-logs/
+
+CMD napalm-logs --config-file /etc/napalm/logs

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2017 Napalm Automation Team
+   Copyright 2017-2019 Mircea Ulinic
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

NOTICE

@@ -0,0 +1,2 @@
+napalm-logs
+Copyright (c) 2017-2019, Mircea Ulinic

README.md

@@ -1,7 +1,7 @@
 # napalm-logs
 
 napalm-logs is a Python library that listens to syslog messages from network
-devices and returns strucuted data following the [OpenConfig](http://www.openconfig.net/)
+devices and returns structured data following the [OpenConfig](http://www.openconfig.net/)
 or [IETF](https://github.com/YangModels/yang/tree/master/standard/ietf) YANG models.
 
 <img src="logo.png" data-canonical-src="logo.png" width="300" />

docker/Makefile

@@ -1,22 +1,20 @@
 NAME := napalm_logs
-# This is the version of napalm-logs that will be installed
-VERSION ?= $(shell cat ../VERSION)
 # This can be changed to build with different BUILD_REQS
-INTERATION := 0
+VERSION ?= $(shell git describe --tags $(git rev-list --tags --max-count=1))
 
 ## To specify which dependencies to install please set BUILD_REQS
 ## For example:
 ## make BUILD_REQS="kafka tornado" build
 
-IMAGE = $(NAME):$(VERSION)-${INTERATION}
+IMAGE = $(NAME):$(VERSION)
 REPO ?= mirceaulinic/napalm-logs
 
 DOCKERFILE ?= Dockerfile
-DOCKER_BUILD_CONTEXT ?= .
+DOCKER_BUILD_CONTEXT ?= $(dir $(abspath $(PWD)../))
 
 .PHONY: build
 build: clean
-	docker build -f $(DOCKERFILE) -t $(IMAGE) $(DOCKER_BUILD_CONTEXT) --build-arg requirements="${BUILD_REQS}" --build-arg version="${VERSION}"
+	docker build -f ../$(DOCKERFILE) -t $(IMAGE) $(DOCKER_BUILD_CONTEXT) --build-arg requirements="${BUILD_REQS}"
 
 .PHONY: publish
 publish: build

docs/conf.py

@@ -64,17 +64,17 @@
 
 # General information about the project.
 project = u'napalm-logs'
-copyright = u'2017, Mircea Ulinic'
+copyright = u'2017-2019, Mircea Ulinic'
 author = u'Mircea Ulinic'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = napalm_logs.__version__
+# version = napalm_logs.__version__
 # The full version, including alpha/beta/rc tags.
-release = napalm_logs.__version__
+# release = napalm_logs.__version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/developers/device_profiles.rst

@@ -203,7 +203,7 @@ values using replace functions found in napalm_logs.utils.Replace i.e
 This is the YANG model to use to output the log message. You can find all
 models and their structure here_.
 
-.. _here: http://ops.openconfig.net/branches/master/
+.. _here: https://yangcatalog.org/yang-search/
 
 ``mapping``
 ------------
@@ -245,9 +245,10 @@ parts extracted from the syslog message, without any further processing. The
 following keys are mandatory:
 
 - ``host``: the network device hostname, as provided in the syslog message
-prefix section.
+  prefix section.
 - ``tag``: which is the unique identification tag of the syslog message, e.g. in the message ``Mar 30 12:45:19 re0.edge01.bjm01 rpd[15852]: BGP_PREFIX_THRESH_EXCEEDED 1.2.3.4 (External AS 15169): Configured maximum prefix-limit threshold(160) exceeded for inet-unicast nlri: 181 (instance master)``, the ``tag`` is ``BGP_PREFIX_THRESH_EXCEEDED``. Other tag examples: ``bgp_read_message``, ``ROUTING-BGP-5-MAXPFX`` or even ``Alarm set``.
-- ``message``: is the message that what we have defied earlier as *the message section*, e.g. ``User 'dummy' entering configuration mode``.
+- ``message``: is the message that what we have defied earlier as *the message
+  section*, e.g. ``User 'dummy' entering configuration mode``.
 
 .. note::
 

docs/device_config/index.rst

@@ -17,6 +17,7 @@ systems:
    ios
    netiron
    huawei
+   sonic
 
 To see how to configure the network device, check the documents referenced
 above. Note that the examples in each case represents the configuration used to

docs/device_config/ios.rst

@@ -4,4 +4,19 @@
 Cisco IOS
 =========
 
-.. versionadded:: 0.5.0
+The following will configure IOS to send the syslog messages, over UDP, to the
+IP Address ``10.10.10.1``, port 10101:
+
+
+.. code-block:: text
+
+    logging host 10.10.10.1 transport udp port 10101
+
+To correctly send the hostname information, it is also recommended to explicitly
+configure the following:
+
+.. code-block:: text
+
+    logging origin-id hostname
+
+Otherwise the device won't send this information.

docs/device_config/sonic.rst

@@ -0,0 +1,32 @@
+.. _device-configuration-sonic:
+
+===========
+Azure SONiC
+===========
+
+.. versionadded:: 0.8.0
+
+The following will configure Azure Sonic to send syslog messages, over UDP, to the 
+IP Address ``10.10.10.1``, port 514 and using the syslog-protocol-23 format.
+
+The Rsyslogd is configured via the rsyslog.conf file, found in ``/etc``.
+
+
+First, configure Rsyslogd to use the SyslogProtocol23Format template
+
+.. code-block:: text
+
+    $template RSYSLOG_SyslogProtocol23Format,"<%PRI%>1 %TIMESTAMP:::date-rfc3339% %HOSTNAME% %APP-NAME% %PROCID% %MSGID% %STRUCTURED-DATA%%msg%\n"
+
+Then, set the remote syslog server
+
+.. code-block:: text
+
+    *.* @10.10.10.1:514;RSYSLOG_SyslogProtocol23Format
+
+To correctly send the hostname information, it is also recommended to explicitly
+configure the following:
+
+.. code-block:: text
+
+    $PreserveFQDN on

docs/message_template.jinja

@@ -10,7 +10,7 @@
 There is no YANG model available yet to map this class of messages.
 Please check the :ref:`{{ error_name }}-json` section to see the structure.
 {% else -%}
-Maps to the ``{{ error_yang }}`` YANG model.
+Maps to the `{{ error_yang }} <https://yangcatalog.org/yang-search/yang_tree/{{ error_yang }}>`__ YANG model.
 {% endif %}
 
 .. _{{ error_name }}-os-list:

docs/messages_index_template.jinja

@@ -14,7 +14,8 @@ For example, the following syslog message:
 
 ``napalm-logs`` identifies that it was produced by a Junos device and assigns
 the error tag ``BGP_PREFIX_LIMIT_EXCEEDED`` and then will try to map the
-information into the OpenConfig model ``openconfig_bgp``:
+information into the OpenConfig model
+`openconfig-bgp <https://yangcatalog.org/yang-search/yang_tree/openconfig-bgp>`__:
 
 .. code-block:: json
 
@@ -65,7 +66,7 @@ information into the OpenConfig model ``openconfig_bgp``:
 	  "facility": 18,
 	  "ip": "127.0.0.1",
 	  "host": "vmx01",
-	  "yang_model": "openconfig_bgp",
+	  "yang_model": "openconfig-bgp",
 	  "error": "BGP_PREFIX_THRESH_EXCEEDED",
 	  "os": "junos",
 	  "severity": 5

docs/metrics/index.rst

@@ -4,16 +4,16 @@
 Metrics Collection
 ==================
 
-Timeseries metrics my be optionally collected and exported from the server.
-This feature is dissabled by default but can be enabled by passing `--enable-metrics`
+Timeseries metrics may be optionally collected and exported from the server.
+This feature is disabled by default but can be enabled by passing `--enable-metrics`
 when envoking the server.
 
 The metrics offer insight into how the server is processing messages. Each subsystem publishes
-its own set of metrics. These are timeserises data so they are mostly counters of processed
+its own set of metrics. These are timeseries data so they are mostly counters of processed
 elements.
 
 We use the Prometheus metrics format and the metrics are exposed by an HTTP server which by default
-run on port tcp/9443. The implementation is fully complient with Prometheus scraping, so you need only
+run on port tcp/9443. The implementation is fully compliant with Prometheus scraping, so you need only
 point your Prometheus server to the exposed metrics to scrape them.
 
 When using the feature, you must make a directory available which will be used to collect and store
@@ -32,7 +32,7 @@ napalm_logs_listener_logs_ingested
   Count of ingested log messages. Labels are used to seperate metrics for each Listener process.
 
 napalm_logs_listener_messages_published
-  Count of published messages. The are messages published to the message queue for processing by the Server Process.
+  Count of published messages. These are messages published to the message queue for processing by the Server Process.
   Labels are used to seperate metrics for each Listener process.
 
 Server Process

docs/publisher/index.rst

@@ -156,7 +156,7 @@ Configuration example:
 .. _publisher-opts-no-encrypt:
 
 ``no_encrypt``: ``False``
-------------------------
+-------------------------
 
 .. versionadded:: 0.4.2
 

docs/syslog/eos.rst

@@ -20,5 +20,4 @@ Where:
 
 Examples:
 
-``<149>Apr 16 11:04:17 edge01 Rib: %BGP-3-NOTIFICATION: received from neighbor 194.53.172.97 (AS 2611) 6/1 (Cease/maximum number of prefixes reached) 0 bytes
-``
+``<149>Apr 16 11:04:17 edge01 Rib: %BGP-3-NOTIFICATION: received from neighbor 194.53.172.97 (AS 2611) 6/1 (Cease/maximum number of prefixes reached) 0 bytes``

docs/syslog/nxos.rst

@@ -19,5 +19,4 @@ Where:
 
 Examples:
 
-``<190>sw01.pdx01: 2017 Jul 28 14:42:46 UTC: %AUTHPRIV-6-SYSTEM_MSG: pam_unix(dcos_sshd:session): session opened for user luke by (uid=0) - dcos_sshd[12977]
-``
+``<190>sw01.pdx01: 2017 Jul 28 14:42:46 UTC: %AUTHPRIV-6-SYSTEM_MSG: pam_unix(dcos_sshd:session): session opened for user luke by (uid=0) - dcos_sshd[12977]``

examples/client_no_auth.py

@@ -13,6 +13,7 @@
 '''
 import zmq
 import napalm_logs.utils
+import json
 
 server_address = '127.0.0.1'  # --publish-address
 server_port = 49017           # --publish-port
@@ -22,8 +23,8 @@
 socket = context.socket(zmq.SUB)
 socket.connect('tcp://{address}:{port}'.format(address=server_address,
                                                port=server_port))
-socket.setsockopt(zmq.SUBSCRIBE, '')
+socket.setsockopt(zmq.SUBSCRIBE, b'')
 
 while True:
     raw_object = socket.recv()
-    print(napalm_logs.utils.unserialize(raw_object))
+    print(json.dumps(napalm_logs.utils.unserialize(raw_object)))

napalm_logs/config/eos/BFD_STATE_CHANGE.yml

@@ -0,0 +1,20 @@
+# a comment
+messages:
+  - error: BFD_STATE_CHANGE
+    tag: BFD-5-STATE_CHANGE
+    values:
+      ip: ([\w\d:\.]+)
+      srcip: ([\w\d:\.]+)
+      interface: (\w+[\.\-\d\/\w]+)
+      vrf: (\w+)
+      type: (\w+)
+      diag: (\w+)
+      oldState: (\w+)
+      newState|bfd_state_convert: (\w+)
+    line: ': peer (vrf:{vrf}, ip:{ip}, intf:{interface}, srcIp:{srcip}, type:{type}) changed state from {oldState} to {newState} diag {diag}'
+    model: openconfig-bfd
+    mapping:
+      variables:
+        bfd//interfaces//interface//id: interface
+        bfd//interfaces//interface//peers//peer//state//session-state: newState
+      static: {}