From fa7d49fc0ab4ee6b9c574f6b6644d51eab800725 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Tue, 8 May 2018 15:11:28 -0400 Subject: [PATCH 01/45] implements #160 - prometheus metrics --- docs/metrics/index.rst | 17 +++++++++ docs/options/index.rst | 65 ++++++++++++++++++++++++++++++++++ napalm_logs/base.py | 30 ++++++++++++++++ napalm_logs/config/__init__.py | 2 ++ napalm_logs/device.py | 27 ++++++++++++++ napalm_logs/listener_proc.py | 14 ++++++++ napalm_logs/publisher.py | 32 +++++++++++++++++ napalm_logs/scripts/cli.py | 24 +++++++++++++ napalm_logs/server.py | 44 +++++++++++++++++++++++ requirements.txt | 1 + 10 files changed, 256 insertions(+) create mode 100644 docs/metrics/index.rst diff --git a/docs/metrics/index.rst b/docs/metrics/index.rst new file mode 100644 index 00000000..6500fa4f --- /dev/null +++ b/docs/metrics/index.rst @@ -0,0 +1,17 @@ +.. _metrics: + +================== +Metrics Collection +================== + +Timeseries metrics my be optionally collected and exported from the server. +This feature is dissabled by default but can be enabled by passing `--enable-metrics` +when envoking the server. + +The metrics offer insight into how the server is processing messages. Each subsystem publishes +its own set of metrics. These are timeserises data so they are mostly counters of processed +elements. + +We use the Prometheus metrics format and the metrics are exposed by an HTTP server which by default +run on for 9215. The implementation is fully complient with Prometheus scraping, so you need only +point your Prometheus server to the exposed metrics to scrape them. diff --git a/docs/options/index.rst b/docs/options/index.rst index 7ce1a2f8..0fae7c64 100644 --- a/docs/options/index.rst +++ b/docs/options/index.rst @@ -76,6 +76,71 @@ Configuration file example: auth_port: 2022 +.. _configuration-options-enable-metrics: + +``enable-metrics`` +------------- + +Enable metrics collection exposition (Prometheus metrics). + +Default: ``false`` + +CLI usgae example: + +.. code-block:: bash + + $ napalm-logs --enable-metrics + +Configuration file example: + +.. code-block:: yaml + + metrics_enabled: true + +.. _configuration-options-metrics-address: + +``metrics-address`` +----------- + +The IP address to use to listen for all incoming metrics requests. This is the +address that the Prometheus HTTP server exposes metrics from. + +Default: ``0.0.0.0``. + +CLI usage example: + +.. code-block:: bash + + $ napalm-logs --metrics-address 172.17.17.1 + +Configuration file example: + +.. code-block:: yaml + + metrics_address: 172.17.17.1 + +.. _configuration-options-metrics-port: + +``metrics-port`` +------------- + +The port to listen on for incoming metrics requests. This is the port that +the Prometheus HTTP server exposes metrics from. + +Default: ``9215`` + +CLI usgae example: + +.. code-block:: bash + + $ napalm-logs --metrics-port 2022 + +Configuration file example: + +.. code-block:: yaml + + metrics_port: 2022 + .. _configuration-options-certificate: ``certificate`` diff --git a/napalm_logs/base.py b/napalm_logs/base.py index ce44cbec..5f8800e1 100644 --- a/napalm_logs/base.py +++ b/napalm_logs/base.py @@ -21,6 +21,7 @@ import nacl.secret import nacl.signing import nacl.encoding +from prometheus_client import start_http_server, CollectorRegistry, multiprocess # Import napalm-logs pkgs import napalm_logs.utils @@ -48,6 +49,9 @@ def __init__(self, publish_port=49017, auth_address='0.0.0.0', auth_port=49018, + metrics_enabled=False, + metrics_address='0.0.0.0', + metrics_port='9215', certificate=None, keyfile=None, disable_security=False, @@ -80,6 +84,9 @@ def __init__(self, self.publish_port = publish_port self.auth_address = auth_address self.auth_port = auth_port + self.metrics_enabled = metrics_enabled + self.metrics_address = metrics_address + self.metrics_port = metrics_port self.certificate = certificate self.keyfile = keyfile self.disable_security = disable_security @@ -100,6 +107,8 @@ def __init__(self, self._build_config() self._verify_config() self._post_preparation() + # Start the Prometheus metrics server + self._setup_metrics() # Private vars self.__priv_key = None self.__signing_key = None @@ -115,6 +124,27 @@ def __exit__(self, exc_type, exc_value, exc_traceback): log.error('Exiting due to unhandled exception', exc_info=True) self.__raise_clean_exception(exc_type, exc_value, exc_traceback) + def _setup_metrics(self): + """ + Start metric exposition + """ + if self.metrics_enabled: + log.debug("Starting metrics exposition") + path = os.environ.get("prometheus_multiproc_dir") + if path: + log.info("Cleaning metrics collection directory") + files = os.listdir(path) + for f in files: + if f.endswith(".db"): + os.remove(os.path.join(path, f)) + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + start_http_server( + port=self.metrics_port, + addr=self.metrics_address, + registry=registry + ) + def _setup_log(self): ''' Setup the log object. diff --git a/napalm_logs/config/__init__.py b/napalm_logs/config/__init__.py index b620572c..0bd123d6 100644 --- a/napalm_logs/config/__init__.py +++ b/napalm_logs/config/__init__.py @@ -30,6 +30,8 @@ LOG_FILE = os.path.join(ROOT_DIR, 'var', 'log', 'napalm', 'logs') LOG_FILE_CLI_OPTIONS = ('cli', 'screen') ZMQ_INTERNAL_HWM = 1000 +METRICS_ADDRESS = '0.0.0.0' +METRICS_PORT = 9215 # Allowed names for the init files. OS_INIT_FILENAMES = ( diff --git a/napalm_logs/device.py b/napalm_logs/device.py index 0d6a1689..47f14486 100644 --- a/napalm_logs/device.py +++ b/napalm_logs/device.py @@ -15,6 +15,7 @@ # Import thrid party libs import zmq import umsgpack +from prometheus_client import Counter # Import napalm-logs pkgs import napalm_logs.utils @@ -227,6 +228,28 @@ def start(self): ''' Start the worker process. ''' + # metrics + napalm_logs_device_messages_received = Counter( + 'napalm_logs_device_messages_received', + "Count of messages received by the device process", + ['device_os'] + ) + napalm_logs_device_raw_published_messages = Counter( + 'napalm_logs_device_raw_published_messages', + "Count of raw type published messages", + ['device_os'] + ) + napalm_logs_device_published_messages = Counter( + 'napalm_logs_device_published_messages', + "Count of published messages", + ['device_os'] + ) + napalm_logs_device_oc_object_failed = Counter( + 'napalm_logs_device_oc_object_failed', + "Counter of failed OpenConfig object generations", + ['device_os'] + ) + self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),)) @@ -246,6 +269,7 @@ def start(self): else: raise NapalmLogsExit(error) log.debug('%s: dequeued %s, received from %s', self._name, msg_dict, address) + napalm_logs_device_messages_received.labels(device_os=self._name).inc() host = msg_dict.get('host') prefix_id = msg_dict.pop('__prefix_id__') if 'timestamp' in msg_dict: @@ -277,6 +301,7 @@ def start(self): log.debug(to_publish) # self.pub_pipe.send(to_publish) self.pub.send(umsgpack.packb(to_publish)) + napalm_logs_device_raw_published_messages.labels(device_os=self._name).inc() continue try: if '__python_fun__' in kwargs: @@ -286,6 +311,7 @@ def start(self): yang_obj = self._emit(**kwargs) except Exception: log.exception('Unexpected error when generating the OC object.', exc_info=True) + napalm_logs_device_oc_object_failed.labels(device_os=self._name).inc() continue log.debug('Generated OC object:') log.debug(yang_obj) @@ -308,6 +334,7 @@ def start(self): # self.pub_pipe.send(to_publish) self.pub.send(umsgpack.packb(to_publish)) # self._publish(to_publish) + napalm_logs_device_published_messages.labels(device_os=self._name).inc() def stop(self): ''' diff --git a/napalm_logs/listener_proc.py b/napalm_logs/listener_proc.py index 006cce4e..5e3b9acf 100644 --- a/napalm_logs/listener_proc.py +++ b/napalm_logs/listener_proc.py @@ -13,6 +13,7 @@ # Import third party libs import zmq import umsgpack +from prometheus_client import Counter # Import napalm-logs pkgs from napalm_logs.config import LST_IPC_URL @@ -76,6 +77,17 @@ def start(self): ''' Listen to messages and publish them. ''' + # counter metrics for messages + c_logs_ingested = Counter( + 'napalm_logs_listener_logs_ingested', + 'Count of ingested log messages', + ['listener_type', 'address', 'port'], + ) + c_messages_published = Counter( + 'napalm_logs_listener_messages_published', + 'Count of published messages', + ['listener_type', 'address', 'port'], + ) self._setup_ipc() log.debug('Using the %s listener', self._listener_type) self._setup_listener() @@ -99,7 +111,9 @@ def start(self): if not log_message: log.info('Empty message received from %s. Not queueing to the server.', log_source) continue + c_logs_ingested.labels(listener_type=self._listener_type, address=self.address, port=self.port).inc() self.pub.send(umsgpack.packb((log_message, log_source))) + c_messages_published.labels(listener_type=self._listener_type, address=self.address, port=self.port).inc() def stop(self): log.info('Stopping the listener process') diff --git a/napalm_logs/publisher.py b/napalm_logs/publisher.py index f5913585..ec5d920a 100644 --- a/napalm_logs/publisher.py +++ b/napalm_logs/publisher.py @@ -15,6 +15,7 @@ import umsgpack import nacl.utils import nacl.secret +from prometheus_client import Counter # Import napalm-logs pkgs import napalm_logs.utils @@ -131,6 +132,22 @@ def start(self): ''' Listen to messages and publish them. ''' + # metrics + napalm_logs_publisher_received_messages = Counter( + 'napalm_logs_publisher_received_messages', + "Count of messages received by the publisher", + ['publisher_type', 'address', 'port'] + ) + napalm_logs_publisher_whitelist_blacklist_check_fail = Counter( + 'napalm_logs_publisher_whitelist_blacklist_check_fail', + "Count of messages which fail the whitelist/blacklist check", + ['publisher_type', 'address', 'port'] + ) + napalm_logs_publisher_messages_published = Counter( + 'napalm_logs_publisher_messages_published', + "Count of published messages", + ['publisher_type', 'address', 'port'] + ) self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),)) @@ -149,6 +166,11 @@ def start(self): log.error(error, exc_info=True) raise NapalmLogsExit(error) obj = umsgpack.unpackb(bin_obj) + napalm_logs_publisher_received_messages.labels( + publisher_type=self._transport_type, + address=self.address, + port=self.port + ).inc() if not napalm_logs.ext.check_whitelist_blacklist(obj['error'], whitelist=self.error_whitelist, blacklist=self.error_blacklist): @@ -158,6 +180,11 @@ def start(self): obj['error'], self._transport_type, self.pub_id) + napalm_logs_publisher_whitelist_blacklist_check_fail.labels( + publisher_type=self._transport_type, + address=self.address, + port=self.port + ).inc() continue serialized_obj = self._serialize(obj, bin_obj) log.debug('Publishing the OC object') @@ -165,6 +192,11 @@ def start(self): # Encrypt only when needed. serialized_obj = self._prepare(serialized_obj) self.transport.publish(serialized_obj) + napalm_logs_publisher_messages_published.labels( + publisher_type=self._transport_type, + address=self.address, + port=self.port + ).inc() def stop(self): log.info('Stopping publisher process %s (publisher #%d)', self._transport_type, self.pub_id) diff --git a/napalm_logs/scripts/cli.py b/napalm_logs/scripts/cli.py index e3abbdce..0eca7633 100644 --- a/napalm_logs/scripts/cli.py +++ b/napalm_logs/scripts/cli.py @@ -141,6 +141,24 @@ def prepare(self): type=int, help=('Authenticator bind port. Default: {0}'.format(defaults.AUTH_PORT)) ) + self.add_option( + '--enable-metrics', + dest='metrics_enabled', + action="store_true", + default=False, + help=('Enable metrics collection and exporting (Prometheus metrics).') + ) + self.add_option( + '--metrics-address', + dest='metrics_address', + help=('Prometheus metrics HTTP server listener address. Default: {0}'.format(defaults.METRICS_ADDRESS)) + ) + self.add_option( + '--metrics-port', + dest='metrics_port', + type=int, + help=('Prometheus metrics HTTP server listener bind port. Default: {0}'.format(defaults.METRICS_PORT)) + ) self.add_option( '--certificate', dest='certificate', @@ -258,6 +276,7 @@ def parse(self, log, screen_handler): format=log_fmt) # log to filecm cert = self.options.certificate or file_cfg.get('certificate') disable_security = self.options.disable_security or file_cfg.get('disable_security', False) + metrics_enabled = self.options.metrics_enabled or file_cfg.get('metrics_enabled', False) if not cert and disable_security is False: log.error('certfile must be specified for server-side operations') raise ValueError('Please specify a valid SSL certificate.') @@ -325,6 +344,11 @@ def parse(self, log, screen_handler): defaults.AUTH_ADDRESS, # noqa 'auth_port': self.options.auth_port or file_cfg.get('auth_port') or defaults.AUTH_PORT, + 'metrics_enabled': metrics_enabled, + 'metrics_address': self.options.metrics_address or file_cfg.get('metrics_address') or + defaults.METRICS_ADDRESS, + 'metrics_port': self.options.metrics_port or file_cfg.get('metrics_port') or + defaults.METRICS_PORT, 'certificate': cert, 'keyfile': self.options.keyfile or file_cfg.get('keyfile'), 'disable_security': disable_security, diff --git a/napalm_logs/server.py b/napalm_logs/server.py index 84273cc8..215aa7bc 100644 --- a/napalm_logs/server.py +++ b/napalm_logs/server.py @@ -15,6 +15,7 @@ # Import third party libs import zmq import umsgpack +from prometheus_client import Counter # Import napalm-logs pkgs import napalm_logs.ext.six as six @@ -188,6 +189,34 @@ def start(self): inspect and identify the operating system, then queue the message correspondingly. ''' + # metric counters + napalm_logs_server_messages_received = Counter( + "napalm_logs_server_messages_received", + "Count of messages received from listener processes" + ) + napalm_logs_server_messages_with_identified_os = Counter( + "napalm_logs_server_messages_with_identified_os", + "Count of messages with positive os identification", + ['device_os'] + ) + napalm_logs_server_messages_without_identified_os = Counter( + "napalm_logs_server_messages_without_identified_os", + "Count of messages with negative os identification" + ) + napalm_logs_server_messages_failed_device_queuing = Counter( + "napalm_logs_server_messages_failed_device_queuing", + "Count of messages per device os that fail to be queued to a device process", + ['device_os'] + ) + napalm_logs_server_messages_device_queued = Counter( + "napalm_logs_server_messages_device_queued", + "Count of messages queued to device processes", + ['device_os'] + ) + napalm_logs_server_messages_as_is_queued = Counter( + "napalm_logs_server_messages_as_is_queued", + "Count of messages queued as is" + ) self._setup_ipc() # Start suicide polling thread thread = threading.Thread(target=self._suicide_when_without_parent, args=(os.getppid(),)) @@ -211,7 +240,9 @@ def start(self): else: msg = msg.encode('utf-8') log.debug('[%s] Dequeued message from %s: %s', address, msg, time.time()) + napalm_logs_server_messages_received.inc() dev_os, msg_dict = self._identify_os(msg) + if dev_os and dev_os in self.started_os_proc: # Identified the OS and the corresponding process is started. # Then send the message in the right queue @@ -222,9 +253,15 @@ def start(self): self.pub.send_multipart([dev_os, umsgpack.packb((msg_dict, address))]) # self.os_pipes[dev_os].send((msg_dict, address)) + napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os).inc() + napalm_logs_server_messages_device_queued.labels(device_os=dev_os).inc() + elif dev_os and dev_os not in self.started_os_proc: # Identified the OS, but the corresponding process does not seem to be started. log.info('Unable to queue the message to %s. Is the sub-process started?', dev_os) + napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os).inc() + napalm_logs_server_messages_failed_device_queuing.labels(device_os=dev_os).inc() + elif not dev_os and self.opts['_server_send_unknown']: # OS not identified, but the user requested to publish the message as-is log.debug('Unable to identify the OS, sending directly to the publishers') @@ -238,6 +275,13 @@ def start(self): 'model_name': 'unknown' } self.publisher_pub.send(umsgpack.packb(to_publish)) + napalm_logs_server_messages_as_is_queued.inc() + napalm_logs_server_messages_without_identified_os.inc() + + elif not dev_os and not self.opts['_server_send_unknown']: + # OS not identified and we are told to do nothing + log.debug('Unable to identify the OS') + napalm_logs_server_messages_without_identified_os.inc() def stop(self): log.info('Stopping server process') diff --git a/requirements.txt b/requirements.txt index a0b9b34f..aa7474ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pyzmq pyyaml pynacl u-msgpack-python +prometheus_client From df63f514941cc6db029b50f9ca09c9c4880c978e Mon Sep 17 00:00:00 2001 From: John Anderson Date: Tue, 8 May 2018 15:16:17 -0400 Subject: [PATCH 02/45] added docs for prometheus_multiproc_dir env var --- docs/metrics/index.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/metrics/index.rst b/docs/metrics/index.rst index 6500fa4f..853f5896 100644 --- a/docs/metrics/index.rst +++ b/docs/metrics/index.rst @@ -15,3 +15,13 @@ elements. We use the Prometheus metrics format and the metrics are exposed by an HTTP server which by default run on for 9215. The implementation is fully complient with Prometheus scraping, so you need only point your Prometheus server to the exposed metrics to scrape them. + +When using the feature, you need make a directory available which will be used to collect and store +local metrics from the valious system processes. The server will need rights to write to this +directory. A directory in ``/tmp`` is fine. Once created you will need to set the +``prometheus_multiproc_dir`` environment variable which will be read by the server on startup. + +.. code-block:: bash + + $ mkdir /tmp/prom + $ export prometheus_multiproc_dir=/tmp/prom From 1b2d476acb2d7007f489a78715c18d3b871d04d5 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Wed, 9 May 2018 14:25:05 -0400 Subject: [PATCH 03/45] changed the way prometheus_multiproc_dir is initialized, and updated docs --- docs/index.rst | 1 + docs/metrics/index.rst | 83 ++++++++++++++++++++++++++++++---- docs/options/index.rst | 22 ++++++++- napalm_logs/__init__.py | 8 ++++ napalm_logs/base.py | 25 +++++++--- napalm_logs/config/__init__.py | 3 +- napalm_logs/scripts/cli.py | 7 +++ napalm_logs/server.py | 8 ++-- 8 files changed, 137 insertions(+), 20 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4562e49c..a2bfe377 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -309,6 +309,7 @@ daemon): options/index clients/index messages/index + metrics/index authentication/index listener/index publisher/index diff --git a/docs/metrics/index.rst b/docs/metrics/index.rst index 853f5896..c1051774 100644 --- a/docs/metrics/index.rst +++ b/docs/metrics/index.rst @@ -13,15 +13,82 @@ its own set of metrics. These are timeserises data so they are mostly counters o elements. We use the Prometheus metrics format and the metrics are exposed by an HTTP server which by default -run on for 9215. The implementation is fully complient with Prometheus scraping, so you need only +run on port tcp/9443. The implementation is fully complient with Prometheus scraping, so you need only point your Prometheus server to the exposed metrics to scrape them. -When using the feature, you need make a directory available which will be used to collect and store -local metrics from the valious system processes. The server will need rights to write to this -directory. A directory in ``/tmp`` is fine. Once created you will need to set the -``prometheus_multiproc_dir`` environment variable which will be read by the server on startup. +When using the feature, you must make a directory available which will be used to collect and store +local metrics from the various system processes. The server will need rights to write to this +directory. By default `/tmp/napalm_logs_metrics` is used. This is configurable with +`--metrics-dir`. Whatever the value, if the directory does not exist, we attempt to create it +on startup. Also note than this directory is cleared of all previous metrics entries on each run. +This means that metric values are not persisted through runs. -.. code-block:: bash +Here is a listing of the exposed metrics and their meaning: - $ mkdir /tmp/prom - $ export prometheus_multiproc_dir=/tmp/prom +Listener Process(es) +-------------------- + +napalm_logs_listener_logs_ingested + Count of ingested log messages. Labels are used to seperate metrics for each Listener process. + +napalm_logs_listener_messages_published + Count of published messages. The are messages published to the message queue for processing by the Server Process. + Labels are used to seperate metrics for each Listener process. + +Server Process +-------------- + +napalm_logs_server_messages_received + Count of messages received from Listener processes. + +napalm_logs_server_messages_with_identified_os + Count of messages with positive OS identification. Labels are used to seperate metrics for each Device OS. + +napalm_logs_server_messages_without_identified_os + Count of messages which fail OS identification. + +napalm_logs_server_messages_failed_device_queuing + Count of messages per device OS that fail to be queued to a proper Device process. Note these are messages that + pass OS identification and we know how to route them but fail to be queued. Labels are used to seperate metrics + for each Device OS. + +napalm_logs_server_messages_device_queued + Count of messages sucessfully queued to Device processes. Labels are used to seperate metrics for each Device OS process. + +napalm_logs_server_messages_unknown_queued + Count of messages which fail OS indentification and thus we don't know how to route them, but the user has instructed + the system to queue them "as-is." + +Device Process(es) +------------------ + +napalm_logs_device_messages_received + Count of messages received from the Server process. Labels are used to seperate metrics for each Device OS process. + +napalm_logs_device_raw_published_messages + Count of raw type published messages. In this case, the message did not match a configured message type but the + user has instructed the system to publish the message in a raw format. Labels are used to seperate metrics for + each Device OS process. + +napalm_logs_device_published_messages + Count of published messages. These are messages which are sucessfully converted to an OpenConfig format. Labels + are used to seperate metrics for each Device OS process. + +napalm_logs_device_oc_object_failed + Counter of failed OpenConfig object generations. These are messages for which the system attempts to map to a + known OpenConfig object model but fails. Labels are used to seperate metrics for each Device OS process. + +Publisher Process(es) +--------------------- + +napalm_logs_publisher_received_messages + Count of messages received by the Publisher from Device Process(es). Labels are used to seperate metrics for + each Publisher process. + +napalm_logs_publisher_whitelist_blacklist_check_fail + Count of messages which fail the whitelist/blacklist check. Labels are used to seperate metrics for each + Publisher process. + +napalm_logs_publisher_messages_published + Count of published messages. These are messages which are published for clients to receive (i.e. output of the + system). Labels are used to seperate metrics for each Publisher process. diff --git a/docs/options/index.rst b/docs/options/index.rst index 0fae7c64..86b15e87 100644 --- a/docs/options/index.rst +++ b/docs/options/index.rst @@ -127,7 +127,7 @@ Configuration file example: The port to listen on for incoming metrics requests. This is the port that the Prometheus HTTP server exposes metrics from. -Default: ``9215`` +Default: ``9443`` CLI usgae example: @@ -141,6 +141,26 @@ Configuration file example: metrics_port: 2022 +``metrics-dir`` +------------- + +The directory used by the processes for metrics collection. This directory must +be writable. If the directory does not exist, we attempt to create it. + +Default: ``/tmp/napalm_logs_metrics`` + +CLI usgae example: + +.. code-block:: bash + + $ napalm-logs --metrics-dir /tmp/a_new_dir_for_metrics + +Configuration file example: + +.. code-block:: yaml + + metrics_dir: /tmp/a_new_dir_for_metrics + .. _configuration-options-certificate: ``certificate`` diff --git a/napalm_logs/__init__.py b/napalm_logs/__init__.py index ea73e2c6..02d77944 100644 --- a/napalm_logs/__init__.py +++ b/napalm_logs/__init__.py @@ -6,6 +6,14 @@ from __future__ import unicode_literals import pkg_resources +import os + +# For metrics collection, the prometheus_client library +# requires that the env var `prometheus_multiproc_dir` +# be set before start up. At startup, it doesn't matter +# what the value is, and we always overwrite it to either +# the poperly defined default or the user propvided value. +os.environ['prometheus_multiproc_dir'] = "/tmp/napalm_logs_metrics" # Import napalm-logs pkgs from napalm_logs.base import NapalmLogs diff --git a/napalm_logs/base.py b/napalm_logs/base.py index 5f8800e1..83867356 100644 --- a/napalm_logs/base.py +++ b/napalm_logs/base.py @@ -52,6 +52,7 @@ def __init__(self, metrics_enabled=False, metrics_address='0.0.0.0', metrics_port='9215', + metrics_dir=None, certificate=None, keyfile=None, disable_security=False, @@ -87,6 +88,7 @@ def __init__(self, self.metrics_enabled = metrics_enabled self.metrics_address = metrics_address self.metrics_port = metrics_port + self.metrics_dir = metrics_dir self.certificate = certificate self.keyfile = keyfile self.disable_security = disable_security @@ -131,12 +133,23 @@ def _setup_metrics(self): if self.metrics_enabled: log.debug("Starting metrics exposition") path = os.environ.get("prometheus_multiproc_dir") - if path: - log.info("Cleaning metrics collection directory") - files = os.listdir(path) - for f in files: - if f.endswith(".db"): - os.remove(os.path.join(path, f)) + if not os.path.exists(self.metrics_dir): + try: + log.info("Creating metrics directory") + os.makedirs(self.metrics_dir) + except OSError: + log.error("Failed to create metrics directory!") + raise ConfigurationException("Failed to create metrics directory!") + path = self.metrics_dir + elif path != self.metrics_dir: + path = self.metrics_dir + os.environ['prometheus_multiproc_dir'] = path + log.info("Cleaning metrics collection directory") + log.debug("Metrics directory set to: {}".format(path)) + files = os.listdir(path) + for f in files: + if f.endswith(".db"): + os.remove(os.path.join(path, f)) registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) start_http_server( diff --git a/napalm_logs/config/__init__.py b/napalm_logs/config/__init__.py index 0bd123d6..612c45d1 100644 --- a/napalm_logs/config/__init__.py +++ b/napalm_logs/config/__init__.py @@ -31,7 +31,8 @@ LOG_FILE_CLI_OPTIONS = ('cli', 'screen') ZMQ_INTERNAL_HWM = 1000 METRICS_ADDRESS = '0.0.0.0' -METRICS_PORT = 9215 +METRICS_PORT = 9443 +METRICS_DIR = "/tmp/napalm_logs_metrics" # Allowed names for the init files. OS_INIT_FILENAMES = ( diff --git a/napalm_logs/scripts/cli.py b/napalm_logs/scripts/cli.py index 0eca7633..8f3f6aed 100644 --- a/napalm_logs/scripts/cli.py +++ b/napalm_logs/scripts/cli.py @@ -159,6 +159,11 @@ def prepare(self): type=int, help=('Prometheus metrics HTTP server listener bind port. Default: {0}'.format(defaults.METRICS_PORT)) ) + self.add_option( + '--metrics-dir', + dest='metrics_dir', + help=('Directory to store metrics in. Must be writable by the processes. Default: {0}'.format(defaults.METRICS_DIR)) + ) self.add_option( '--certificate', dest='certificate', @@ -349,6 +354,8 @@ def parse(self, log, screen_handler): defaults.METRICS_ADDRESS, 'metrics_port': self.options.metrics_port or file_cfg.get('metrics_port') or defaults.METRICS_PORT, + 'metrics_dir': self.options.metrics_dir or file_cfg.get('metrics_dir') or + defaults.METRICS_DIR, 'certificate': cert, 'keyfile': self.options.keyfile or file_cfg.get('keyfile'), 'disable_security': disable_security, diff --git a/napalm_logs/server.py b/napalm_logs/server.py index 215aa7bc..cbc29e22 100644 --- a/napalm_logs/server.py +++ b/napalm_logs/server.py @@ -213,9 +213,9 @@ def start(self): "Count of messages queued to device processes", ['device_os'] ) - napalm_logs_server_messages_as_is_queued = Counter( - "napalm_logs_server_messages_as_is_queued", - "Count of messages queued as is" + napalm_logs_server_messages_unknown_queued = Counter( + "napalm_logs_server_messages_unknown_queued", + "Count of messages queued as unknown" ) self._setup_ipc() # Start suicide polling thread @@ -275,7 +275,7 @@ def start(self): 'model_name': 'unknown' } self.publisher_pub.send(umsgpack.packb(to_publish)) - napalm_logs_server_messages_as_is_queued.inc() + napalm_logs_server_messages_unknown_queued.inc() napalm_logs_server_messages_without_identified_os.inc() elif not dev_os and not self.opts['_server_send_unknown']: From e556d3599b6e269be88c867510ad8138962e4129 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Wed, 9 May 2018 15:08:47 -0400 Subject: [PATCH 04/45] refactored matris startup and pass at pep8 --- napalm_logs/__init__.py | 2 +- napalm_logs/base.py | 40 +++++++++++++++++++------------------- napalm_logs/scripts/cli.py | 3 ++- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/napalm_logs/__init__.py b/napalm_logs/__init__.py index 02d77944..88096f61 100644 --- a/napalm_logs/__init__.py +++ b/napalm_logs/__init__.py @@ -16,7 +16,7 @@ os.environ['prometheus_multiproc_dir'] = "/tmp/napalm_logs_metrics" # Import napalm-logs pkgs -from napalm_logs.base import NapalmLogs +from napalm_logs.base import NapalmLogs # noqa: E402 try: __version__ = pkg_resources.get_distribution('napalm-logs').version diff --git a/napalm_logs/base.py b/napalm_logs/base.py index 83867356..49ff9162 100644 --- a/napalm_logs/base.py +++ b/napalm_logs/base.py @@ -21,7 +21,7 @@ import nacl.secret import nacl.signing import nacl.encoding -from prometheus_client import start_http_server, CollectorRegistry, multiprocess +from prometheus_client import start_http_server, CollectorRegistry, multiprocess, core # Import napalm-logs pkgs import napalm_logs.utils @@ -130,26 +130,26 @@ def _setup_metrics(self): """ Start metric exposition """ - if self.metrics_enabled: + path = os.environ.get("prometheus_multiproc_dir") + if not os.path.exists(self.metrics_dir): + try: + log.info("Creating metrics directory") + os.makedirs(self.metrics_dir) + except OSError: + log.error("Failed to create metrics directory!") + raise ConfigurationException("Failed to create metrics directory!") + path = self.metrics_dir + elif path != self.metrics_dir: + path = self.metrics_dir + os.environ['prometheus_multiproc_dir'] = path + log.info("Cleaning metrics collection directory") + log.debug("Metrics directory set to: {}".format(path)) + files = os.listdir(path) + for f in files: + if f.endswith(".db"): + os.remove(os.path.join(path, f)) log.debug("Starting metrics exposition") - path = os.environ.get("prometheus_multiproc_dir") - if not os.path.exists(self.metrics_dir): - try: - log.info("Creating metrics directory") - os.makedirs(self.metrics_dir) - except OSError: - log.error("Failed to create metrics directory!") - raise ConfigurationException("Failed to create metrics directory!") - path = self.metrics_dir - elif path != self.metrics_dir: - path = self.metrics_dir - os.environ['prometheus_multiproc_dir'] = path - log.info("Cleaning metrics collection directory") - log.debug("Metrics directory set to: {}".format(path)) - files = os.listdir(path) - for f in files: - if f.endswith(".db"): - os.remove(os.path.join(path, f)) + if self.metrics_enabled: registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) start_http_server( diff --git a/napalm_logs/scripts/cli.py b/napalm_logs/scripts/cli.py index 8f3f6aed..2380a308 100644 --- a/napalm_logs/scripts/cli.py +++ b/napalm_logs/scripts/cli.py @@ -162,7 +162,8 @@ def prepare(self): self.add_option( '--metrics-dir', dest='metrics_dir', - help=('Directory to store metrics in. Must be writable by the processes. Default: {0}'.format(defaults.METRICS_DIR)) + help=('Directory to store metrics in. Must be writable by the processes. ' + 'Default: {0}'.format(defaults.METRICS_DIR)) ) self.add_option( '--certificate', From 57aea7abba9b2e1ab3122c299b1ce71a2d6afb59 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Wed, 9 May 2018 15:16:05 -0400 Subject: [PATCH 05/45] fix tox complaint about bad default arg --- napalm_logs/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/napalm_logs/base.py b/napalm_logs/base.py index 49ff9162..dacc3686 100644 --- a/napalm_logs/base.py +++ b/napalm_logs/base.py @@ -21,7 +21,7 @@ import nacl.secret import nacl.signing import nacl.encoding -from prometheus_client import start_http_server, CollectorRegistry, multiprocess, core +from prometheus_client import start_http_server, CollectorRegistry, multiprocess # Import napalm-logs pkgs import napalm_logs.utils @@ -52,7 +52,7 @@ def __init__(self, metrics_enabled=False, metrics_address='0.0.0.0', metrics_port='9215', - metrics_dir=None, + metrics_dir='/tmp/napalm_logs_metrics', certificate=None, keyfile=None, disable_security=False, From 5f78086280502a0d2ed19c941e36180dc3064c36 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Wed, 9 May 2018 21:58:47 +0200 Subject: [PATCH 06/45] Enhance INTERFACE_DOWN for IOS --- napalm_logs/config/ios/init.yml | 8 ++++---- .../ios/INTERFACE_DOWN/time_synchronized/syslog.msg | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 tests/config/ios/INTERFACE_DOWN/time_synchronized/syslog.msg diff --git a/napalm_logs/config/ios/init.yml b/napalm_logs/config/ios/init.yml index bab7955e..14f7e20d 100644 --- a/napalm_logs/config/ios/init.yml +++ b/napalm_logs/config/ios/init.yml @@ -6,19 +6,19 @@ prefixes: values: messageId: (\d+) host: ([^ ]+) - date: (\w+\s?\d+) + date: \*?(\w+\s?\W?\d+) time: (\d\d:\d\d:\d\d) milliseconds: (\d\d\d) tag: ([\w-]+) # some messages contain a process number (eg. OSPF related messages) processId: (\d+) - line: '{messageId}: {host}: *{date} {time}.{milliseconds}: %{tag}: Process {processId}, ' + line: '{messageId}: {host}: {date} {time}.{milliseconds}: %{tag}: Process {processId}, ' - time_format: "%b %d %H:%M:%S" values: messageId: (\d+) host: ([^ ]+) - date: (\w+\s?\d+) + date: \*?(\w+\s?\W?\d+) time: (\d\d:\d\d:\d\d) milliseconds: (\d+) tag: ([\w-]+) - line: '{messageId}: {host}: *{date} {time}.{milliseconds}: %{tag}: ' + line: '{messageId}: {host}: {date} {time}.{milliseconds}: %{tag}: ' diff --git a/tests/config/ios/INTERFACE_DOWN/time_synchronized/syslog.msg b/tests/config/ios/INTERFACE_DOWN/time_synchronized/syslog.msg new file mode 100644 index 00000000..3630899b --- /dev/null +++ b/tests/config/ios/INTERFACE_DOWN/time_synchronized/syslog.msg @@ -0,0 +1 @@ +<189>117: NetAuto_CSRv-03: May 9 12:49:27.098: %LINK-5-CHANGED: Interface GigabitEthernet2, changed state to administratively down \ No newline at end of file From a3a905936a2bc2f9c58df9d95e1e034f4dfb5c2f Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 10 May 2018 11:08:25 +0200 Subject: [PATCH 07/45] yang.json added --- .../time_synchronized/yang.json | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json diff --git a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json new file mode 100644 index 00000000..eab85a44 --- /dev/null +++ b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json @@ -0,0 +1,33 @@ +{ + "yang_message": { + "interfaces": { + "interface": { + "GigabitEthernet2": { + "state": { + "admin_status": "DOWN" + } + } + } + } + }, + "message_details": { + "facility": 23, + "time": "08:30:56", + "pri": "189", + "host": "router1", + "tag": "LINK-5-CHANGED", + "messageId": "521", + "date": "Nov 14", + "message": "Interface GigabitEthernet2, changed state to administratively down", + "milliseconds": "699", + "severity": 5 + }, + "host": "router1", + "timestamp": 1510648256, + "error": "INTERFACE_DOWN", + "ip": "127.0.0.1", + "facility": 23, + "os": "ios", + "yang_model": "openconfig-interfaces", + "severity": 5 +} \ No newline at end of file From 0b44e7f6fc8c67563feab0ea9074e95970890b01 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 10 May 2018 11:44:34 +0200 Subject: [PATCH 08/45] yang.json changed --- .../time_synchronized/yang.json | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json index eab85a44..db56a971 100644 --- a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json +++ b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json @@ -1,33 +1,33 @@ { - "yang_message": { - "interfaces": { - "interface": { - "GigabitEthernet2": { - "state": { - "admin_status": "DOWN" - } - } - } - } - }, - "message_details": { - "facility": 23, - "time": "08:30:56", - "pri": "189", - "host": "router1", - "tag": "LINK-5-CHANGED", - "messageId": "521", - "date": "Nov 14", - "message": "Interface GigabitEthernet2, changed state to administratively down", - "milliseconds": "699", - "severity": 5 - }, - "host": "router1", - "timestamp": 1510648256, - "error": "INTERFACE_DOWN", - "ip": "127.0.0.1", - "facility": 23, - "os": "ios", - "yang_model": "openconfig-interfaces", - "severity": 5 + "yang_message": { + "interfaces": { + "interface": { + "GigabitEthernet2": { + "state": { + "admin_status": "DOWN" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "messageId": "117", + "pri": "189", + "host": "NetAuto_CSRv-03", + "tag": "LINK-5-CHANGED", + "time": "12:49:27", + "date": "May 9", + "message": "Interface GigabitEthernet2, changed state to administratively down", + "milliseconds": "098" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "INTERFACE_DOWN", + "host": "NetAuto_CSRv-03", + "yang_model": "openconfig-interfaces", + "timestamp": 1525870167, + "os": "ios", + "severity": 5 } \ No newline at end of file From e2c454d7092f07e44e94e6cc7004621ca1848682 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 10 May 2018 12:01:16 +0200 Subject: [PATCH 09/45] multiple whitespaces allowed between month and day --- napalm_logs/config/ios/init.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/napalm_logs/config/ios/init.yml b/napalm_logs/config/ios/init.yml index 14f7e20d..4428b214 100644 --- a/napalm_logs/config/ios/init.yml +++ b/napalm_logs/config/ios/init.yml @@ -6,7 +6,7 @@ prefixes: values: messageId: (\d+) host: ([^ ]+) - date: \*?(\w+\s?\W?\d+) + date: \*?(\w+\s*\d+) time: (\d\d:\d\d:\d\d) milliseconds: (\d\d\d) tag: ([\w-]+) @@ -17,7 +17,7 @@ prefixes: values: messageId: (\d+) host: ([^ ]+) - date: \*?(\w+\s?\W?\d+) + date: \*?(\w+\s*\d+) time: (\d\d:\d\d:\d\d) milliseconds: (\d+) tag: ([\w-]+) From e1efb8ffe0519504991435311021607370b4674a Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 10 May 2018 12:07:34 +0200 Subject: [PATCH 10/45] 1 or more whitespaces allowed between month and day --- napalm_logs/config/ios/init.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/napalm_logs/config/ios/init.yml b/napalm_logs/config/ios/init.yml index 4428b214..94047879 100644 --- a/napalm_logs/config/ios/init.yml +++ b/napalm_logs/config/ios/init.yml @@ -6,7 +6,7 @@ prefixes: values: messageId: (\d+) host: ([^ ]+) - date: \*?(\w+\s*\d+) + date: \*?(\w+\s+\d+) time: (\d\d:\d\d:\d\d) milliseconds: (\d\d\d) tag: ([\w-]+) @@ -17,7 +17,7 @@ prefixes: values: messageId: (\d+) host: ([^ ]+) - date: \*?(\w+\s*\d+) + date: \*?(\w+\s+\d+) time: (\d\d:\d\d:\d\d) milliseconds: (\d+) tag: ([\w-]+) From 9be0462d2ef37b4cda01509e658723fa8bc2a42e Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Fri, 11 May 2018 09:08:43 +0200 Subject: [PATCH 11/45] 2nd space added to the date in yang.json --- tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json index db56a971..f36caafc 100644 --- a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json +++ b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json @@ -18,7 +18,7 @@ "host": "NetAuto_CSRv-03", "tag": "LINK-5-CHANGED", "time": "12:49:27", - "date": "May 9", + "date": "May 9", "message": "Interface GigabitEthernet2, changed state to administratively down", "milliseconds": "098" }, From bb4282b09c4a07b7c88177e656968f94228f5e40 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Fri, 11 May 2018 15:48:24 +0100 Subject: [PATCH 12/45] Fix #239: Use the most specific address and port options for the listener --- napalm_logs/listener_proc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/napalm_logs/listener_proc.py b/napalm_logs/listener_proc.py index 006cce4e..67ea1856 100644 --- a/napalm_logs/listener_proc.py +++ b/napalm_logs/listener_proc.py @@ -52,6 +52,8 @@ def _setup_listener(self): Setup the transport. ''' listener_class = get_listener(self._listener_type) + self.address = self.listener_opts.pop('address', self.address) + self.port = self.listener_opts.pop('port', self.port) self.listener = listener_class(self.address, self.port, **self.listener_opts) From 353fdbc4b58f81030cb61692b6f935108c77a29f Mon Sep 17 00:00:00 2001 From: Thomas Menari Date: Thu, 16 Nov 2017 14:28:25 -0800 Subject: [PATCH 13/45] Add support for BGP up/down for brocade netiron --- napalm_logs/config/netiron/bgp_down.yml | 28 +++++++++++++++++++++++++ napalm_logs/config/netiron/bgp_up.yml | 28 +++++++++++++++++++++++++ napalm_logs/config/netiron/init.yml | 13 ++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 napalm_logs/config/netiron/bgp_down.yml create mode 100644 napalm_logs/config/netiron/bgp_up.yml create mode 100644 napalm_logs/config/netiron/init.yml diff --git a/napalm_logs/config/netiron/bgp_down.yml b/napalm_logs/config/netiron/bgp_down.yml new file mode 100644 index 00000000..17597076 --- /dev/null +++ b/napalm_logs/config/netiron/bgp_down.yml @@ -0,0 +1,28 @@ +messages: + - error: BGP_NEIGHBOR_DOWN_DEFAULT_VRF + tag: BGP + values: + peer: ([\w\d:\.]+) + reason: ([^\(\)]+) + line: 'Peer (VRF: default-vrf) {peer} DOWN ({reason})' + model: openconfig-bgp + mapping: + variables: + bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason + static: + bgp//neighbors//neighbor//{peer}//state//session-state: IDLE + bgp//neighbors//neighbor//{peer}//state//session-state-old: ESTABLISHED + - error: BGP_NEIGHBOR_DOWN_WITHIN_VRF + tag: BGP + values: + instance: (\w+) + peer: ([\w\d:\.]+) + reason: ([^\(\)]+) + line: 'Peer (VRF: {instance}) {peer} DOWN ({reason})' + model: openconfig-network-instance + mapping: + variables: + network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason + static: + network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state: IDLE + network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state-old: ESTABLISHED diff --git a/napalm_logs/config/netiron/bgp_up.yml b/napalm_logs/config/netiron/bgp_up.yml new file mode 100644 index 00000000..3c75a261 --- /dev/null +++ b/napalm_logs/config/netiron/bgp_up.yml @@ -0,0 +1,28 @@ +messages: + - error: BGP_NEIGHBOR_UP_DEFAULT_VRF + tag: BGP + values: + peer: ([\w\d:\.]+) + reason: ([^\(\)]+) + line: 'Peer (VRF: default-vrf) {peer} UP ({reason})' + model: openconfig-bgp + mapping: + variables: + bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason + static: + bgp//neighbors//neighbor//{peer}//state//session-state: ESTABLISHED + bgp//neighbors//neighbor//{peer}//state//session-state-old: OPEN_CONFIRM + - error: BGP_NEIGHBOR_UP_WITHIN_VRF + tag: BGP + values: + instance: (\w+) + peer: ([\w\d:\.]+) + reason: ([^\(\)]+) + line: 'Peer (VRF: {instance}) {peer} UP ({reason})' + model: openconfig-network-instance + mapping: + variables: + network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason + static: + network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state: ESTABLISHED + network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state-old: OPEN_CONFIRM diff --git a/napalm_logs/config/netiron/init.yml b/napalm_logs/config/netiron/init.yml new file mode 100644 index 00000000..fbd7c407 --- /dev/null +++ b/napalm_logs/config/netiron/init.yml @@ -0,0 +1,13 @@ +# You should not use special characters in the value keys +# +# A single whitespace in `line` will match any number of whitespaces. You should be explicit when +# Matching white space in `values` +# +prefixes: + - time_format: "%b %d %H:%M:%S" + values: + date: (\w+ +\d+) + time: (\d\d:\d\d:\d\d) + host: ([^ ]+) + tag: (\w+) + line: '{date} {time} {host} {tag}: ' From 415d4fa8e45ada99714594cb561dc385afa8660f Mon Sep 17 00:00:00 2001 From: Johan van den Dorpe Date: Fri, 6 Apr 2018 15:24:39 +0100 Subject: [PATCH 14/45] Move all BGP state changes into single config --- .../netiron/BGP_NEIGHBOR_STATE_CHANGED.yml | 53 +++++++++++++++++++ napalm_logs/config/netiron/bgp_down.yml | 28 ---------- napalm_logs/config/netiron/bgp_up.yml | 28 ---------- 3 files changed, 53 insertions(+), 56 deletions(-) create mode 100644 napalm_logs/config/netiron/BGP_NEIGHBOR_STATE_CHANGED.yml delete mode 100644 napalm_logs/config/netiron/bgp_down.yml delete mode 100644 napalm_logs/config/netiron/bgp_up.yml diff --git a/napalm_logs/config/netiron/BGP_NEIGHBOR_STATE_CHANGED.yml b/napalm_logs/config/netiron/BGP_NEIGHBOR_STATE_CHANGED.yml new file mode 100644 index 00000000..3165512e --- /dev/null +++ b/napalm_logs/config/netiron/BGP_NEIGHBOR_STATE_CHANGED.yml @@ -0,0 +1,53 @@ +# This error tag corresponds to syslog messages notifying that the configured +# bgp neighbor has changed state +messages: + - error: BGP_NEIGHBOR_STATE_CHANGED + tag: BGP + values: + instance: ([\w-]+) + peer: ([\w\d:\.]+) + reason: (.*) + line: 'Peer (VRF: {instance}) {peer} UP ({reason})' + model: openconfig-bgp + mapping: + variables: + bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason + static: + bgp//neighbors//neighbor//{peer}//state//session-state: ESTABLISHED + - error: BGP_NEIGHBOR_STATE_CHANGED + tag: BGP + values: + instance: ([\w-]+) + peer: ([\w\d:\.]+) + reason: (.*) + line: 'Peer (VRF: {instance}) {peer} DOWN ({reason})' + model: openconfig-bgp + mapping: + variables: + bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason + static: + bgp//neighbors//neighbor//{peer}//state//session-state: IDLE + - error: BGP_NEIGHBOR_STATE_CHANGED + tag: BGP + values: + peer: ([\w\d:\.]+) + reason: (.*) + line: 'Peer {peer} UP ({reason})' + model: openconfig-bgp + mapping: + variables: + bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason + static: + bgp//neighbors//neighbor//{peer}//state//session-state: ESTABLISHED + - error: BGP_NEIGHBOR_STATE_CHANGED + tag: BGP + values: + peer: ([\w\d:\.]+) + reason: (.*) + line: 'Peer {peer} DOWN ({reason})' + model: openconfig-bgp + mapping: + variables: + bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason + static: + bgp//neighbors//neighbor//{peer}//state//session-state: IDLE \ No newline at end of file diff --git a/napalm_logs/config/netiron/bgp_down.yml b/napalm_logs/config/netiron/bgp_down.yml deleted file mode 100644 index 17597076..00000000 --- a/napalm_logs/config/netiron/bgp_down.yml +++ /dev/null @@ -1,28 +0,0 @@ -messages: - - error: BGP_NEIGHBOR_DOWN_DEFAULT_VRF - tag: BGP - values: - peer: ([\w\d:\.]+) - reason: ([^\(\)]+) - line: 'Peer (VRF: default-vrf) {peer} DOWN ({reason})' - model: openconfig-bgp - mapping: - variables: - bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason - static: - bgp//neighbors//neighbor//{peer}//state//session-state: IDLE - bgp//neighbors//neighbor//{peer}//state//session-state-old: ESTABLISHED - - error: BGP_NEIGHBOR_DOWN_WITHIN_VRF - tag: BGP - values: - instance: (\w+) - peer: ([\w\d:\.]+) - reason: ([^\(\)]+) - line: 'Peer (VRF: {instance}) {peer} DOWN ({reason})' - model: openconfig-network-instance - mapping: - variables: - network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason - static: - network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state: IDLE - network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state-old: ESTABLISHED diff --git a/napalm_logs/config/netiron/bgp_up.yml b/napalm_logs/config/netiron/bgp_up.yml deleted file mode 100644 index 3c75a261..00000000 --- a/napalm_logs/config/netiron/bgp_up.yml +++ /dev/null @@ -1,28 +0,0 @@ -messages: - - error: BGP_NEIGHBOR_UP_DEFAULT_VRF - tag: BGP - values: - peer: ([\w\d:\.]+) - reason: ([^\(\)]+) - line: 'Peer (VRF: default-vrf) {peer} UP ({reason})' - model: openconfig-bgp - mapping: - variables: - bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason - static: - bgp//neighbors//neighbor//{peer}//state//session-state: ESTABLISHED - bgp//neighbors//neighbor//{peer}//state//session-state-old: OPEN_CONFIRM - - error: BGP_NEIGHBOR_UP_WITHIN_VRF - tag: BGP - values: - instance: (\w+) - peer: ([\w\d:\.]+) - reason: ([^\(\)]+) - line: 'Peer (VRF: {instance}) {peer} UP ({reason})' - model: openconfig-network-instance - mapping: - variables: - network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state-change-event: reason - static: - network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state: ESTABLISHED - network-instances//network-instance//{instance}//protocols//bgp//bgp//neighbors//neighbor//{peer}//state//session-state-old: OPEN_CONFIRM From 053dd7de4a992008a56971da516e1178284865a7 Mon Sep 17 00:00:00 2001 From: Johan van den Dorpe Date: Tue, 22 May 2018 12:21:51 +0100 Subject: [PATCH 15/45] Add tests for netiron --- .../established_to_idle_no_vrf/syslog.msg | 1 + .../established_to_idle_no_vrf/yang.json | 34 +++++++++++++++++++ .../established_to_idle_with_vrf/syslog.msg | 1 + .../established_to_idle_with_vrf/yang.json | 34 +++++++++++++++++++ .../idle_to_established_no_vrf/syslog.msg | 1 + .../idle_to_established_no_vrf/yang.json | 34 +++++++++++++++++++ .../idle_to_established_with_vrf/syslog.msg | 1 + .../idle_to_established_with_vrf/yang.json | 34 +++++++++++++++++++ 8 files changed, 140 insertions(+) create mode 100644 tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_no_vrf/syslog.msg create mode 100644 tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_no_vrf/yang.json create mode 100644 tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_with_vrf/syslog.msg create mode 100644 tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_with_vrf/yang.json create mode 100644 tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_no_vrf/syslog.msg create mode 100644 tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_no_vrf/yang.json create mode 100644 tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_with_vrf/syslog.msg create mode 100644 tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_with_vrf/yang.json diff --git a/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_no_vrf/syslog.msg b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_no_vrf/syslog.msg new file mode 100644 index 00000000..e80c228f --- /dev/null +++ b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_no_vrf/syslog.msg @@ -0,0 +1 @@ +<13>May 19 03:57:38 HOSTNAME BGP: Peer 10.10.7.134 DOWN (Hold Timer Expired) \ No newline at end of file diff --git a/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_no_vrf/yang.json b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_no_vrf/yang.json new file mode 100644 index 00000000..4a9b89bf --- /dev/null +++ b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_no_vrf/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "bgp": { + "neighbors": { + "neighbor": { + "10.10.7.134": { + "state": { + "session-state": "IDLE", + "session-state-change-event": "Hold Timer Expired" + } + } + } + } + } + }, + "message_details": { + "severity": 5, + "date": "May 19", + "pri": "13", + "host": "HOSTNAME", + "tag": "BGP", + "time": "03:57:38", + "message": "Peer 10.10.7.134 DOWN (Hold Timer Expired)", + "facility": 1 + }, + "yang_model": "openconfig-bgp", + "os": "netiron", + "facility": 1, + "severity": 5, + "error": "BGP_NEIGHBOR_STATE_CHANGED", + "host": "HOSTNAME", + "ip": "127.0.0.1", + "timestamp": 1526706119764 +} \ No newline at end of file diff --git a/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_with_vrf/syslog.msg b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_with_vrf/syslog.msg new file mode 100644 index 00000000..9a57bbfd --- /dev/null +++ b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_with_vrf/syslog.msg @@ -0,0 +1 @@ +<13>May 19 05:01:59 HOSTNAME BGP: Peer (VRF: default-vrf) 2607:a999:4999::1 DOWN (Rcv Notification:Hold Timer Expired) \ No newline at end of file diff --git a/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_with_vrf/yang.json b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_with_vrf/yang.json new file mode 100644 index 00000000..b94d03a3 --- /dev/null +++ b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/established_to_idle_with_vrf/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "bgp": { + "neighbors": { + "neighbor": { + "2607:a999:4999::1": { + "state": { + "session-state": "IDLE", + "session-state-change-event": "Rcv Notification:Hold Timer Expired" + } + } + } + } + } + }, + "message_details": { + "severity": 5, + "date": "May 19", + "pri": "13", + "host": "HOSTNAME", + "tag": "BGP", + "time": "05:01:59", + "message": "Peer (VRF: default-vrf) 2607:a999:4999::1 DOWN (Rcv Notification:Hold Timer Expired)", + "facility": 1 + }, + "yang_model": "openconfig-bgp", + "os": "netiron", + "facility": 1, + "severity": 5, + "error": "BGP_NEIGHBOR_STATE_CHANGED", + "host": "HOSTNAME", + "ip": "127.0.0.1", + "timestamp": 1526706119764 +} \ No newline at end of file diff --git a/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_no_vrf/syslog.msg b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_no_vrf/syslog.msg new file mode 100644 index 00000000..b593d589 --- /dev/null +++ b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_no_vrf/syslog.msg @@ -0,0 +1 @@ +<13>May 19 03:57:43 HOSTNAME BGP: Peer 10.10.7.137 UP (ESTABLISHED) \ No newline at end of file diff --git a/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_no_vrf/yang.json b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_no_vrf/yang.json new file mode 100644 index 00000000..04e0b425 --- /dev/null +++ b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_no_vrf/yang.json @@ -0,0 +1,34 @@ +{ +"yang_message": { + "bgp": { + "neighbors": { + "neighbor": { + "10.10.7.137": { + "state": { + "session-state": "ESTABLISHED", + "session-state-change-event": "ESTABLISHED" + } + } + } + } + } + }, + "message_details": { + "severity": 5, + "date": "May 19", + "pri": "13", + "host": "HOSTNAME", + "tag": "BGP", + "time": "03:57:43", + "message": "Peer 10.10.7.137 UP (ESTABLISHED)", + "facility": 1 + }, + "yang_model": "openconfig-bgp", + "os": "netiron", + "facility": 1, + "severity": 5, + "error": "BGP_NEIGHBOR_STATE_CHANGED", + "host": "HOSTNAME", + "ip": "127.0.0.1", + "timestamp": 1526701532 +} \ No newline at end of file diff --git a/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_with_vrf/syslog.msg b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_with_vrf/syslog.msg new file mode 100644 index 00000000..32bc4bbe --- /dev/null +++ b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_with_vrf/syslog.msg @@ -0,0 +1 @@ +<13>May 22 09:20:00 HOSTNAME BGP: Peer (VRF: VRF-NAME) 10.65.5.25 UP (ESTABLISHED) diff --git a/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_with_vrf/yang.json b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_with_vrf/yang.json new file mode 100644 index 00000000..a90dc5e4 --- /dev/null +++ b/tests/config/netiron/BGP_NEIGHBOR_STATE_CHANGED/idle_to_established_with_vrf/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "bgp": { + "neighbors": { + "neighbor": { + "10.65.5.25": { + "state": { + "session-state-change-event": "ESTABLISHED", + "session-state": "ESTABLISHED" + } + } + } + } + } + }, + "message_details": { + "date": "May 22", + "time": "09:20:00", + "host": "HOSTNAME", + "tag": "BGP", + "pri": "13", + "message": "Peer (VRF: VRF-NAME) 10.65.5.25 UP (ESTABLISHED)", + "facility": 1, + "severity": 5 + }, + "yang_model": "openconfig-bgp", + "os": "netiron", + "facility": 1, + "severity": 5, + "error": "BGP_NEIGHBOR_STATE_CHANGED", + "host": "HOSTNAME", + "ip": "127.0.0.1", + "timestamp": 1526980800 +} \ No newline at end of file From 41fd208cb84a2f1671e43520d36e4677a59dfb3f Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Wed, 23 May 2018 13:49:10 +0200 Subject: [PATCH 16/45] CONFIGURED_FROM.yml --- napalm_logs/config/ios/CONFIGURED_FROM.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 napalm_logs/config/ios/CONFIGURED_FROM.yml diff --git a/napalm_logs/config/ios/CONFIGURED_FROM.yml b/napalm_logs/config/ios/CONFIGURED_FROM.yml new file mode 100644 index 00000000..94340141 --- /dev/null +++ b/napalm_logs/config/ios/CONFIGURED_FROM.yml @@ -0,0 +1,15 @@ +# <189>24: test-ztp: May 23 2018 12:09:09 MEST: %SYS-5-CONFIG_I: Configured from tftp://10.83.21.224//cisco-bootstrap by console +# <189>28: test-ztp: May 23 2018 10:19:33 MEST: %SYS-5-CONFIG_I: Configured from console by admin on vty0 (10.31.0.24) + +messages: + - error: CONFIGURED_FROM + tag: SYS-5-CONFIG_I + values: + configured_from: (\w+[\W\w]+) + configured_by: (\w+[\s\.\(\)\d\w]+) + line: 'Configured from {configured_from} by {configured_by}' + model: NO_MODEL + mapping: + variables: + system//logging//console//config//from: configured_from + system//logging//console//config//by: configured_by \ No newline at end of file From fdb3072e6516be01c76ddf8b34bca2d907c761e0 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Wed, 23 May 2018 14:10:53 +0200 Subject: [PATCH 17/45] Tests for CONFIGURED_FROM added --- napalm_logs/config/ios/CONFIGURED_FROM.yml | 5 ++--- tests/config/ios/CONFIGURED_FROM/bootstrap/syslog.msg | 1 + tests/config/ios/CONFIGURED_FROM/default/syslog.msg | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 tests/config/ios/CONFIGURED_FROM/bootstrap/syslog.msg create mode 100644 tests/config/ios/CONFIGURED_FROM/default/syslog.msg diff --git a/napalm_logs/config/ios/CONFIGURED_FROM.yml b/napalm_logs/config/ios/CONFIGURED_FROM.yml index 94340141..bb6af7e4 100644 --- a/napalm_logs/config/ios/CONFIGURED_FROM.yml +++ b/napalm_logs/config/ios/CONFIGURED_FROM.yml @@ -1,6 +1,5 @@ -# <189>24: test-ztp: May 23 2018 12:09:09 MEST: %SYS-5-CONFIG_I: Configured from tftp://10.83.21.224//cisco-bootstrap by console -# <189>28: test-ztp: May 23 2018 10:19:33 MEST: %SYS-5-CONFIG_I: Configured from console by admin on vty0 (10.31.0.24) - +# <189>24: test-ztp: May 23 15:49:32.302: %SYS-5-CONFIG_I: Configured from tftp://10.83.21.224//cisco-bootstrap by console +# <189>30: test-ztp: May 23 13:56:15.055: %SYS-5-CONFIG_I: Configured from console by admin on vty0 (10.31.0.24) messages: - error: CONFIGURED_FROM tag: SYS-5-CONFIG_I diff --git a/tests/config/ios/CONFIGURED_FROM/bootstrap/syslog.msg b/tests/config/ios/CONFIGURED_FROM/bootstrap/syslog.msg new file mode 100644 index 00000000..7346f6d9 --- /dev/null +++ b/tests/config/ios/CONFIGURED_FROM/bootstrap/syslog.msg @@ -0,0 +1 @@ +<189>24: test-ztp: May 23 15:49:32.302: %SYS-5-CONFIG_I: Configured from tftp://10.83.21.224//cisco-bootstrap by console \ No newline at end of file diff --git a/tests/config/ios/CONFIGURED_FROM/default/syslog.msg b/tests/config/ios/CONFIGURED_FROM/default/syslog.msg new file mode 100644 index 00000000..49ddacf9 --- /dev/null +++ b/tests/config/ios/CONFIGURED_FROM/default/syslog.msg @@ -0,0 +1 @@ +<189>30: test-ztp: May 23 13:56:15.055: %SYS-5-CONFIG_I: Configured from console by admin on vty0 (10.31.0.24) \ No newline at end of file From 48feb78a93137b26ee1c735020ef9286b9f32ed9 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Wed, 23 May 2018 15:18:52 +0200 Subject: [PATCH 18/45] key names changed --- napalm_logs/config/ios/CONFIGURED_FROM.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/napalm_logs/config/ios/CONFIGURED_FROM.yml b/napalm_logs/config/ios/CONFIGURED_FROM.yml index bb6af7e4..d83e3be6 100644 --- a/napalm_logs/config/ios/CONFIGURED_FROM.yml +++ b/napalm_logs/config/ios/CONFIGURED_FROM.yml @@ -4,11 +4,12 @@ messages: - error: CONFIGURED_FROM tag: SYS-5-CONFIG_I values: - configured_from: (\w+[\W\w]+) - configured_by: (\w+[\s\.\(\)\d\w]+) - line: 'Configured from {configured_from} by {configured_by}' + configuredfrom: (\w+[\W\w]+) + configuredby: (\w+[\s\.\(\)\d\w]+) + line: 'Configured from {configuredfrom} by {configuredby}' model: NO_MODEL mapping: variables: - system//logging//console//config//from: configured_from - system//logging//console//config//by: configured_by \ No newline at end of file + system//logging//console//config//from: configuredfrom + system//logging//console//config//by: configuredby + static: {} From 6fd69f85b6278cb6f193a539e0d3d3da23f64c68 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Wed, 23 May 2018 15:35:15 +0200 Subject: [PATCH 19/45] yang.json files added for ios: CONFIGURED_FROM tests --- .../ios/CONFIGURED_FROM/bootstrap/yang.json | 34 +++++++++++++++++++ .../ios/CONFIGURED_FROM/default/yang.json | 34 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/config/ios/CONFIGURED_FROM/bootstrap/yang.json create mode 100644 tests/config/ios/CONFIGURED_FROM/default/yang.json diff --git a/tests/config/ios/CONFIGURED_FROM/bootstrap/yang.json b/tests/config/ios/CONFIGURED_FROM/bootstrap/yang.json new file mode 100644 index 00000000..318f56c3 --- /dev/null +++ b/tests/config/ios/CONFIGURED_FROM/bootstrap/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "system": { + "logging": { + "console": { + "config": { + "from": "tftp://10.83.21.224//cisco-bootstrap", + "by": "console" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "messageId": "24", + "pri": "189", + "host": "test-ztp", + "tag": "SYS-5-CONFIG_I", + "time": "15:49:32", + "date": "May 23", + "message": "Configured from tftp://10.83.21.224//cisco-bootstrap by console", + "milliseconds": "302" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "CONFIGURED_FROM", + "host": "test-ztp", + "yang_model": "NO_MODEL", + "timestamp": 1527090572, + "os": "ios", + "severity": 5 +} \ No newline at end of file diff --git a/tests/config/ios/CONFIGURED_FROM/default/yang.json b/tests/config/ios/CONFIGURED_FROM/default/yang.json new file mode 100644 index 00000000..aa34da10 --- /dev/null +++ b/tests/config/ios/CONFIGURED_FROM/default/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "system": { + "logging": { + "console": { + "config": { + "from": "console", + "by": "admin on vty0 (10.31.0.24)" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "messageId": "30", + "pri": "189", + "host": "test-ztp", + "tag": "SYS-5-CONFIG_I", + "time": "13:56:15", + "date": "May 23", + "message": "Configured from console by admin on vty0 (10.31.0.24)", + "milliseconds": "055" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "CONFIGURED_FROM", + "host": "test-ztp", + "yang_model": "NO_MODEL", + "timestamp": 1527083775, + "os": "ios", + "severity": 5 +} \ No newline at end of file From d244b5379431d32e505a42d318d8c56b47106ddc Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Sat, 26 May 2018 07:51:11 +0200 Subject: [PATCH 20/45] Model changed to USER_EXIT_CONFIG_MODE --- ...RED_FROM.yml => USER_EXIT_CONFIG_MODE.yml} | 15 ++++---- .../CONFIGURED_FROM/bootstrap/syslog.msg | 0 .../CONFIGURED_FROM/bootstrap/yang.json | 0 .../CONFIGURED_FROM/default/syslog.msg | 0 .../CONFIGURED_FROM/default/yang.json | 0 .../bootstrap/syslog.msg | 1 + .../USER_EXIT_CONFIG_MODE/bootstrap/yang.json | 35 +++++++++++++++++++ .../USER_EXIT_CONFIG_MODE/default/syslog.msg | 1 + .../USER_EXIT_CONFIG_MODE/default/yang.json | 35 +++++++++++++++++++ 9 files changed, 81 insertions(+), 6 deletions(-) rename napalm_logs/config/ios/{CONFIGURED_FROM.yml => USER_EXIT_CONFIG_MODE.yml} (51%) rename tests/{config/ios => }/CONFIGURED_FROM/bootstrap/syslog.msg (100%) rename tests/{config/ios => }/CONFIGURED_FROM/bootstrap/yang.json (100%) rename tests/{config/ios => }/CONFIGURED_FROM/default/syslog.msg (100%) rename tests/{config/ios => }/CONFIGURED_FROM/default/yang.json (100%) create mode 100644 tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/syslog.msg create mode 100644 tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json create mode 100644 tests/config/ios/USER_EXIT_CONFIG_MODE/default/syslog.msg create mode 100644 tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json diff --git a/napalm_logs/config/ios/CONFIGURED_FROM.yml b/napalm_logs/config/ios/USER_EXIT_CONFIG_MODE.yml similarity index 51% rename from napalm_logs/config/ios/CONFIGURED_FROM.yml rename to napalm_logs/config/ios/USER_EXIT_CONFIG_MODE.yml index d83e3be6..fbbcd043 100644 --- a/napalm_logs/config/ios/CONFIGURED_FROM.yml +++ b/napalm_logs/config/ios/USER_EXIT_CONFIG_MODE.yml @@ -1,15 +1,18 @@ # <189>24: test-ztp: May 23 15:49:32.302: %SYS-5-CONFIG_I: Configured from tftp://10.83.21.224//cisco-bootstrap by console # <189>30: test-ztp: May 23 13:56:15.055: %SYS-5-CONFIG_I: Configured from console by admin on vty0 (10.31.0.24) messages: - - error: CONFIGURED_FROM + - error: USER_EXIT_CONFIG_MODE tag: SYS-5-CONFIG_I values: configuredfrom: (\w+[\W\w]+) - configuredby: (\w+[\s\.\(\)\d\w]+) - line: 'Configured from {configuredfrom} by {configuredby}' + user: (\w+) + placeholder: ( on )? + vty: (.*) + line: 'Configured from {configuredfrom} by {user}{placeholder}{vty}' model: NO_MODEL mapping: + static: + users//user//{user}//action//exit_config_mode: true variables: - system//logging//console//config//from: configuredfrom - system//logging//console//config//by: configuredby - static: {} + users//user//{user}//action//from: configuredfrom + users//user//{user}//action//vty: vty diff --git a/tests/config/ios/CONFIGURED_FROM/bootstrap/syslog.msg b/tests/CONFIGURED_FROM/bootstrap/syslog.msg similarity index 100% rename from tests/config/ios/CONFIGURED_FROM/bootstrap/syslog.msg rename to tests/CONFIGURED_FROM/bootstrap/syslog.msg diff --git a/tests/config/ios/CONFIGURED_FROM/bootstrap/yang.json b/tests/CONFIGURED_FROM/bootstrap/yang.json similarity index 100% rename from tests/config/ios/CONFIGURED_FROM/bootstrap/yang.json rename to tests/CONFIGURED_FROM/bootstrap/yang.json diff --git a/tests/config/ios/CONFIGURED_FROM/default/syslog.msg b/tests/CONFIGURED_FROM/default/syslog.msg similarity index 100% rename from tests/config/ios/CONFIGURED_FROM/default/syslog.msg rename to tests/CONFIGURED_FROM/default/syslog.msg diff --git a/tests/config/ios/CONFIGURED_FROM/default/yang.json b/tests/CONFIGURED_FROM/default/yang.json similarity index 100% rename from tests/config/ios/CONFIGURED_FROM/default/yang.json rename to tests/CONFIGURED_FROM/default/yang.json diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/syslog.msg b/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/syslog.msg new file mode 100644 index 00000000..7346f6d9 --- /dev/null +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/syslog.msg @@ -0,0 +1 @@ +<189>24: test-ztp: May 23 15:49:32.302: %SYS-5-CONFIG_I: Configured from tftp://10.83.21.224//cisco-bootstrap by console \ No newline at end of file diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json b/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json new file mode 100644 index 00000000..5cbd0d72 --- /dev/null +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json @@ -0,0 +1,35 @@ +{ + "yang_message": { + "users": { + "user": { + "console": { + "action": { + "vty": "", + "exit_config_mode": true, + "from": "tftp://10.83.21.224//cisco-bootstrap" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "messageId": "24", + "pri": "189", + "host": "test-ztp", + "tag": "SYS-5-CONFIG_I", + "time": "15:49:32", + "date": "May 23", + "message": "Configured from tftp://10.83.21.224//cisco-bootstrap by console", + "milliseconds": "302" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "USER_EXIT_CONFIG_MODE", + "host": "test-ztp", + "yang_model": "NO_MODEL", + "timestamp": 1527090572, + "os": "ios", + "severity": 5 +} \ No newline at end of file diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/default/syslog.msg b/tests/config/ios/USER_EXIT_CONFIG_MODE/default/syslog.msg new file mode 100644 index 00000000..49ddacf9 --- /dev/null +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/default/syslog.msg @@ -0,0 +1 @@ +<189>30: test-ztp: May 23 13:56:15.055: %SYS-5-CONFIG_I: Configured from console by admin on vty0 (10.31.0.24) \ No newline at end of file diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json b/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json new file mode 100644 index 00000000..c4ddaea9 --- /dev/null +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json @@ -0,0 +1,35 @@ +{ + "yang_message": { + "users": { + "user": { + "admin": { + "action": { + "vty": "vty0 (10.31.0.24)", + "exit_config_mode": true, + "from": "console" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "messageId": "30", + "pri": "189", + "host": "test-ztp", + "tag": "SYS-5-CONFIG_I", + "time": "13:56:15", + "date": "May 23", + "message": "Configured from console by admin on vty0 (10.31.0.24)", + "milliseconds": "055" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "USER_EXIT_CONFIG_MODE", + "host": "test-ztp", + "yang_model": "NO_MODEL", + "timestamp": 1527083775, + "os": "ios", + "severity": 5 +} \ No newline at end of file From 3b5983374f40f43f49a135a846d42cb1bcef5815 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Mon, 28 May 2018 09:12:14 +0200 Subject: [PATCH 21/45] USER_EXIT_CONFIG_MODE for IOS: cleanup --- tests/CONFIGURED_FROM/bootstrap/syslog.msg | 1 - tests/CONFIGURED_FROM/bootstrap/yang.json | 34 ---------------------- tests/CONFIGURED_FROM/default/syslog.msg | 1 - tests/CONFIGURED_FROM/default/yang.json | 34 ---------------------- 4 files changed, 70 deletions(-) delete mode 100644 tests/CONFIGURED_FROM/bootstrap/syslog.msg delete mode 100644 tests/CONFIGURED_FROM/bootstrap/yang.json delete mode 100644 tests/CONFIGURED_FROM/default/syslog.msg delete mode 100644 tests/CONFIGURED_FROM/default/yang.json diff --git a/tests/CONFIGURED_FROM/bootstrap/syslog.msg b/tests/CONFIGURED_FROM/bootstrap/syslog.msg deleted file mode 100644 index 7346f6d9..00000000 --- a/tests/CONFIGURED_FROM/bootstrap/syslog.msg +++ /dev/null @@ -1 +0,0 @@ -<189>24: test-ztp: May 23 15:49:32.302: %SYS-5-CONFIG_I: Configured from tftp://10.83.21.224//cisco-bootstrap by console \ No newline at end of file diff --git a/tests/CONFIGURED_FROM/bootstrap/yang.json b/tests/CONFIGURED_FROM/bootstrap/yang.json deleted file mode 100644 index 318f56c3..00000000 --- a/tests/CONFIGURED_FROM/bootstrap/yang.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "yang_message": { - "system": { - "logging": { - "console": { - "config": { - "from": "tftp://10.83.21.224//cisco-bootstrap", - "by": "console" - } - } - } - } - }, - "message_details": { - "severity": 5, - "facility": 23, - "messageId": "24", - "pri": "189", - "host": "test-ztp", - "tag": "SYS-5-CONFIG_I", - "time": "15:49:32", - "date": "May 23", - "message": "Configured from tftp://10.83.21.224//cisco-bootstrap by console", - "milliseconds": "302" - }, - "facility": 23, - "ip": "127.0.0.1", - "error": "CONFIGURED_FROM", - "host": "test-ztp", - "yang_model": "NO_MODEL", - "timestamp": 1527090572, - "os": "ios", - "severity": 5 -} \ No newline at end of file diff --git a/tests/CONFIGURED_FROM/default/syslog.msg b/tests/CONFIGURED_FROM/default/syslog.msg deleted file mode 100644 index 49ddacf9..00000000 --- a/tests/CONFIGURED_FROM/default/syslog.msg +++ /dev/null @@ -1 +0,0 @@ -<189>30: test-ztp: May 23 13:56:15.055: %SYS-5-CONFIG_I: Configured from console by admin on vty0 (10.31.0.24) \ No newline at end of file diff --git a/tests/CONFIGURED_FROM/default/yang.json b/tests/CONFIGURED_FROM/default/yang.json deleted file mode 100644 index aa34da10..00000000 --- a/tests/CONFIGURED_FROM/default/yang.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "yang_message": { - "system": { - "logging": { - "console": { - "config": { - "from": "console", - "by": "admin on vty0 (10.31.0.24)" - } - } - } - } - }, - "message_details": { - "severity": 5, - "facility": 23, - "messageId": "30", - "pri": "189", - "host": "test-ztp", - "tag": "SYS-5-CONFIG_I", - "time": "13:56:15", - "date": "May 23", - "message": "Configured from console by admin on vty0 (10.31.0.24)", - "milliseconds": "055" - }, - "facility": 23, - "ip": "127.0.0.1", - "error": "CONFIGURED_FROM", - "host": "test-ztp", - "yang_model": "NO_MODEL", - "timestamp": 1527083775, - "os": "ios", - "severity": 5 -} \ No newline at end of file From 0c90d6e987b98070ece0b9c0c5b2b2fc2307af91 Mon Sep 17 00:00:00 2001 From: Luke Overend Date: Tue, 29 May 2018 11:35:51 +0100 Subject: [PATCH 22/45] Allow message to match on multiple OS This fixes #242 Currently when a matching OS is found we do not check remaining OS. This mean that if 2 OS have message prefixes that match each other's logs then you cannot be sure that the message will be forwarded onto the correct `device` process. I have updated so the message is now forwarded onto all matching os, not just the first match. To allow the tests to work correctly I have removed `'send_raw': True` from the config test. --- napalm_logs/server.py | 169 +++++++++++++++++++++++------------------- tests/test_config.py | 2 +- 2 files changed, 92 insertions(+), 79 deletions(-) diff --git a/napalm_logs/server.py b/napalm_logs/server.py index cbc29e22..d6fc9c0c 100644 --- a/napalm_logs/server.py +++ b/napalm_logs/server.py @@ -130,58 +130,70 @@ def _compile_prefixes(self): # log.debug('Compiled prefixes') # log.debug(self.compiled_prefixes) + def _identify_prefix(self, msg, data): + ''' + Check the message again each OS prefix and if matched return the + message dict + ''' + prefix_id = -1 + for prefix in data: + msg_dict = {} + prefix_id += 1 + match = None + if '__python_fun__' in prefix: + log.debug('Trying to match using the %s custom python profiler', prefix['__python_mod__']) + try: + match = prefix['__python_fun__'](msg) + except Exception: + log.error('Exception while parsing %s with the %s python profiler', + msg, prefix['__python_mod__'], exc_info=True) + else: + log.debug('Matching using YAML-defined profiler:') + log.debug(prefix['raw_prefix']) + match = prefix['prefix'].search(msg) + if not match: + log.debug('Match not found') + continue + if '__python_fun__' in prefix: + log.debug('%s matched using the custom python profiler %s', msg, prefix['__python_mod__']) + msg_dict = match # the output as-is from the custom function + else: + positions = prefix.get('prefix_positions', {}) + values = prefix.get('values') + msg_dict = {} + for key in values.keys(): + msg_dict[key] = match.group(positions.get(key)) + # Remove whitespace from the start or end of the message + msg_dict['__prefix_id__'] = prefix_id + msg_dict['message'] = msg_dict['message'].strip() + + # The pri has to be an int as it is retrived using regex '\<(\d+)\>' + if 'pri' in msg_dict: + msg_dict['facility'] = int(int(msg_dict['pri']) / 8) + msg_dict['severity'] = int(int(msg_dict['pri']) - (msg_dict['facility'] * 8)) + return msg_dict + def _identify_os(self, msg): ''' Using the prefix of the syslog message, we are able to identify the operating system and then continue parsing. ''' - ret = {} + ret = [] for dev_os, data in self.compiled_prefixes.items(): # TODO Should we prevent attepmting to determine the OS for the blacklisted? # [mircea] I think its good from a logging perspective to know at least that # that the server found the matching and it tells that it won't be processed # further. Later, we could potentially add an option to control this. log.debug('Matching under %s', dev_os) - prefix_id = -1 - for prefix in data: - prefix_id += 1 - match = None - if '__python_fun__' in prefix: - log.debug('Trying to match using the %s custom python profiler', prefix['__python_mod__']) - try: - match = prefix['__python_fun__'](msg) - except Exception: - log.error('Exception while parsing %s with the %s python profiler', - msg, prefix['__python_mod__'], exc_info=True) - else: - log.debug('Matching using YAML-defined profiler:') - log.debug(prefix['raw_prefix']) - match = prefix['prefix'].search(msg) - if not match: - log.debug('Match not found') - continue - if '__python_fun__' in prefix: - log.debug('%s matched using the custom python profiler %s', msg, prefix['__python_mod__']) - ret = match # the output as-is from the custom function - else: - positions = prefix.get('prefix_positions', {}) - values = prefix.get('values') - ret = {} - for key in values.keys(): - ret[key] = match.group(positions.get(key)) - # Remove whitespace from the start or end of the message - ret['__prefix_id__'] = prefix_id - ret['message'] = ret['message'].strip() - - # The pri has to be an int as it is retrived using regex '\<(\d+)\>' - if 'pri' in ret: - ret['facility'] = int(int(ret['pri']) / 8) - ret['severity'] = int(int(ret['pri']) - (ret['facility'] * 8)) - # TODO Should we stop searching and just return, or should we return all matches OS? - return dev_os, ret - log.debug('No prefix matched under %s', dev_os) - log.debug('No OS matched for: %s', msg) - return '', ret + msg_dict = self._identify_prefix(msg, data) + if msg_dict: + log.debug('Adding %s to list of matched OS', dev_os) + ret.append((dev_os, msg_dict)) + else: + log.debug('No match found for %s', dev_os) + if not ret: + log.debug('Not matched any OS') + return ret def start(self): ''' @@ -241,47 +253,48 @@ def start(self): msg = msg.encode('utf-8') log.debug('[%s] Dequeued message from %s: %s', address, msg, time.time()) napalm_logs_server_messages_received.inc() - dev_os, msg_dict = self._identify_os(msg) + os_list = self._identify_os(msg) - if dev_os and dev_os in self.started_os_proc: - # Identified the OS and the corresponding process is started. - # Then send the message in the right queue - log.debug('Identified OS: %s', dev_os) - log.debug('Queueing message to %s', dev_os) - if six.PY3: - dev_os = bytes(dev_os, 'utf-8') - self.pub.send_multipart([dev_os, - umsgpack.packb((msg_dict, address))]) - # self.os_pipes[dev_os].send((msg_dict, address)) - napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os).inc() - napalm_logs_server_messages_device_queued.labels(device_os=dev_os).inc() + for dev_os, msg_dict in os_list: + if dev_os and dev_os in self.started_os_proc: + # Identified the OS and the corresponding process is started. + # Then send the message in the right queue + log.debug('Identified OS: %s', dev_os) + log.debug('Queueing message to %s', dev_os) + if six.PY3: + dev_os = bytes(dev_os, 'utf-8') + self.pub.send_multipart([dev_os, + umsgpack.packb((msg_dict, address))]) + # self.os_pipes[dev_os].send((msg_dict, address)) + napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os).inc() + napalm_logs_server_messages_device_queued.labels(device_os=dev_os).inc() - elif dev_os and dev_os not in self.started_os_proc: - # Identified the OS, but the corresponding process does not seem to be started. - log.info('Unable to queue the message to %s. Is the sub-process started?', dev_os) - napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os).inc() - napalm_logs_server_messages_failed_device_queuing.labels(device_os=dev_os).inc() + elif dev_os and dev_os not in self.started_os_proc: + # Identified the OS, but the corresponding process does not seem to be started. + log.info('Unable to queue the message to %s. Is the sub-process started?', dev_os) + napalm_logs_server_messages_with_identified_os.labels(device_os=dev_os).inc() + napalm_logs_server_messages_failed_device_queuing.labels(device_os=dev_os).inc() - elif not dev_os and self.opts['_server_send_unknown']: - # OS not identified, but the user requested to publish the message as-is - log.debug('Unable to identify the OS, sending directly to the publishers') - to_publish = { - 'ip': address, - 'host': 'unknown', - 'timestamp': int(time.time()), - 'message_details': msg_dict, - 'os': UNKNOWN_DEVICE_NAME, - 'error': 'UNKNOWN', - 'model_name': 'unknown' - } - self.publisher_pub.send(umsgpack.packb(to_publish)) - napalm_logs_server_messages_unknown_queued.inc() - napalm_logs_server_messages_without_identified_os.inc() + elif not dev_os and self.opts['_server_send_unknown']: + # OS not identified, but the user requested to publish the message as-is + log.debug('Unable to identify the OS, sending directly to the publishers') + to_publish = { + 'ip': address, + 'host': 'unknown', + 'timestamp': int(time.time()), + 'message_details': msg_dict, + 'os': UNKNOWN_DEVICE_NAME, + 'error': 'UNKNOWN', + 'model_name': 'unknown' + } + self.publisher_pub.send(umsgpack.packb(to_publish)) + napalm_logs_server_messages_unknown_queued.inc() + napalm_logs_server_messages_without_identified_os.inc() - elif not dev_os and not self.opts['_server_send_unknown']: - # OS not identified and we are told to do nothing - log.debug('Unable to identify the OS') - napalm_logs_server_messages_without_identified_os.inc() + elif not dev_os and not self.opts['_server_send_unknown']: + # OS not identified and we are told to do nothing + log.debug('Unable to identify the OS') + napalm_logs_server_messages_without_identified_os.inc() def stop(self): log.info('Stopping server process') diff --git a/tests/test_config.py b/tests/test_config.py index 1dd6e4e4..be3449fc 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -47,7 +47,7 @@ def startup_proc(): NL_BASE = NapalmLogs(disable_security=True, address=NAPALM_LOGS_TEST_ADDR, port=NAPALM_LOGS_TEST_PORT, - publisher=[{'zmq': {'send_raw': True, 'send_unknown': True}}], + publisher=[{'zmq': {'send_unknown': True}}], listener=[{'udp': {}}], publish_address=NAPALM_LOGS_TEST_PUB_ADDR, publish_port=NAPALM_LOGS_TEST_PUB_PORT, From 15adfe302c0cf3f38ff54930303305021ec8b3cd Mon Sep 17 00:00:00 2001 From: Luke Overend Date: Tue, 29 May 2018 15:29:21 +0100 Subject: [PATCH 23/45] Remove unused variable `err` This fixes the following: ``` =================================== FAILURES =================================== _______________________________________ _______________________________________ napalm_logs/listener/kafka.py:17:1: W0612 local variable 'err' is assigned to but never used [pyflakes] _______________________________________ _______________________________________ napalm_logs/listener/zeromq.py:16:1: W0612 local variable 'err' is assigned to but never used [pyflakes] _______________________________________ _______________________________________ napalm_logs/transport/kafka.py:15:1: W0612 local variable 'err' is assigned to but never used [pyflakes] ``` --- napalm_logs/listener/kafka.py | 2 +- napalm_logs/listener/zeromq.py | 2 +- napalm_logs/transport/kafka.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/napalm_logs/listener/kafka.py b/napalm_logs/listener/kafka.py index 6ad9d2f8..fcd81cbe 100644 --- a/napalm_logs/listener/kafka.py +++ b/napalm_logs/listener/kafka.py @@ -14,7 +14,7 @@ try: import kafka HAS_KAFKA = True -except ImportError as err: +except ImportError: HAS_KAFKA = False # Import napalm-logs pkgs diff --git a/napalm_logs/listener/zeromq.py b/napalm_logs/listener/zeromq.py index 9f235423..ddb7cb53 100644 --- a/napalm_logs/listener/zeromq.py +++ b/napalm_logs/listener/zeromq.py @@ -13,7 +13,7 @@ try: import zmq HAS_ZMQ = True -except ImportError as err: +except ImportError: HAS_ZMQ = False # Import napalm-logs pkgs diff --git a/napalm_logs/transport/kafka.py b/napalm_logs/transport/kafka.py index 6a40f8d8..f9614ecd 100644 --- a/napalm_logs/transport/kafka.py +++ b/napalm_logs/transport/kafka.py @@ -12,7 +12,7 @@ try: import kafka HAS_KAFKA = True -except ImportError as err: +except ImportError: HAS_KAFKA = False # Import napalm-logs pkgs From 83e50fc61ee2f7c638aea1af19ae7e832cb553c3 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 31 May 2018 15:04:34 +0200 Subject: [PATCH 24/45] IOS timezone added --- napalm_logs/config/ios/init.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/napalm_logs/config/ios/init.yml b/napalm_logs/config/ios/init.yml index 94047879..99f238c7 100644 --- a/napalm_logs/config/ios/init.yml +++ b/napalm_logs/config/ios/init.yml @@ -9,10 +9,11 @@ prefixes: date: \*?(\w+\s+\d+) time: (\d\d:\d\d:\d\d) milliseconds: (\d\d\d) + timeZone: \s?(\w+)? tag: ([\w-]+) # some messages contain a process number (eg. OSPF related messages) processId: (\d+) - line: '{messageId}: {host}: {date} {time}.{milliseconds}: %{tag}: Process {processId}, ' + line: '{messageId}: {host}: {date} {time}.{milliseconds}{timeZone}: %{tag}: Process {processId}, ' - time_format: "%b %d %H:%M:%S" values: messageId: (\d+) @@ -20,5 +21,6 @@ prefixes: date: \*?(\w+\s+\d+) time: (\d\d:\d\d:\d\d) milliseconds: (\d+) + timeZone: \s?(\w+)? tag: ([\w-]+) - line: '{messageId}: {host}: {date} {time}.{milliseconds}: %{tag}: ' + line: '{messageId}: {host}: {date} {time}.{milliseconds}{timeZone}: %{tag}: ' From 0a471ac93c889dc57090096e234a3d83fe796b6f Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 31 May 2018 15:29:27 +0200 Subject: [PATCH 25/45] Testfile for IOS timezone added --- .../message_with_timezone/syslog.msg | 1 + .../message_with_timezone/yang.json | 34 ++++++++++++++++++ .../message_with_timezone/syslog.msg | 1 + .../message_with_timezone/yang.json | 36 +++++++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 tests/config/ios/INTERFACE_DOWN/message_with_timezone/syslog.msg create mode 100644 tests/config/ios/INTERFACE_DOWN/message_with_timezone/yang.json create mode 100644 tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/syslog.msg create mode 100644 tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/yang.json diff --git a/tests/config/ios/INTERFACE_DOWN/message_with_timezone/syslog.msg b/tests/config/ios/INTERFACE_DOWN/message_with_timezone/syslog.msg new file mode 100644 index 00000000..e47e008c --- /dev/null +++ b/tests/config/ios/INTERFACE_DOWN/message_with_timezone/syslog.msg @@ -0,0 +1 @@ +<189>1179: NetAuto_CSRv-03: May 31 15:25:53.743 MEST: %LINK-5-CHANGED: Interface GigabitEthernet2, changed state to administratively down from 10.83.75.201 \ No newline at end of file diff --git a/tests/config/ios/INTERFACE_DOWN/message_with_timezone/yang.json b/tests/config/ios/INTERFACE_DOWN/message_with_timezone/yang.json new file mode 100644 index 00000000..5a7b321d --- /dev/null +++ b/tests/config/ios/INTERFACE_DOWN/message_with_timezone/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "interfaces": { + "interface": { + "GigabitEthernet2": { + "state": { + "admin_status": "DOWN" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "time": "15:25:53", + "pri": "189", + "host": "NetAuto_CSRv-03", + "tag": "LINK-5-CHANGED", + "messageId": "1179", + "date": "May 31", + "timeZone": "MEST", + "message": "Interface GigabitEthernet2, changed state to administratively down from 10.83.75.201", + "milliseconds": "743" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "INTERFACE_DOWN", + "host": "NetAuto_CSRv-03", + "yang_model": "openconfig-interfaces", + "timestamp": 1527780353, + "os": "ios", + "severity": 5 +} \ No newline at end of file diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/syslog.msg b/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/syslog.msg new file mode 100644 index 00000000..8218d14d --- /dev/null +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/syslog.msg @@ -0,0 +1 @@ +<189>27: test-ztp: May 31 15:54:54.567 MEST: %SYS-5-CONFIG_I: Configured from console by admin on console \ No newline at end of file diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/yang.json b/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/yang.json new file mode 100644 index 00000000..3af3aaf0 --- /dev/null +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/yang.json @@ -0,0 +1,36 @@ +{ + "yang_message": { + "users": { + "user": { + "admin": { + "action": { + "vty": "console", + "exit_config_mode": true, + "from": "console" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "time": "15:54:54", + "pri": "189", + "host": "test-ztp", + "tag": "SYS-5-CONFIG_I", + "messageId": "27", + "date": "May 31", + "timeZone": "MEST", + "message": "Configured from console by admin on console", + "milliseconds": "567" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "USER_EXIT_CONFIG_MODE", + "host": "test-ztp", + "yang_model": "NO_MODEL", + "timestamp": 1527782094, + "os": "ios", + "severity": 5 +} \ No newline at end of file From 49c9b02c3267e9e521e35d1e73d3906ff5f8a8ff Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Fri, 1 Jun 2018 15:42:09 +0200 Subject: [PATCH 26/45] LINE_PROTOCOL for IOS added --- napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml | 14 ++++++++++++++ napalm_logs/config/ios/LINE_PROTOCOL_UP.yml | 14 ++++++++++++++ .../ios/LINE_PROTOCOL_DOWN/default/syslog.msg | 1 + .../config/ios/LINE_PROTOCOL_UP/default/syslog.msg | 1 + 4 files changed, 30 insertions(+) create mode 100644 napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml create mode 100644 napalm_logs/config/ios/LINE_PROTOCOL_UP.yml create mode 100644 tests/config/ios/LINE_PROTOCOL_DOWN/default/syslog.msg create mode 100644 tests/config/ios/LINE_PROTOCOL_UP/default/syslog.msg diff --git a/napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml b/napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml new file mode 100644 index 00000000..2ed2bc11 --- /dev/null +++ b/napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml @@ -0,0 +1,14 @@ +# Interface GigabitEthernet2, changed state to administratively down + +messages: + - error: INTERFACE_DOWN + tag: LINEPROTO-5-UPDOWN + values: + interface: (\w+[\.\-\d\/\w+]+) + adminStatus|upper: (\w+) + line: 'Line protocol on Interface {interface}, changed state to down' + model: openconfig-interfaces + mapping: + variables: {} + static: + interfaces//interface//{interface}//state//oper_status: DOWN diff --git a/napalm_logs/config/ios/LINE_PROTOCOL_UP.yml b/napalm_logs/config/ios/LINE_PROTOCOL_UP.yml new file mode 100644 index 00000000..1aae13f0 --- /dev/null +++ b/napalm_logs/config/ios/LINE_PROTOCOL_UP.yml @@ -0,0 +1,14 @@ +# Interface GigabitEthernet2, changed state to administratively down + +messages: + - error: INTERFACE_UP + tag: LINEPROTO-5-UPDOWN + values: + interface: (\w+[\.\-\d\/\w+]+) + adminStatus|upper: (\w+) + line: 'Line protocol on Interface {interface}, changed state to up' + model: openconfig-interfaces + mapping: + variables: {} + static: + interfaces//interface//{interface}//state//oper_status: UP diff --git a/tests/config/ios/LINE_PROTOCOL_DOWN/default/syslog.msg b/tests/config/ios/LINE_PROTOCOL_DOWN/default/syslog.msg new file mode 100644 index 00000000..67ef8ea3 --- /dev/null +++ b/tests/config/ios/LINE_PROTOCOL_DOWN/default/syslog.msg @@ -0,0 +1 @@ +<189>1432: test-switch1: Jun 1 12:50:26.172: %LINEPROTO-5-UPDOWN: Line protocol on Interface FastEthernet0/2, changed state to down \ No newline at end of file diff --git a/tests/config/ios/LINE_PROTOCOL_UP/default/syslog.msg b/tests/config/ios/LINE_PROTOCOL_UP/default/syslog.msg new file mode 100644 index 00000000..db674941 --- /dev/null +++ b/tests/config/ios/LINE_PROTOCOL_UP/default/syslog.msg @@ -0,0 +1 @@ +<189>1440: test-switch1: Jun 1 13:35:45.562: %LINEPROTO-5-UPDOWN: Line protocol on Interface FastEthernet0/2, changed state to up \ No newline at end of file From eabae0a517086766f8bafa85da040229a90769a9 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Fri, 1 Jun 2018 15:59:43 +0200 Subject: [PATCH 27/45] LINE_PROTOCOL for IOS: corrections --- napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml | 6 ++---- napalm_logs/config/ios/LINE_PROTOCOL_UP.yml | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml b/napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml index 2ed2bc11..df778877 100644 --- a/napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml +++ b/napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml @@ -1,14 +1,12 @@ -# Interface GigabitEthernet2, changed state to administratively down - +# Line protocol on Interface FastEthernet0/2, changed state to down messages: - error: INTERFACE_DOWN tag: LINEPROTO-5-UPDOWN values: interface: (\w+[\.\-\d\/\w+]+) - adminStatus|upper: (\w+) line: 'Line protocol on Interface {interface}, changed state to down' model: openconfig-interfaces mapping: - variables: {} static: interfaces//interface//{interface}//state//oper_status: DOWN + variables: {} diff --git a/napalm_logs/config/ios/LINE_PROTOCOL_UP.yml b/napalm_logs/config/ios/LINE_PROTOCOL_UP.yml index 1aae13f0..25fe4106 100644 --- a/napalm_logs/config/ios/LINE_PROTOCOL_UP.yml +++ b/napalm_logs/config/ios/LINE_PROTOCOL_UP.yml @@ -1,14 +1,12 @@ -# Interface GigabitEthernet2, changed state to administratively down - +# Line protocol on Interface FastEthernet0/2, changed state to up messages: - error: INTERFACE_UP tag: LINEPROTO-5-UPDOWN values: interface: (\w+[\.\-\d\/\w+]+) - adminStatus|upper: (\w+) line: 'Line protocol on Interface {interface}, changed state to up' model: openconfig-interfaces mapping: - variables: {} static: interfaces//interface//{interface}//state//oper_status: UP + variables: {} From 55d02a083f433b39701ed51b76d1ac72fb0a4d23 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Fri, 1 Jun 2018 16:05:02 +0200 Subject: [PATCH 28/45] LINE_PROTOCOL for IOS: yang.json added --- .../ios/LINE_PROTOCOL_DOWN/default/yang.json | 33 +++++++++++++++++++ .../ios/LINE_PROTOCOL_UP/default/yang.json | 33 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 tests/config/ios/LINE_PROTOCOL_DOWN/default/yang.json create mode 100644 tests/config/ios/LINE_PROTOCOL_UP/default/yang.json diff --git a/tests/config/ios/LINE_PROTOCOL_DOWN/default/yang.json b/tests/config/ios/LINE_PROTOCOL_DOWN/default/yang.json new file mode 100644 index 00000000..b3bd4eac --- /dev/null +++ b/tests/config/ios/LINE_PROTOCOL_DOWN/default/yang.json @@ -0,0 +1,33 @@ +{ + "yang_message": { + "interfaces": { + "interface": { + "FastEthernet0/2": { + "state": { + "oper_status": "DOWN" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "messageId": "1432", + "pri": "189", + "host": "test-switch1", + "tag": "LINEPROTO-5-UPDOWN", + "time": "12:50:26", + "date": "Jun 1", + "message": "Line protocol on Interface FastEthernet0/2, changed state to down", + "milliseconds": "172" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "INTERFACE_DOWN", + "host": "test-switch1", + "yang_model": "openconfig-interfaces", + "timestamp": 1527857426, + "os": "ios", + "severity": 5 +} diff --git a/tests/config/ios/LINE_PROTOCOL_UP/default/yang.json b/tests/config/ios/LINE_PROTOCOL_UP/default/yang.json new file mode 100644 index 00000000..894284f7 --- /dev/null +++ b/tests/config/ios/LINE_PROTOCOL_UP/default/yang.json @@ -0,0 +1,33 @@ +{ + "yang_message": { + "interfaces": { + "interface": { + "FastEthernet0/2": { + "state": { + "oper_status": "UP" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "messageId": "1440", + "pri": "189", + "host": "test-switch1", + "tag": "LINEPROTO-5-UPDOWN", + "time": "13:35:45", + "date": "Jun 1", + "message": "Line protocol on Interface FastEthernet0/2, changed state to up", + "milliseconds": "562" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "INTERFACE_UP", + "host": "test-switch1", + "yang_model": "openconfig-interfaces", + "timestamp": 1527860145, + "os": "ios", + "severity": 5 +} From 02ad607e574e5919de6c2395b0b80d1d171cde19 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Mon, 4 Jun 2018 10:59:56 +0200 Subject: [PATCH 29/45] Renaming for INTERFACE_DOWN to INTERFACE_ADMIN_DOWN --- .../config/ios/{INTERFACE_DOWN.yml => INTERFACE_ADMIN_DOWN.yml} | 2 +- .../{INTERFACE_DOWN => INTERFACE_ADMIN_DOWN}/default/syslog.msg | 0 .../{INTERFACE_DOWN => INTERFACE_ADMIN_DOWN}/default/yang.json | 2 +- .../time_synchronized/syslog.msg | 0 .../time_synchronized/yang.json | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) rename napalm_logs/config/ios/{INTERFACE_DOWN.yml => INTERFACE_ADMIN_DOWN.yml} (92%) rename tests/config/ios/{INTERFACE_DOWN => INTERFACE_ADMIN_DOWN}/default/syslog.msg (100%) rename tests/config/ios/{INTERFACE_DOWN => INTERFACE_ADMIN_DOWN}/default/yang.json (95%) rename tests/config/ios/{INTERFACE_DOWN => INTERFACE_ADMIN_DOWN}/time_synchronized/syslog.msg (100%) rename tests/config/ios/{INTERFACE_DOWN => INTERFACE_ADMIN_DOWN}/time_synchronized/yang.json (95%) diff --git a/napalm_logs/config/ios/INTERFACE_DOWN.yml b/napalm_logs/config/ios/INTERFACE_ADMIN_DOWN.yml similarity index 92% rename from napalm_logs/config/ios/INTERFACE_DOWN.yml rename to napalm_logs/config/ios/INTERFACE_ADMIN_DOWN.yml index 853ff29b..c27ee64b 100644 --- a/napalm_logs/config/ios/INTERFACE_DOWN.yml +++ b/napalm_logs/config/ios/INTERFACE_ADMIN_DOWN.yml @@ -1,7 +1,7 @@ # Interface GigabitEthernet2, changed state to administratively down messages: - - error: INTERFACE_DOWN + - error: INTERFACE_ADMIN_DOWN tag: LINK-5-CHANGED values: interface: (\w+[\.\-\d\/\w+]+) diff --git a/tests/config/ios/INTERFACE_DOWN/default/syslog.msg b/tests/config/ios/INTERFACE_ADMIN_DOWN/default/syslog.msg similarity index 100% rename from tests/config/ios/INTERFACE_DOWN/default/syslog.msg rename to tests/config/ios/INTERFACE_ADMIN_DOWN/default/syslog.msg diff --git a/tests/config/ios/INTERFACE_DOWN/default/yang.json b/tests/config/ios/INTERFACE_ADMIN_DOWN/default/yang.json similarity index 95% rename from tests/config/ios/INTERFACE_DOWN/default/yang.json rename to tests/config/ios/INTERFACE_ADMIN_DOWN/default/yang.json index e24d7729..7af9a14e 100644 --- a/tests/config/ios/INTERFACE_DOWN/default/yang.json +++ b/tests/config/ios/INTERFACE_ADMIN_DOWN/default/yang.json @@ -27,7 +27,7 @@ "ip": "127.0.0.1", "host": "router1", "yang_model": "openconfig-interfaces", - "error": "INTERFACE_DOWN", + "error": "INTERFACE_ADMIN_DOWN", "os": "ios", "severity": 5 } diff --git a/tests/config/ios/INTERFACE_DOWN/time_synchronized/syslog.msg b/tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/syslog.msg similarity index 100% rename from tests/config/ios/INTERFACE_DOWN/time_synchronized/syslog.msg rename to tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/syslog.msg diff --git a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json b/tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/yang.json similarity index 95% rename from tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json rename to tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/yang.json index f36caafc..7d8ff552 100644 --- a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json +++ b/tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/yang.json @@ -24,7 +24,7 @@ }, "facility": 23, "ip": "127.0.0.1", - "error": "INTERFACE_DOWN", + "error": "INTERFACE_ADMIN_DOWN", "host": "NetAuto_CSRv-03", "yang_model": "openconfig-interfaces", "timestamp": 1525870167, From a290497a7febc0a3144b555bfdc50fe17c15217b Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Mon, 4 Jun 2018 11:10:03 +0200 Subject: [PATCH 30/45] Oper state INTERFACE_DOWN and INTERFACE_UP added --- .../config/ios/{LINE_PROTOCOL_DOWN.yml => INTERFACE_DOWN.yml} | 0 napalm_logs/config/ios/{LINE_PROTOCOL_UP.yml => INTERFACE_UP.yml} | 0 .../ios/{LINE_PROTOCOL_DOWN => INTERFACE_DOWN}/default/syslog.msg | 0 .../ios/{LINE_PROTOCOL_DOWN => INTERFACE_DOWN}/default/yang.json | 0 .../ios/{LINE_PROTOCOL_UP => INTERFACE_UP}/default/syslog.msg | 0 .../ios/{LINE_PROTOCOL_UP => INTERFACE_UP}/default/yang.json | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename napalm_logs/config/ios/{LINE_PROTOCOL_DOWN.yml => INTERFACE_DOWN.yml} (100%) rename napalm_logs/config/ios/{LINE_PROTOCOL_UP.yml => INTERFACE_UP.yml} (100%) rename tests/config/ios/{LINE_PROTOCOL_DOWN => INTERFACE_DOWN}/default/syslog.msg (100%) rename tests/config/ios/{LINE_PROTOCOL_DOWN => INTERFACE_DOWN}/default/yang.json (100%) rename tests/config/ios/{LINE_PROTOCOL_UP => INTERFACE_UP}/default/syslog.msg (100%) rename tests/config/ios/{LINE_PROTOCOL_UP => INTERFACE_UP}/default/yang.json (100%) diff --git a/napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml b/napalm_logs/config/ios/INTERFACE_DOWN.yml similarity index 100% rename from napalm_logs/config/ios/LINE_PROTOCOL_DOWN.yml rename to napalm_logs/config/ios/INTERFACE_DOWN.yml diff --git a/napalm_logs/config/ios/LINE_PROTOCOL_UP.yml b/napalm_logs/config/ios/INTERFACE_UP.yml similarity index 100% rename from napalm_logs/config/ios/LINE_PROTOCOL_UP.yml rename to napalm_logs/config/ios/INTERFACE_UP.yml diff --git a/tests/config/ios/LINE_PROTOCOL_DOWN/default/syslog.msg b/tests/config/ios/INTERFACE_DOWN/default/syslog.msg similarity index 100% rename from tests/config/ios/LINE_PROTOCOL_DOWN/default/syslog.msg rename to tests/config/ios/INTERFACE_DOWN/default/syslog.msg diff --git a/tests/config/ios/LINE_PROTOCOL_DOWN/default/yang.json b/tests/config/ios/INTERFACE_DOWN/default/yang.json similarity index 100% rename from tests/config/ios/LINE_PROTOCOL_DOWN/default/yang.json rename to tests/config/ios/INTERFACE_DOWN/default/yang.json diff --git a/tests/config/ios/LINE_PROTOCOL_UP/default/syslog.msg b/tests/config/ios/INTERFACE_UP/default/syslog.msg similarity index 100% rename from tests/config/ios/LINE_PROTOCOL_UP/default/syslog.msg rename to tests/config/ios/INTERFACE_UP/default/syslog.msg diff --git a/tests/config/ios/LINE_PROTOCOL_UP/default/yang.json b/tests/config/ios/INTERFACE_UP/default/yang.json similarity index 100% rename from tests/config/ios/LINE_PROTOCOL_UP/default/yang.json rename to tests/config/ios/INTERFACE_UP/default/yang.json From d5f80cbca28021592c0774a91244a18021bfb063 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Fri, 22 Jun 2018 14:47:52 +0200 Subject: [PATCH 31/45] timezone added to yang.json for IOS tests --- tests/config/ios/INTERFACE_DOWN/default/yang.json | 1 + tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json | 1 + tests/config/ios/OSPF_NEIGHBOR_DOWN/init_to_down/yang.json | 1 + tests/config/ios/OSPF_NEIGHBOR_UP/loading_to_full/yang.json | 1 + tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json | 1 + tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json | 1 + 6 files changed, 6 insertions(+) diff --git a/tests/config/ios/INTERFACE_DOWN/default/yang.json b/tests/config/ios/INTERFACE_DOWN/default/yang.json index e24d7729..b1c7fd64 100644 --- a/tests/config/ios/INTERFACE_DOWN/default/yang.json +++ b/tests/config/ios/INTERFACE_DOWN/default/yang.json @@ -17,6 +17,7 @@ "host": "router1", "tag": "LINK-5-CHANGED", "time": "08:30:56", + "timeZone": null, "date": "Nov 14", "messageId": "521", "milliseconds": "699", diff --git a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json index f36caafc..393fa47e 100644 --- a/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json +++ b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json @@ -18,6 +18,7 @@ "host": "NetAuto_CSRv-03", "tag": "LINK-5-CHANGED", "time": "12:49:27", + "timeZone": null, "date": "May 9", "message": "Interface GigabitEthernet2, changed state to administratively down", "milliseconds": "098" diff --git a/tests/config/ios/OSPF_NEIGHBOR_DOWN/init_to_down/yang.json b/tests/config/ios/OSPF_NEIGHBOR_DOWN/init_to_down/yang.json index bcaa2d19..12274c1b 100644 --- a/tests/config/ios/OSPF_NEIGHBOR_DOWN/init_to_down/yang.json +++ b/tests/config/ios/OSPF_NEIGHBOR_DOWN/init_to_down/yang.json @@ -41,6 +41,7 @@ "message_details": { "date": "Mar 15", "time": "10:05:53", + "timeZone": null, "host": "router1", "processId": "1", "tag": "OSPF-5-ADJCHG", diff --git a/tests/config/ios/OSPF_NEIGHBOR_UP/loading_to_full/yang.json b/tests/config/ios/OSPF_NEIGHBOR_UP/loading_to_full/yang.json index a82e27ed..8ef04c2b 100644 --- a/tests/config/ios/OSPF_NEIGHBOR_UP/loading_to_full/yang.json +++ b/tests/config/ios/OSPF_NEIGHBOR_UP/loading_to_full/yang.json @@ -39,6 +39,7 @@ "message_details": { "date": "Mar 15", "time": "10:05:53", + "timeZone": null, "host": "router1", "processId": "1", "tag": "OSPF-5-ADJCHG", diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json b/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json index 5cbd0d72..0dc84163 100644 --- a/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json @@ -20,6 +20,7 @@ "host": "test-ztp", "tag": "SYS-5-CONFIG_I", "time": "15:49:32", + "timeZone": null, "date": "May 23", "message": "Configured from tftp://10.83.21.224//cisco-bootstrap by console", "milliseconds": "302" diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json b/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json index c4ddaea9..09794b11 100644 --- a/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json @@ -20,6 +20,7 @@ "host": "test-ztp", "tag": "SYS-5-CONFIG_I", "time": "13:56:15", + "timeZone": null, "date": "May 23", "message": "Configured from console by admin on vty0 (10.31.0.24)", "milliseconds": "055" From bf18a936f0b9555eaab58efbf7cde999ff3686ae Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 28 Jun 2018 16:32:36 +0200 Subject: [PATCH 32/45] ios: changed oper- and admin-status for INTERFACE_DWON --- .../config/ios/INTERFACE_ADMIN_DOWN.yml | 14 -------- napalm_logs/config/ios/INTERFACE_DOWN.yml | 15 +++++++- .../admin_down}/syslog.msg | 0 .../admin_down}/yang.json | 3 +- .../admin_down_time_synchronized}/syslog.msg | 0 .../admin_down_time_synchronized}/yang.json | 3 +- .../admin_down_with_timezone/syslog.msg | 1 + .../admin_down_with_timezone/yang.json | 34 ++++++++++++++++++ .../config/ios/INTERFACE_UP/default/yang.json | 1 + .../USER_EXIT_CONFIG_MODE/bootstrap/yang.json | 1 + .../USER_EXIT_CONFIG_MODE/default/yang.json | 1 + .../message_with_timezone/syslog.msg | 1 + .../message_with_timezone/yang.json | 36 +++++++++++++++++++ 13 files changed, 93 insertions(+), 17 deletions(-) delete mode 100644 napalm_logs/config/ios/INTERFACE_ADMIN_DOWN.yml rename tests/config/ios/{INTERFACE_ADMIN_DOWN/default => INTERFACE_DOWN/admin_down}/syslog.msg (100%) rename tests/config/ios/{INTERFACE_ADMIN_DOWN/default => INTERFACE_DOWN/admin_down}/yang.json (92%) rename tests/config/ios/{INTERFACE_ADMIN_DOWN/time_synchronized => INTERFACE_DOWN/admin_down_time_synchronized}/syslog.msg (100%) rename tests/config/ios/{INTERFACE_ADMIN_DOWN/time_synchronized => INTERFACE_DOWN/admin_down_time_synchronized}/yang.json (93%) create mode 100644 tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/syslog.msg create mode 100644 tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/yang.json create mode 100644 tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/syslog.msg create mode 100644 tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/yang.json diff --git a/napalm_logs/config/ios/INTERFACE_ADMIN_DOWN.yml b/napalm_logs/config/ios/INTERFACE_ADMIN_DOWN.yml deleted file mode 100644 index c27ee64b..00000000 --- a/napalm_logs/config/ios/INTERFACE_ADMIN_DOWN.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Interface GigabitEthernet2, changed state to administratively down - -messages: - - error: INTERFACE_ADMIN_DOWN - tag: LINK-5-CHANGED - values: - interface: (\w+[\.\-\d\/\w+]+) - adminStatus|upper: (\w+) - line: 'Interface {interface}, changed state to administratively {adminStatus}' - model: openconfig-interfaces - mapping: - variables: - interfaces//interface//{interface}//state//admin_status: adminStatus - static: {} diff --git a/napalm_logs/config/ios/INTERFACE_DOWN.yml b/napalm_logs/config/ios/INTERFACE_DOWN.yml index df778877..ef4cb5e5 100644 --- a/napalm_logs/config/ios/INTERFACE_DOWN.yml +++ b/napalm_logs/config/ios/INTERFACE_DOWN.yml @@ -1,6 +1,19 @@ -# Line protocol on Interface FastEthernet0/2, changed state to down messages: + # Interface GigabitEthernet2, changed state to administratively down - error: INTERFACE_DOWN + tag: LINK-5-CHANGED + values: + interface: (\w+[\.\-\d\/\w+]+) + adminStatus|upper: (\w+) + line: 'Interface {interface}, changed state to administratively {adminStatus}' + model: openconfig-interfaces + mapping: + variables: + interfaces//interface//{interface}//state//admin_status: adminStatus + static: {} + + - error: INTERFACE_DOWN + # Line protocol on Interface FastEthernet0/2, changed state to down tag: LINEPROTO-5-UPDOWN values: interface: (\w+[\.\-\d\/\w+]+) diff --git a/tests/config/ios/INTERFACE_ADMIN_DOWN/default/syslog.msg b/tests/config/ios/INTERFACE_DOWN/admin_down/syslog.msg similarity index 100% rename from tests/config/ios/INTERFACE_ADMIN_DOWN/default/syslog.msg rename to tests/config/ios/INTERFACE_DOWN/admin_down/syslog.msg diff --git a/tests/config/ios/INTERFACE_ADMIN_DOWN/default/yang.json b/tests/config/ios/INTERFACE_DOWN/admin_down/yang.json similarity index 92% rename from tests/config/ios/INTERFACE_ADMIN_DOWN/default/yang.json rename to tests/config/ios/INTERFACE_DOWN/admin_down/yang.json index 7af9a14e..b1c7fd64 100644 --- a/tests/config/ios/INTERFACE_ADMIN_DOWN/default/yang.json +++ b/tests/config/ios/INTERFACE_DOWN/admin_down/yang.json @@ -17,6 +17,7 @@ "host": "router1", "tag": "LINK-5-CHANGED", "time": "08:30:56", + "timeZone": null, "date": "Nov 14", "messageId": "521", "milliseconds": "699", @@ -27,7 +28,7 @@ "ip": "127.0.0.1", "host": "router1", "yang_model": "openconfig-interfaces", - "error": "INTERFACE_ADMIN_DOWN", + "error": "INTERFACE_DOWN", "os": "ios", "severity": 5 } diff --git a/tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/syslog.msg b/tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/syslog.msg similarity index 100% rename from tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/syslog.msg rename to tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/syslog.msg diff --git a/tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/yang.json b/tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/yang.json similarity index 93% rename from tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/yang.json rename to tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/yang.json index 7d8ff552..393fa47e 100644 --- a/tests/config/ios/INTERFACE_ADMIN_DOWN/time_synchronized/yang.json +++ b/tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/yang.json @@ -18,13 +18,14 @@ "host": "NetAuto_CSRv-03", "tag": "LINK-5-CHANGED", "time": "12:49:27", + "timeZone": null, "date": "May 9", "message": "Interface GigabitEthernet2, changed state to administratively down", "milliseconds": "098" }, "facility": 23, "ip": "127.0.0.1", - "error": "INTERFACE_ADMIN_DOWN", + "error": "INTERFACE_DOWN", "host": "NetAuto_CSRv-03", "yang_model": "openconfig-interfaces", "timestamp": 1525870167, diff --git a/tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/syslog.msg b/tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/syslog.msg new file mode 100644 index 00000000..e47e008c --- /dev/null +++ b/tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/syslog.msg @@ -0,0 +1 @@ +<189>1179: NetAuto_CSRv-03: May 31 15:25:53.743 MEST: %LINK-5-CHANGED: Interface GigabitEthernet2, changed state to administratively down from 10.83.75.201 \ No newline at end of file diff --git a/tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/yang.json b/tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/yang.json new file mode 100644 index 00000000..5a7b321d --- /dev/null +++ b/tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "interfaces": { + "interface": { + "GigabitEthernet2": { + "state": { + "admin_status": "DOWN" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "time": "15:25:53", + "pri": "189", + "host": "NetAuto_CSRv-03", + "tag": "LINK-5-CHANGED", + "messageId": "1179", + "date": "May 31", + "timeZone": "MEST", + "message": "Interface GigabitEthernet2, changed state to administratively down from 10.83.75.201", + "milliseconds": "743" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "INTERFACE_DOWN", + "host": "NetAuto_CSRv-03", + "yang_model": "openconfig-interfaces", + "timestamp": 1527780353, + "os": "ios", + "severity": 5 +} \ No newline at end of file diff --git a/tests/config/ios/INTERFACE_UP/default/yang.json b/tests/config/ios/INTERFACE_UP/default/yang.json index 894284f7..c5479ab6 100644 --- a/tests/config/ios/INTERFACE_UP/default/yang.json +++ b/tests/config/ios/INTERFACE_UP/default/yang.json @@ -18,6 +18,7 @@ "host": "test-switch1", "tag": "LINEPROTO-5-UPDOWN", "time": "13:35:45", + "timeZone": null, "date": "Jun 1", "message": "Line protocol on Interface FastEthernet0/2, changed state to up", "milliseconds": "562" diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json b/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json index 5cbd0d72..0dc84163 100644 --- a/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/bootstrap/yang.json @@ -20,6 +20,7 @@ "host": "test-ztp", "tag": "SYS-5-CONFIG_I", "time": "15:49:32", + "timeZone": null, "date": "May 23", "message": "Configured from tftp://10.83.21.224//cisco-bootstrap by console", "milliseconds": "302" diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json b/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json index c4ddaea9..09794b11 100644 --- a/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/default/yang.json @@ -20,6 +20,7 @@ "host": "test-ztp", "tag": "SYS-5-CONFIG_I", "time": "13:56:15", + "timeZone": null, "date": "May 23", "message": "Configured from console by admin on vty0 (10.31.0.24)", "milliseconds": "055" diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/syslog.msg b/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/syslog.msg new file mode 100644 index 00000000..8218d14d --- /dev/null +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/syslog.msg @@ -0,0 +1 @@ +<189>27: test-ztp: May 31 15:54:54.567 MEST: %SYS-5-CONFIG_I: Configured from console by admin on console \ No newline at end of file diff --git a/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/yang.json b/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/yang.json new file mode 100644 index 00000000..3af3aaf0 --- /dev/null +++ b/tests/config/ios/USER_EXIT_CONFIG_MODE/message_with_timezone/yang.json @@ -0,0 +1,36 @@ +{ + "yang_message": { + "users": { + "user": { + "admin": { + "action": { + "vty": "console", + "exit_config_mode": true, + "from": "console" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "time": "15:54:54", + "pri": "189", + "host": "test-ztp", + "tag": "SYS-5-CONFIG_I", + "messageId": "27", + "date": "May 31", + "timeZone": "MEST", + "message": "Configured from console by admin on console", + "milliseconds": "567" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "USER_EXIT_CONFIG_MODE", + "host": "test-ztp", + "yang_model": "NO_MODEL", + "timestamp": 1527782094, + "os": "ios", + "severity": 5 +} \ No newline at end of file From d17283ca4cb85defaaf8b9b2da3838ca74047b8c Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 28 Jun 2018 16:38:53 +0200 Subject: [PATCH 33/45] ios: INTERFACE_DOWN resolving merge conflicts with ios_timezone --- .../ios/INTERFACE_DOWN/admin_down/syslog.msg | 1 - .../ios/INTERFACE_DOWN/admin_down/yang.json | 34 ------------------- .../ios/INTERFACE_DOWN/default/syslog.msg | 2 +- .../ios/INTERFACE_DOWN/default/yang.json | 25 +++++++------- .../syslog.msg | 0 .../yang.json | 0 .../ios/INTERFACE_DOWN/oper_down/syslog.msg | 1 + .../ios/INTERFACE_DOWN/oper_down/yang.json | 33 ++++++++++++++++++ .../syslog.msg | 0 .../yang.json | 0 10 files changed, 48 insertions(+), 48 deletions(-) delete mode 100644 tests/config/ios/INTERFACE_DOWN/admin_down/syslog.msg delete mode 100644 tests/config/ios/INTERFACE_DOWN/admin_down/yang.json rename tests/config/ios/INTERFACE_DOWN/{admin_down_with_timezone => message_with_timezone}/syslog.msg (100%) rename tests/config/ios/INTERFACE_DOWN/{admin_down_with_timezone => message_with_timezone}/yang.json (100%) create mode 100644 tests/config/ios/INTERFACE_DOWN/oper_down/syslog.msg create mode 100644 tests/config/ios/INTERFACE_DOWN/oper_down/yang.json rename tests/config/ios/INTERFACE_DOWN/{admin_down_time_synchronized => time_synchronized}/syslog.msg (100%) rename tests/config/ios/INTERFACE_DOWN/{admin_down_time_synchronized => time_synchronized}/yang.json (100%) diff --git a/tests/config/ios/INTERFACE_DOWN/admin_down/syslog.msg b/tests/config/ios/INTERFACE_DOWN/admin_down/syslog.msg deleted file mode 100644 index 40b3e826..00000000 --- a/tests/config/ios/INTERFACE_DOWN/admin_down/syslog.msg +++ /dev/null @@ -1 +0,0 @@ -<189>521: router1: *Nov 14 08:30:56.699: %LINK-5-CHANGED: Interface GigabitEthernet2, changed state to administratively down \ No newline at end of file diff --git a/tests/config/ios/INTERFACE_DOWN/admin_down/yang.json b/tests/config/ios/INTERFACE_DOWN/admin_down/yang.json deleted file mode 100644 index b1c7fd64..00000000 --- a/tests/config/ios/INTERFACE_DOWN/admin_down/yang.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "yang_message": { - "interfaces": { - "interface": { - "GigabitEthernet2": { - "state": { - "admin_status": "DOWN" - } - } - } - } - }, - "message_details": { - "severity": 5, - "facility": 23, - "pri": "189", - "host": "router1", - "tag": "LINK-5-CHANGED", - "time": "08:30:56", - "timeZone": null, - "date": "Nov 14", - "messageId": "521", - "milliseconds": "699", - "message": "Interface GigabitEthernet2, changed state to administratively down" - }, - "timestamp": 1507969856, - "facility": 23, - "ip": "127.0.0.1", - "host": "router1", - "yang_model": "openconfig-interfaces", - "error": "INTERFACE_DOWN", - "os": "ios", - "severity": 5 -} diff --git a/tests/config/ios/INTERFACE_DOWN/default/syslog.msg b/tests/config/ios/INTERFACE_DOWN/default/syslog.msg index 67ef8ea3..40b3e826 100644 --- a/tests/config/ios/INTERFACE_DOWN/default/syslog.msg +++ b/tests/config/ios/INTERFACE_DOWN/default/syslog.msg @@ -1 +1 @@ -<189>1432: test-switch1: Jun 1 12:50:26.172: %LINEPROTO-5-UPDOWN: Line protocol on Interface FastEthernet0/2, changed state to down \ No newline at end of file +<189>521: router1: *Nov 14 08:30:56.699: %LINK-5-CHANGED: Interface GigabitEthernet2, changed state to administratively down \ No newline at end of file diff --git a/tests/config/ios/INTERFACE_DOWN/default/yang.json b/tests/config/ios/INTERFACE_DOWN/default/yang.json index b3bd4eac..b1c7fd64 100644 --- a/tests/config/ios/INTERFACE_DOWN/default/yang.json +++ b/tests/config/ios/INTERFACE_DOWN/default/yang.json @@ -2,9 +2,9 @@ "yang_message": { "interfaces": { "interface": { - "FastEthernet0/2": { + "GigabitEthernet2": { "state": { - "oper_status": "DOWN" + "admin_status": "DOWN" } } } @@ -13,21 +13,22 @@ "message_details": { "severity": 5, "facility": 23, - "messageId": "1432", "pri": "189", - "host": "test-switch1", - "tag": "LINEPROTO-5-UPDOWN", - "time": "12:50:26", - "date": "Jun 1", - "message": "Line protocol on Interface FastEthernet0/2, changed state to down", - "milliseconds": "172" + "host": "router1", + "tag": "LINK-5-CHANGED", + "time": "08:30:56", + "timeZone": null, + "date": "Nov 14", + "messageId": "521", + "milliseconds": "699", + "message": "Interface GigabitEthernet2, changed state to administratively down" }, + "timestamp": 1507969856, "facility": 23, "ip": "127.0.0.1", - "error": "INTERFACE_DOWN", - "host": "test-switch1", + "host": "router1", "yang_model": "openconfig-interfaces", - "timestamp": 1527857426, + "error": "INTERFACE_DOWN", "os": "ios", "severity": 5 } diff --git a/tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/syslog.msg b/tests/config/ios/INTERFACE_DOWN/message_with_timezone/syslog.msg similarity index 100% rename from tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/syslog.msg rename to tests/config/ios/INTERFACE_DOWN/message_with_timezone/syslog.msg diff --git a/tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/yang.json b/tests/config/ios/INTERFACE_DOWN/message_with_timezone/yang.json similarity index 100% rename from tests/config/ios/INTERFACE_DOWN/admin_down_with_timezone/yang.json rename to tests/config/ios/INTERFACE_DOWN/message_with_timezone/yang.json diff --git a/tests/config/ios/INTERFACE_DOWN/oper_down/syslog.msg b/tests/config/ios/INTERFACE_DOWN/oper_down/syslog.msg new file mode 100644 index 00000000..67ef8ea3 --- /dev/null +++ b/tests/config/ios/INTERFACE_DOWN/oper_down/syslog.msg @@ -0,0 +1 @@ +<189>1432: test-switch1: Jun 1 12:50:26.172: %LINEPROTO-5-UPDOWN: Line protocol on Interface FastEthernet0/2, changed state to down \ No newline at end of file diff --git a/tests/config/ios/INTERFACE_DOWN/oper_down/yang.json b/tests/config/ios/INTERFACE_DOWN/oper_down/yang.json new file mode 100644 index 00000000..b3bd4eac --- /dev/null +++ b/tests/config/ios/INTERFACE_DOWN/oper_down/yang.json @@ -0,0 +1,33 @@ +{ + "yang_message": { + "interfaces": { + "interface": { + "FastEthernet0/2": { + "state": { + "oper_status": "DOWN" + } + } + } + } + }, + "message_details": { + "severity": 5, + "facility": 23, + "messageId": "1432", + "pri": "189", + "host": "test-switch1", + "tag": "LINEPROTO-5-UPDOWN", + "time": "12:50:26", + "date": "Jun 1", + "message": "Line protocol on Interface FastEthernet0/2, changed state to down", + "milliseconds": "172" + }, + "facility": 23, + "ip": "127.0.0.1", + "error": "INTERFACE_DOWN", + "host": "test-switch1", + "yang_model": "openconfig-interfaces", + "timestamp": 1527857426, + "os": "ios", + "severity": 5 +} diff --git a/tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/syslog.msg b/tests/config/ios/INTERFACE_DOWN/time_synchronized/syslog.msg similarity index 100% rename from tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/syslog.msg rename to tests/config/ios/INTERFACE_DOWN/time_synchronized/syslog.msg diff --git a/tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/yang.json b/tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json similarity index 100% rename from tests/config/ios/INTERFACE_DOWN/admin_down_time_synchronized/yang.json rename to tests/config/ios/INTERFACE_DOWN/time_synchronized/yang.json From 4739a67d97dd907cd75682c91053f891d6cc6413 Mon Sep 17 00:00:00 2001 From: Simon Bitterli Date: Thu, 28 Jun 2018 16:44:09 +0200 Subject: [PATCH 34/45] ios: adding timezone attribute to testcase OPER_DOWN in INTERFACE_DOWN --- tests/config/ios/INTERFACE_DOWN/oper_down/yang.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/ios/INTERFACE_DOWN/oper_down/yang.json b/tests/config/ios/INTERFACE_DOWN/oper_down/yang.json index b3bd4eac..c9984762 100644 --- a/tests/config/ios/INTERFACE_DOWN/oper_down/yang.json +++ b/tests/config/ios/INTERFACE_DOWN/oper_down/yang.json @@ -18,6 +18,7 @@ "host": "test-switch1", "tag": "LINEPROTO-5-UPDOWN", "time": "12:50:26", + "timeZone": null, "date": "Jun 1", "message": "Line protocol on Interface FastEthernet0/2, changed state to down", "milliseconds": "172" From a194685714cf42c8d9a32f597c4a52df61383b09 Mon Sep 17 00:00:00 2001 From: Antoine Meillet Date: Tue, 17 Jul 2018 11:41:13 +0200 Subject: [PATCH 35/45] Correct small typo --- docs/device_config/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/device_config/index.rst b/docs/device_config/index.rst index 25ef2886..c9fdcd9f 100644 --- a/docs/device_config/index.rst +++ b/docs/device_config/index.rst @@ -30,6 +30,6 @@ partially parsed messages), but this behaviour is disabled by default. To allow publishing messages from operating systems that are not supported yet by napalm-logs (but they will not be parsed at all), you can configure the :ref:`publisher-opts-send-unknown` option on the publisher (i.e., -``send_unknow: true``). To publish partially parsed messages from supported +``send_unknown: true``). To publish partially parsed messages from supported operating systems, but without a mapping for a certain class of messages, you can use the :ref:`publisher-opts-send-raw` option. From c352b16d6296bd0380f036562eefa08147dcef8b Mon Sep 17 00:00:00 2001 From: Antoine Meillet Date: Wed, 25 Jul 2018 10:09:31 +0200 Subject: [PATCH 36/45] Support for Huawei VRP logs - INTERFACE_UP and INTERFACE_DOWN - associated tests --- napalm_logs/config/huawei/INTERFACE_DOWN.yml | 13 +++++++ napalm_logs/config/huawei/INTERFACE_UP.yml | 13 +++++++ napalm_logs/config/huawei/init.yml | 19 +++++++++++ .../huawei/INTERFACE_DOWN/default/syslog.msg | 1 + .../huawei/INTERFACE_DOWN/default/yang.json | 34 +++++++++++++++++++ .../huawei/INTERFACE_UP/default/syslog.msg | 1 + .../huawei/INTERFACE_UP/default/yang.json | 34 +++++++++++++++++++ 7 files changed, 115 insertions(+) create mode 100644 napalm_logs/config/huawei/INTERFACE_DOWN.yml create mode 100644 napalm_logs/config/huawei/INTERFACE_UP.yml create mode 100644 napalm_logs/config/huawei/init.yml create mode 100644 tests/config/huawei/INTERFACE_DOWN/default/syslog.msg create mode 100644 tests/config/huawei/INTERFACE_DOWN/default/yang.json create mode 100644 tests/config/huawei/INTERFACE_UP/default/syslog.msg create mode 100644 tests/config/huawei/INTERFACE_UP/default/yang.json diff --git a/napalm_logs/config/huawei/INTERFACE_DOWN.yml b/napalm_logs/config/huawei/INTERFACE_DOWN.yml new file mode 100644 index 00000000..6f9aaa85 --- /dev/null +++ b/napalm_logs/config/huawei/INTERFACE_DOWN.yml @@ -0,0 +1,13 @@ +# %%01IFNET/4/IF_STATE(l)[8211]:Interface Ethernet0/0/48 has turned into DOWN state. + +messages: + - error: INTERFACE_DOWN + tag: 01IFNET/4/IF_STATE + values: + interface: (\w+[\.\-\d\/\w+]+) + line: 'Interface {interface} has turned into DOWN state.' + model: openconfig-interfaces + mapping: + variables: {} + static: + interfaces//interface//{interface}//state//oper_status: DOWN diff --git a/napalm_logs/config/huawei/INTERFACE_UP.yml b/napalm_logs/config/huawei/INTERFACE_UP.yml new file mode 100644 index 00000000..ff9ba9d0 --- /dev/null +++ b/napalm_logs/config/huawei/INTERFACE_UP.yml @@ -0,0 +1,13 @@ +# %%01IFNET/4/IF_STATE(l)[8211]:Interface Ethernet0/0/48 has turned into UP state. + +messages: + - error: INTERFACE_UP + tag: 01IFNET/4/IF_STATE + values: + interface: (\w+[\.\-\d\/\w+]+) + line: 'Interface {interface} has turned into UP state.' + model: openconfig-interfaces + mapping: + variables: {} + static: + interfaces//interface//{interface}//state//oper_status: UP diff --git a/napalm_logs/config/huawei/init.yml b/napalm_logs/config/huawei/init.yml new file mode 100644 index 00000000..7fe18ce7 --- /dev/null +++ b/napalm_logs/config/huawei/init.yml @@ -0,0 +1,19 @@ +# Prefix profiler for Huawei devices. +# This profiler matches messages having the following form: +# +# .. code-block:: text +# +# 2018-7-19 11:58:03.150.1 this-is-my-hostname %%01LLDP/4/BAD_PACKET(l)[87319]:4 invalid packets were received after +# latest notification. The last invalid packet came from interface Ethernet0/0/25. +# +prefixes: + - time_format: "%Y-%m-%d %H:%M:%S" + values: + date: (\d+-\d+-\d+) + time: (\d+:\d+:\d+) + milliseconds: (\d{3}) + host: ([^ ]+) + tag: (\w+\/\w+\/\w+) + otherThings: (\(\w+\)) + messageId: \[(\d+)\] + line: '{date} {time}.{milliseconds}.1 {host} %%{tag}{otherThings}{messageId}:' diff --git a/tests/config/huawei/INTERFACE_DOWN/default/syslog.msg b/tests/config/huawei/INTERFACE_DOWN/default/syslog.msg new file mode 100644 index 00000000..682112cc --- /dev/null +++ b/tests/config/huawei/INTERFACE_DOWN/default/syslog.msg @@ -0,0 +1 @@ +<29>2018-7-23 01:00:34.270.1 my-awesome-huawei-switch %%01IFNET/4/IF_STATE(l)[4997]:Interface Ethernet0/0/8 has turned into DOWN state. diff --git a/tests/config/huawei/INTERFACE_DOWN/default/yang.json b/tests/config/huawei/INTERFACE_DOWN/default/yang.json new file mode 100644 index 00000000..dd989149 --- /dev/null +++ b/tests/config/huawei/INTERFACE_DOWN/default/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "interfaces": { + "interface": { + "Ethernet0/0/8": { + "state": { + "oper_status": "DOWN" + } + } + } + } + }, + "message_details": { + "facility": 3, + "severity": 5, + "otherThings": "(l)", + "pri": "29", + "messageId": "4997", + "host": "my-awesome-huawei-switch", + "tag": "01IFNET/4/IF_STATE", + "time": "01:00:34", + "date": "2018-7-23", + "message": "Interface Ethernet0/0/8 has turned into DOWN state.", + "milliseconds": "270" + }, + "ip": "127.0.0.1", + "error": "INTERFACE_DOWN", + "host": "my-awesome-huawei-switch", + "yang_model": "openconfig-interfaces", + "timestamp": 1532300434, + "os": "huawei", + "facility": 3, + "severity": 5 +} diff --git a/tests/config/huawei/INTERFACE_UP/default/syslog.msg b/tests/config/huawei/INTERFACE_UP/default/syslog.msg new file mode 100644 index 00000000..2b50232d --- /dev/null +++ b/tests/config/huawei/INTERFACE_UP/default/syslog.msg @@ -0,0 +1 @@ +<29>2018-7-23 01:00:33.270.1 my-awesome-huawei-switch %%01IFNET/4/IF_STATE(l)[4997]:Interface Ethernet0/0/8 has turned into UP state. diff --git a/tests/config/huawei/INTERFACE_UP/default/yang.json b/tests/config/huawei/INTERFACE_UP/default/yang.json new file mode 100644 index 00000000..44abfafc --- /dev/null +++ b/tests/config/huawei/INTERFACE_UP/default/yang.json @@ -0,0 +1,34 @@ +{ + "yang_message": { + "interfaces": { + "interface": { + "Ethernet0/0/8": { + "state": { + "oper_status": "UP" + } + } + } + } + }, + "message_details": { + "facility": 3, + "severity": 5, + "otherThings": "(l)", + "pri": "29", + "messageId": "4997", + "host": "my-awesome-huawei-switch", + "tag": "01IFNET/4/IF_STATE", + "time": "01:00:33", + "date": "2018-7-23", + "message": "Interface Ethernet0/0/8 has turned into UP state.", + "milliseconds": "270" + }, + "ip": "127.0.0.1", + "error": "INTERFACE_UP", + "host": "my-awesome-huawei-switch", + "yang_model": "openconfig-interfaces", + "timestamp": 1532300433, + "os": "huawei", + "facility": 3, + "severity": 5 +} From 42e3ec2e88b06c49dab0d1421be0e460496d2337 Mon Sep 17 00:00:00 2001 From: Antoine Meillet Date: Thu, 26 Jul 2018 14:54:41 +0200 Subject: [PATCH 37/45] Rename field --- napalm_logs/config/huawei/init.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/napalm_logs/config/huawei/init.yml b/napalm_logs/config/huawei/init.yml index 7fe18ce7..17ad95e3 100644 --- a/napalm_logs/config/huawei/init.yml +++ b/napalm_logs/config/huawei/init.yml @@ -14,6 +14,6 @@ prefixes: milliseconds: (\d{3}) host: ([^ ]+) tag: (\w+\/\w+\/\w+) - otherThings: (\(\w+\)) + informationType: (\(\w+\)) messageId: \[(\d+)\] - line: '{date} {time}.{milliseconds}.1 {host} %%{tag}{otherThings}{messageId}:' + line: '{date} {time}.{milliseconds}.1 {host} %%{tag}{informationType}{messageId}:' From c079cf57ba975725223016cd8f4ee482b611a7e6 Mon Sep 17 00:00:00 2001 From: Antoine Meillet Date: Thu, 26 Jul 2018 14:58:20 +0200 Subject: [PATCH 38/45] Field renaming in tests --- tests/config/huawei/INTERFACE_DOWN/default/yang.json | 2 +- tests/config/huawei/INTERFACE_UP/default/yang.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config/huawei/INTERFACE_DOWN/default/yang.json b/tests/config/huawei/INTERFACE_DOWN/default/yang.json index dd989149..17e41a69 100644 --- a/tests/config/huawei/INTERFACE_DOWN/default/yang.json +++ b/tests/config/huawei/INTERFACE_DOWN/default/yang.json @@ -13,7 +13,7 @@ "message_details": { "facility": 3, "severity": 5, - "otherThings": "(l)", + "informationType": "(l)", "pri": "29", "messageId": "4997", "host": "my-awesome-huawei-switch", diff --git a/tests/config/huawei/INTERFACE_UP/default/yang.json b/tests/config/huawei/INTERFACE_UP/default/yang.json index 44abfafc..639a9fb3 100644 --- a/tests/config/huawei/INTERFACE_UP/default/yang.json +++ b/tests/config/huawei/INTERFACE_UP/default/yang.json @@ -13,7 +13,7 @@ "message_details": { "facility": 3, "severity": 5, - "otherThings": "(l)", + "informationType": "(l)", "pri": "29", "messageId": "4997", "host": "my-awesome-huawei-switch", From aad3324b9acc980479e849b097ec8fa662a4f4b1 Mon Sep 17 00:00:00 2001 From: Antoine Meillet Date: Thu, 26 Jul 2018 17:25:00 +0200 Subject: [PATCH 39/45] Return message even if no OS has matched This commit fixes https://github.com/napalm-automation/napalm-logs/issues/252 and might also help keeping track of messages that do not match any OS in general. --- napalm_logs/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/napalm_logs/server.py b/napalm_logs/server.py index d6fc9c0c..0431a358 100644 --- a/napalm_logs/server.py +++ b/napalm_logs/server.py @@ -193,6 +193,7 @@ def _identify_os(self, msg): log.debug('No match found for %s', dev_os) if not ret: log.debug('Not matched any OS') + ret.append((None, msg_dict)) return ret def start(self): From 9a0afc81fb9a6c22a2ac12d93c4d6d6711babaeb Mon Sep 17 00:00:00 2001 From: John Anderson Date: Tue, 7 Aug 2018 23:06:53 -0400 Subject: [PATCH 40/45] fixes #255 - handling of invalid syslog dates --- docs/syslog/index.rst | 6 ++++++ napalm_logs/device.py | 26 ++++++++++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docs/syslog/index.rst b/docs/syslog/index.rst index 2ebab4de..572d5c55 100644 --- a/docs/syslog/index.rst +++ b/docs/syslog/index.rst @@ -83,3 +83,9 @@ MSG --- The MSG part contains the text of the message itself. + +Date and Time Notes +------------------- + +Most syslog messages do not incude the year in the message timestamp. Napalm-logs overcomes this by appending the year from the server. While rare, it is possible to form an invalid date when the device +emitting the syslog has an incorrect local time set (e.g. February 29, 2018 where 2018 is not a leap year). In such cases, Napalm-logs falls back to using the date and time of the server. diff --git a/napalm_logs/device.py b/napalm_logs/device.py index 47f14486..5dfeccad 100644 --- a/napalm_logs/device.py +++ b/napalm_logs/device.py @@ -216,12 +216,26 @@ def _format_time(self, time, date, timezone, prefix_id): parsed_time = datetime.strptime('{} {}'.format(date, time), time_format) else: year = datetime.now().year - parsed_time = datetime.strptime('{} {} {}'.format(year, date, time), '%Y {}'.format(time_format)) - # If the timestamp is in the future then it is likely that the year - # is wrong. We subtract 1 day from the parsed time to eleminate any - # difference between clocks. - if parsed_time - timedelta(days=1) > datetime.now(): - parsed_time = datetime.strptime('{} {} {}'.format(year - 1, date, time), '%Y {}'.format(time_format)) + try: + parsed_time = datetime.strptime('{} {} {}'.format(year, date, time), '%Y {}'.format(time_format)) + # If the timestamp is in the future then it is likely that the year + # is wrong. We subtract 1 day from the parsed time to eleminate any + # difference between clocks. + if parsed_time - timedelta(days=1) > datetime.now(): + parsed_time = datetime.strptime( + '{} {} {}'.format(year - 1, date, time), + '%Y {}'.format(time_format) + ) + except ValueError: + # It is rare but by appending the year from the server, we could produce + # an invalid date such as February 29, 2018 (2018 is not a leap year). This + # is caused by the device emitting the syslog having an incorrect local date set. + # In such cases, we fall back to the full date from the server and log this action. + parsed_time = datetime.now().strftime(time_format) + log.info( + "Invalid date produced while formatting syslog date. Falling back to server date [%s]", + self._name + ) return int((parsed_time - datetime(1970, 1, 1)).total_seconds()) def start(self): From 00ca163f0977dd202efff39281a358862200ef00 Mon Sep 17 00:00:00 2001 From: Antoine Meillet Date: Wed, 8 Aug 2018 15:43:30 +0200 Subject: [PATCH 41/45] Allow SO_REUSEPORT on TCP and UDP listeners --- docs/listener/tcp.rst | 13 +++++++++++++ docs/listener/udp.rst | 13 +++++++++++++ napalm_logs/config/__init__.py | 1 + napalm_logs/listener/tcp.py | 8 ++++++++ napalm_logs/listener/udp.py | 8 ++++++++ 5 files changed, 43 insertions(+) diff --git a/docs/listener/tcp.rst b/docs/listener/tcp.rst index a03c16ee..e03ed6a1 100644 --- a/docs/listener/tcp.rst +++ b/docs/listener/tcp.rst @@ -21,6 +21,19 @@ Example: tcp: buffer_size: 2048 +``reuse_port``: ``false`` +------------------------- + +Enable or disable SO_REUSEPORT on listener's socket. + +Example: + +.. code-block:: yaml + + listener: + udp: + reuse_port: true + ``socket_timeout``: ``60`` -------------------------- diff --git a/docs/listener/udp.rst b/docs/listener/udp.rst index e1a28019..99f2a195 100644 --- a/docs/listener/udp.rst +++ b/docs/listener/udp.rst @@ -20,3 +20,16 @@ Example: listener: udp: buffer_size: 2048 + +``reuse_port``: ``false`` +------------------------- + +Enable or disable SO_REUSEPORT on listener's socket. + +Example: + +.. code-block:: yaml + + listener: + udp: + reuse_port: true diff --git a/napalm_logs/config/__init__.py b/napalm_logs/config/__init__.py index 612c45d1..466ef4bb 100644 --- a/napalm_logs/config/__init__.py +++ b/napalm_logs/config/__init__.py @@ -99,6 +99,7 @@ # listener BUFFER_SIZE = 1024 +REUSE_PORT = False TIMEOUT = 60 # device diff --git a/napalm_logs/listener/tcp.py b/napalm_logs/listener/tcp.py index 479cdd0a..02bc2f6f 100644 --- a/napalm_logs/listener/tcp.py +++ b/napalm_logs/listener/tcp.py @@ -21,6 +21,7 @@ # Import napalm-logs pkgs from napalm_logs.config import TIMEOUT from napalm_logs.config import BUFFER_SIZE +from napalm_logs.config import REUSE_PORT from napalm_logs.config import MAX_TCP_CLIENTS from napalm_logs.listener.base import ListenerBase # exceptions @@ -44,6 +45,7 @@ def __init__(self, address, port, **kwargs): else: self.port = port self.buffer_size = kwargs.get('buffer_size', BUFFER_SIZE) + self.reuse_port = kwargs.get('reuse_port', REUSE_PORT) self.socket_timeout = kwargs.get('socket_timeout', TIMEOUT) self.max_clients = kwargs.get('max_clients', MAX_TCP_CLIENTS) self.buffer = queue.Queue() @@ -100,6 +102,12 @@ def start(self): self.skt = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) else: self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if self.reuse_port: + self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + else: + log.error('SO_REUSEPORT not supported') try: self.skt.bind((self.address, int(self.port))) except socket.error as msg: diff --git a/napalm_logs/listener/udp.py b/napalm_logs/listener/udp.py index a3cdf1d3..fc4e03f4 100644 --- a/napalm_logs/listener/udp.py +++ b/napalm_logs/listener/udp.py @@ -14,6 +14,7 @@ # Import napalm-logs pkgs from napalm_logs.config import BUFFER_SIZE +from napalm_logs.config import REUSE_PORT from napalm_logs.listener.base import ListenerBase # exceptions from napalm_logs.exceptions import BindException @@ -36,6 +37,7 @@ def __init__(self, address, port, **kwargs): else: self.port = port self.buffer_size = kwargs.get('buffer_size', BUFFER_SIZE) + self.reuse_port = kwargs.get('reuse_port', REUSE_PORT) log.debug('Buffer size: %d', self.buffer_size) def start(self): @@ -46,6 +48,12 @@ def start(self): self.skt = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) else: self.skt = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + if self.reuse_port: + self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + else: + log.error('SO_REUSEPORT not supported') try: self.skt.bind((self.address, int(self.port))) except socket.error as msg: From 3aee9dfacf41de7763deeecfdd8c2dcf8a839dbe Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 8 Aug 2018 22:10:55 +0100 Subject: [PATCH 42/45] Bump version to 0.6.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 79a2734b..a918a2aa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0 \ No newline at end of file +0.6.0 From 9fe28b6d5f018702260ad9039b9a50cbd4a8a5d9 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Fri, 10 Aug 2018 16:21:46 +0100 Subject: [PATCH 43/45] Add release notes for 0.6.0 --- docs/releases/0.6.0.rst | 92 +++++++++++++++++++++++++++++++++++++++++ docs/releases/index.rst | 3 +- 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 docs/releases/0.6.0.rst diff --git a/docs/releases/0.6.0.rst b/docs/releases/0.6.0.rst new file mode 100644 index 00000000..abdf05d8 --- /dev/null +++ b/docs/releases/0.6.0.rst @@ -0,0 +1,92 @@ +.. _release-0.6.0: + +================================ +Release 0.6.0 - Codename Fretsaw +================================ + +Prometheus Metrics +------------------ + +John Anderson (@lampwins) added support for Prometheus metrics. While this +feature is optional, ``napalm-logs`` has an additional requirements starting +with this release: the `prometheus_client +`_ Python library for Prometheus +instrumentation, which is used to expose the metrics via an HTTP server. +The implementation is fully compliant with Prometheus scraping, so you need +only point your Prometheus server to the exposed metrics to scrape them. + +See :ref:`metrics` for further details and configuration options, as well as +the new options: :ref:`onfiguration-options-enable-metrics`, +:ref:`configuration-options-metrics-address`, +:ref:`configuration-options-metrics-port`, and +:ref:`configuration-options-metrics-dir`. + +Brocade NetIron Support +----------------------- + +Initial support for Brocade NetIron has been added thanks to Johan van den +Dorpe (@johanek), providing notifications for +:ref:`BGP_NEIGHBOR_STATE_CHANGED`, when the BGP peer is ``UP``, ``DOWN``, +``IDLE``, or ``ESTABLISHED`` (as defined in the ``openconfig-bgp`` YANG model). +The platforms is available as ``netiron``. + +Huawei VRP Support +------------------ + +Starting with this release, also Huawei VRP is supported under the ``huawei`` +name, firing notifications for :ref:`INTERFACE_UP` and :ref:`INTERFACE_DOWN`. +This has been added thanks to Antoine Meillet (@inetAnt). + +As the list of supported platforms keeps growing, do consider using the +:ref:`configuration-options-device-blacklist` and +:ref:`configuration-options-device-whitelist` for fine tuning. + +More configuration options +-------------------------- + +Antoine Meillet (@inetAnt) also added more options for the :ref:`listener-tcp` +and :ref:`listener-udp`, to enable or disable the ``SO_REUSEPORT`` on +listener's socket - see :ref:`listener-tcp-reuse-port` and +:ref:`listener-udp-reuse-port`, respectively. + +Improvements +------------ + +Simon Bitterli (@bittsi) added timezone support for Cisco IOS. + +Bug fixes +--------- + +Due to a known issue, multiple platforms having the same +:ref:`syslog-header`, ``napalm-logs`` was unable to match them. Luke Overend +fixed this, which would also permit us in the future to have both ``junos`` +and UNIX message parsing concomitantly. + +John Anderson (@lampwins) also fixed a core issue with the time extraction when +the syslog message doesn't provide it. + +Known issues +------------ + +When a message is not fully parsed and matched, but there are multiple +platforms identified as it could belong to, when the +:ref:`publisher-opts-send-raw` Publisher option is enabled, the message will be +published twice. This is currently being tracked under +https://github.com/napalm-automation/napalm-logs/issues/246. + + +New Structured Messages +----------------------- + +- :ref:`USER_EXIT_CONFIG_MODE` added by Simon Bitterli (@bittsi) providing the + counterpart of the existing :ref:`USER_ENTER_CONFIG_MODE`. This message is + particularly important on Cisco IOS platforms (but not limited to), after an + user applied a configuration change into the running config, or bootstrapped + via TFTP. This message is currently available for ``ios`` only. + +The following messages were already defined, now extending the list of +supported platforms: + +- :ref:`INTERFACE_UP` added for ``huawei``. +- :ref:`INTERFACE_DOWN` added for ``huawei``. +- :ref:`BGP_NEIGHBOR_STATE_CHANGED` added for ``netiron``. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 39f08f9a..56f59fd4 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -8,11 +8,12 @@ Release Notes Latest Release ^^^^^^^^^^^^^^ -- :ref:`release-0.5.0` +- :ref:`release-0.6.0` Previous Releases ^^^^^^^^^^^^^^^^^ +- :ref:`release-0.5.0` - :ref:`release-0.4.2` - :ref:`release-0.4.1` - :ref:`release-0.4.0` From 8cccf054cb7f5dddd3cc11305a9e0a1ca18aeb2c Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Mon, 13 Aug 2018 08:06:58 +0100 Subject: [PATCH 44/45] More crossrefs and some minor corrections --- docs/listener/tcp.rst | 15 ++++++++++++++- docs/listener/udp.rst | 9 ++++++++- docs/messages/UNKNOWN.rst | 22 +++++++++++----------- docs/options/index.rst | 18 ++++++++++++++---- docs/releases/0.6.0.rst | 3 ++- 5 files changed, 49 insertions(+), 18 deletions(-) diff --git a/docs/listener/tcp.rst b/docs/listener/tcp.rst index e03ed6a1..6b33463e 100644 --- a/docs/listener/tcp.rst +++ b/docs/listener/tcp.rst @@ -8,6 +8,8 @@ Receive the unstructured syslog messages over TCP. Available options: +.. _listener-tcp-buffer-size: + ``buffer_size``: ``1024`` ------------------------- @@ -21,10 +23,15 @@ Example: tcp: buffer_size: 2048 + +.. _listener-tcp-reuse-port: + ``reuse_port``: ``false`` ------------------------- -Enable or disable SO_REUSEPORT on listener's socket. +.. versionadded:: 0.6.0 + +Enable or disable ``SO_REUSEPORT`` on listener's socket. Example: @@ -34,6 +41,9 @@ Example: udp: reuse_port: true + +.. _listener-tcp-socket-timeout: + ``socket_timeout``: ``60`` -------------------------- @@ -47,6 +57,9 @@ Example: tcp: socket_timeout: 5 + +.. _listener-tcp-max-clients: + ``max_clients``: ``5`` ---------------------- diff --git a/docs/listener/udp.rst b/docs/listener/udp.rst index 99f2a195..dc2a5ea9 100644 --- a/docs/listener/udp.rst +++ b/docs/listener/udp.rst @@ -8,6 +8,8 @@ Receive the unstructured syslog messages over UDP. Available options: +.. _listener-udp-buffer-size: + ``buffer_size``: ``1024`` ------------------------- @@ -21,10 +23,15 @@ Example: udp: buffer_size: 2048 + +.. _listener-udp-reuse-port: + ``reuse_port``: ``false`` ------------------------- -Enable or disable SO_REUSEPORT on listener's socket. +.. versionadded:: 0.6.0 + +Enable or disable ``SO_REUSEPORT`` on listener's socket. Example: diff --git a/docs/messages/UNKNOWN.rst b/docs/messages/UNKNOWN.rst index 4bf6c510..f779dc1f 100644 --- a/docs/messages/UNKNOWN.rst +++ b/docs/messages/UNKNOWN.rst @@ -19,15 +19,15 @@ Example: .. code-block:: json - { - "message_details": { - "message": "<28>Jul 10 10:32:00 vmx1 inetd[2397]: /usr/sbin/sshd[89736]: exited, status 255\n" - }, - "timestamp": 1501685287, - "ip": "127.0.0.1", - "host": "unknown", - "error": "UNKNOWN", - "os": "unknown", - "model_name": "unknown" - } + { + "message_details": { + "message": "<28>Jul 10 10:32:00 vmx1 inetd[2397]: /usr/sbin/sshd[89736]: exited, status 255\n" + }, + "timestamp": 1501685287, + "ip": "127.0.0.1", + "host": "unknown", + "error": "UNKNOWN", + "os": "unknown", + "model_name": "unknown" + } diff --git a/docs/options/index.rst b/docs/options/index.rst index 86b15e87..00a30b8b 100644 --- a/docs/options/index.rst +++ b/docs/options/index.rst @@ -79,7 +79,9 @@ Configuration file example: .. _configuration-options-enable-metrics: ``enable-metrics`` -------------- +------------------ + +.. versionadded:: 0.6.0 Enable metrics collection exposition (Prometheus metrics). @@ -100,7 +102,9 @@ Configuration file example: .. _configuration-options-metrics-address: ``metrics-address`` ------------ +------------------- + +.. versionadded:: 0.6.0 The IP address to use to listen for all incoming metrics requests. This is the address that the Prometheus HTTP server exposes metrics from. @@ -122,7 +126,9 @@ Configuration file example: .. _configuration-options-metrics-port: ``metrics-port`` -------------- +---------------- + +.. versionadded:: 0.6.0 The port to listen on for incoming metrics requests. This is the port that the Prometheus HTTP server exposes metrics from. @@ -141,8 +147,12 @@ Configuration file example: metrics_port: 2022 +.. _configuration-options-metrics-dir: + ``metrics-dir`` -------------- +--------------- + +.. versionadded:: 0.6.0 The directory used by the processes for metrics collection. This directory must be writable. If the directory does not exist, we attempt to create it. diff --git a/docs/releases/0.6.0.rst b/docs/releases/0.6.0.rst index abdf05d8..9fae4504 100644 --- a/docs/releases/0.6.0.rst +++ b/docs/releases/0.6.0.rst @@ -28,7 +28,7 @@ Initial support for Brocade NetIron has been added thanks to Johan van den Dorpe (@johanek), providing notifications for :ref:`BGP_NEIGHBOR_STATE_CHANGED`, when the BGP peer is ``UP``, ``DOWN``, ``IDLE``, or ``ESTABLISHED`` (as defined in the ``openconfig-bgp`` YANG model). -The platforms is available as ``netiron``. +The platform is available as ``netiron``. Huawei VRP Support ------------------ @@ -36,6 +36,7 @@ Huawei VRP Support Starting with this release, also Huawei VRP is supported under the ``huawei`` name, firing notifications for :ref:`INTERFACE_UP` and :ref:`INTERFACE_DOWN`. This has been added thanks to Antoine Meillet (@inetAnt). +The platform is available as ``huawei``. As the list of supported platforms keeps growing, do consider using the :ref:`configuration-options-device-blacklist` and From 86965cc515acbe3cbe8ee956f93ff571e502afc4 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Mon, 13 Aug 2018 08:13:44 +0100 Subject: [PATCH 45/45] Add doc for device config for ios, netiron, huawei - to be completed --- docs/device_config/huawei.rst | 7 +++++++ docs/device_config/index.rst | 3 +++ docs/device_config/ios.rst | 7 +++++++ docs/device_config/netiron.rst | 7 +++++++ 4 files changed, 24 insertions(+) create mode 100644 docs/device_config/huawei.rst create mode 100644 docs/device_config/ios.rst create mode 100644 docs/device_config/netiron.rst diff --git a/docs/device_config/huawei.rst b/docs/device_config/huawei.rst new file mode 100644 index 00000000..9a237be7 --- /dev/null +++ b/docs/device_config/huawei.rst @@ -0,0 +1,7 @@ +.. _device-configuration-huawei: + +========== +Huawei VRP +========== + +.. versionadded:: 0.6.0 diff --git a/docs/device_config/index.rst b/docs/device_config/index.rst index c9fdcd9f..f325b32e 100644 --- a/docs/device_config/index.rst +++ b/docs/device_config/index.rst @@ -14,6 +14,9 @@ systems: iosxr eos nxos + ios + netiron + huawei To see how to configure the network device, check the documents referenced above. Note that the examples in each case represents the configuration used to diff --git a/docs/device_config/ios.rst b/docs/device_config/ios.rst new file mode 100644 index 00000000..fb056a10 --- /dev/null +++ b/docs/device_config/ios.rst @@ -0,0 +1,7 @@ +.. _device-configuration-ios: + +========= +Cisco IOS +========= + +.. versionadded:: 0.5.0 diff --git a/docs/device_config/netiron.rst b/docs/device_config/netiron.rst new file mode 100644 index 00000000..766136f5 --- /dev/null +++ b/docs/device_config/netiron.rst @@ -0,0 +1,7 @@ +.. _device-configuration-netiron: + +=============== +Brocade NetIron +=============== + +.. versionadded:: 0.6.0