Reorganize tests directory to add plugins tests

.gitignore

@@ -8,3 +8,4 @@ docs/_build/*
 
 .cache/v/cache/lastfailed
 dashboards/archive/*
+tests/.cache

.travis.yml

@@ -10,12 +10,10 @@ services:
 install:
   - pip install -r requirements.txt
   - docker pull dgarros/tcpreplay
-  - docker pull juniper/open-nti-input-jti
-  - docker pull juniper/open-nti-input-syslog
-  - docker build -t juniper/open-nti:unittest .
+  - docker pull spotify/kafka
 
 script:
-  - python -m pytest -v --durations=10
+  - make test
   - cd dashboards
   - python gendashboard.py --file data_streaming_collector.yaml
   - python gendashboard.py --file data_collection_agent.yaml

Dockerfile

@@ -116,7 +116,7 @@ ADD     docker/nginx/run.sh /etc/service/nginx/run
 ### open-nti python scripts (for gathering informatino from server to router)  ###
 ADD     open-nti/open-nti.py /opt/open-nti/open-nti.py
 ADD     open-nti/startcron.py /opt/open-nti/startcron.py
-ADD     tests/pyez_mock.py /opt/open-nti/pyez_mock.py
+ADD     tests/main/pyez_mock.py /opt/open-nti/pyez_mock.py
 
 ### Add test files
 RUN     mkdir /opt/open-nti/tests

Makefile

@@ -8,6 +8,8 @@ else
   IMAGE_TAG = $(BRANCH)
 endif
 
+TEST_TAG=unittest
+
 PWD = $(shell pwd)
 VAR_FILE ?= open-nti.params
 DOCKER_FILE = docker-compose.yml
@@ -18,7 +20,10 @@ TAG ?= all
 #Load params file with all variables
 include $(VAR_FILE)
 
-build: build-main build-jti 
+# Define run options for Docker-compose
+RUN_OPTIONS = IMAGE_TAG=$(IMAGE_TAG)
+
+build: build-main build-jti build-syslog
 
 build-main:
 	@echo "======================================================================"
@@ -30,44 +35,46 @@ build-jti:
 	@echo "======================================================================"
 	@echo "Build Docker image - $(INPUT_JTI_IMAGE_NAME):$(IMAGE_TAG)"
 	@echo "======================================================================"
-	docker build -f plugins/input-jti/Dockerfile -t $(INPUT_JTI_IMAGE_NAME):$(IMAGE_TAG) plugins/input-jti
+	docker build -f $(INPUT_JTI_DIR)/Dockerfile -t $(INPUT_JTI_IMAGE_NAME):$(IMAGE_TAG) $(INPUT_JTI_DIR)
 
 build-syslog:
 	@echo "======================================================================"
 	@echo "Build Docker image - $(INPUT_SYSLOG_IMAGE_NAME):$(IMAGE_TAG)"
 	@echo "======================================================================"
-	docker build -f plugins/input-syslog/Dockerfile -t $(INPUT_SYSLOG_IMAGE_NAME):$(IMAGE_TAG) plugins/input-syslog
+	docker build -f $(INPUT_SYSLOG_DIR)/Dockerfile -t $(INPUT_SYSLOG_IMAGE_NAME):$(IMAGE_TAG) $(INPUT_SYSLOG_DIR)
 
 test:
-	docker build -t $(IMAGE_NAME):unittest .
+	docker build -t $(MAIN_IMAGE_NAME):$(TEST_TAG) .
+	docker build -f $(INPUT_JTI_DIR)/Dockerfile -t $(INPUT_JTI_IMAGE_NAME):$(TEST_TAG) $(INPUT_JTI_DIR)
+	docker build -f $(INPUT_SYSLOG_DIR)/Dockerfile -t $(INPUT_SYSLOG_IMAGE_NAME):$(TEST_TAG) $(INPUT_SYSLOG_DIR)
 	python -m pytest -v
 
 cli:
 	docker exec -it $(CONTAINER_NAME) /bin/bash
 
 start:
 	echo "Use docker compose file : $(DOCKER_FILE)"
-	docker-compose -f $(DOCKER_FILE) up -d
+	$(RUN_OPTIONS) docker-compose -f $(DOCKER_FILE) up -d
 
 start-persistent:
 	echo "Use docker compose file : $(DOCKER_FILE_P)"
-	docker-compose -f $(DOCKER_FILE_P) up -d
+	$(RUN_OPTIONS) docker-compose -f $(DOCKER_FILE_P) up -d
 
 stop:
 	echo "Use docker compose file : $(DOCKER_FILE)"
-	docker-compose -f $(DOCKER_FILE) down
+	$(RUN_OPTIONS) docker-compose -f $(DOCKER_FILE) down
 
 stop-persistent:
 	echo "Use docker compose file : $(DOCKER_FILE_P)"
-	docker-compose -f $(DOCKER_FILE_P) down
+	$(RUN_OPTIONS) docker-compose -f $(DOCKER_FILE_P) down
 
 update:
-	echo "OpenNTI - Update the files from Github"
+	@echo "OpenNTI - Update the files from Github"
 	git pull
-	echo "OpenNTI - Update the containers from Docker Hub"
+	@echo "OpenNTI - Update the containers from Docker Hub"
 	docker pull $(IMAGE_NAME):latest
-	docker pull juniper/open-nti-input-jti:latest
-	docker pull juniper/open-nti-input-syslog:latest
+	docker pull $(INPUT_JTI_IMAGE_NAME):latest
+	docker pull $(INPUT_SYSLOG_IMAGE_NAME):latest
 
 cron-show:
 	# if [ $(TAG) == "all" ]; then

docker-compose.yml

@@ -1,7 +1,7 @@
 
 input-jti:
-  image: juniper/open-nti-input-jti
-  container_name: opennti_input_jti
+  image: $INPUT_JTI_IMAGE_NAME:$IMAGE_TAG
+  container_name: $INPUT_JTI_CONTAINER_NAME
   environment:
    - "INFLUXDB_ADDR=opennti"
    - "OUTPUT_INFLUXDB=true"
@@ -15,8 +15,8 @@ input-jti:
    - opennti
 
 input-syslog:
-  image: juniper/open-nti-input-syslog
-  container_name: opennti_input_syslog
+  image: $INPUT_SYSLOG_IMAGE_NAME:$IMAGE_TAG
+  container_name: $INPUT_SYSLOG_CONTAINER_NAME
   environment:
    - "INFLUXDB_ADDR=opennti"
    - "OUTPUT_INFLUXDB=true"
@@ -29,8 +29,8 @@ input-syslog:
    - opennti
 
 opennti:
-  image: $IMAGE_NAME
-  container_name: $CONTAINER_NAME
+  image: $MAIN_IMAGE_NAME:$IMAGE_TAG
+  container_name: $MAIN_CONTAINER_NAME
   volumes:
    - ./$LOCAL_DIR_DASHBOARD:/src/dashboards
    - ./$LOCAL_DIR_DATA:/opt/open-nti/data

open-nti.params

@@ -4,8 +4,13 @@
 export MAIN_IMAGE_NAME=juniper/open-nti
 export MAIN_CONTAINER_NAME=opennti_con
 
+export INPUT_JTI_DIR=plugins/input-jti
 export INPUT_JTI_IMAGE_NAME=juniper/open-nti-input-jti
+export INPUT_JTI_CONTAINER_NAME=opennti_input_jti
+
+export INPUT_SYSLOG_DIR=plugins/input-syslog
 export INPUT_SYSLOG_IMAGE_NAME=juniper/open-nti-input-syslog
+export INPUT_SYSLOG_CONTAINER_NAME=opennti_input_syslog
 
 export MAIN_CONTAINER_NAME=opennti_con
 

tests/main/pyez_mock.py

@@ -0,0 +1,58 @@
+"""PyEZ Device Mock
+:Author:  Christian Giese
+:Date:    08/03/2016
+:Based on: https://github.com/GIC-de/Juniper-PyEZ-Unit-Testing.git
+"""
+from __future__ import unicode_literals
+from mock import MagicMock, patch
+from jnpr.junos import Device
+from ncclient.manager import Manager, make_device_handler
+from ncclient.transport import SSHSession
+from ncclient.xml_ import NCElement
+import os
+
+
+# ------------------------------------------------------------------------------
+# pytest fixtures
+# ------------------------------------------------------------------------------
+
+def rpc_reply_dict():
+    """Dynamic Generated rpc-replys"""
+    return {}
+
+
+@patch('ncclient.manager.connect')
+def mocked_device(rpc_reply_dict, mock_connect):
+    """Juniper PyEZ Device Fixture"""
+    def mock_manager(*args, **kwargs):
+        if 'device_params' in kwargs:
+            # open connection
+            device_params = kwargs['device_params']
+            device_handler = make_device_handler(device_params)
+            session = SSHSession(device_handler)
+            return Manager(session, device_handler)
+        elif args:
+            # rpc request
+            rpc_request = args[0].tag
+            rpc_command = str(args[0].text)
+            rpc_command = rpc_command.strip()
+            rpc_command = rpc_command.replace(" ", "_")
+            if rpc_request in rpc_reply_dict:
+                xml = rpc_reply_dict[rpc_request]
+            elif 'dir' in rpc_reply_dict:
+                fname = os.path.join(rpc_reply_dict['dir'], 'rpc-reply', rpc_command, rpc_request + '.xml')
+                with open(fname, 'r') as f:
+                    xml = f.read()
+            else:
+                _rpc_reply_dict['dir']
+                fname = os.path.join(os.path.dirname(__file__), 'rpc-reply', rpc_command, rpc_request + '.xml')
+                with open(fname, 'r') as f:
+                    xml = f.read()
+            rpc_reply = NCElement(xml, dev._conn._device_handler.transform_reply())
+            return rpc_reply
+    mock_connect.side_effect = mock_manager
+    dev = Device(host='1.1.1.1', user='juniper', gather_facts=False)
+    dev.open()
+    dev._conn.rpc = MagicMock(side_effect=mock_manager)
+    dev.close = MagicMock()
+    return dev

tests/test_main.py

@@ -25,11 +25,11 @@
 OPENNTI_IP = ''
 
 OPENNTI_IN_JTI_NAME = 'open-nti_in_jti_test'
-OPENNTI_IN_JTI_IMAGE = 'juniper/open-nti-input-jti'
+OPENNTI_IN_JTI_IMAGE = 'juniper/open-nti-input-jti:unittest'
 OPENNTI_IN_JTI_C = ''
 
 OPENNTI_IN_LOG_NAME = 'open-nti_in_log_test'
-OPENNTI_IN_LOG_IMAGE = 'juniper/open-nti-input-syslog'
+OPENNTI_IN_LOG_IMAGE = 'juniper/open-nti-input-syslog:unittest'
 OPENNTI_IN_LOG_C = ''
 
 TCP_REPLAY_IMAGE = 'dgarros/tcpreplay'
@@ -53,11 +53,12 @@
 
 # Local directories that will be mapped into the container
 CURRENT_DIR = os.getcwd()
-LOCAL_DIR_DATA = CURRENT_DIR + '/data'
-LOCAL_DIR_LOG = CURRENT_DIR + '/logs'
-LOCAL_DIR_DB = CURRENT_DIR + '/db'
-LOCAL_DIR_TESTS = CURRENT_DIR + '/tests'
-LOCAL_DIR_DASHBOARD = CURRENT_DIR + '/dashboards'
+LOCAL_DIR_DATA = CURRENT_DIR + '/main/data'
+LOCAL_DIR_LOG = CURRENT_DIR + '/main/logs'
+LOCAL_DIR_DB = CURRENT_DIR + '/main/db'
+LOCAL_DIR_TESTS = CURRENT_DIR + '/tests/main'
+LOCAL_DIR_DASHBOARD = CURRENT_DIR + '/main/dashboards'
+
 LOCAL_PCAP_DIR = LOCAL_DIR_TESTS + '/pcap'
 LOCAL_GRAFANA_DIR = LOCAL_DIR_TESTS + '/grafana'
 LOCAL_GRAFANA_FILE = LOCAL_GRAFANA_DIR + '/test.json'
@@ -94,31 +95,6 @@ def test_connect_docker():
     if _platform == "linux" or _platform == "linux2" or _platform == "darwin":
         # linux
         c = Client(base_url='unix://var/run/docker.sock', version='1.20')
-    # elif _platform == "darwin":
-    #     # MAC OS X
-    #     dockerout = subprocess.check_output(
-    #         ['/usr/local/bin/docker-machine ip default'],
-    #         shell=True, stderr=subprocess.STDOUT
-    #     )
-    #
-    #     DOCKER_IP = dockerout.splitlines()[0]
-    #
-    #     CERTS = path.join(
-    #         path.expanduser('~'), '.docker', 'machine',
-    #         'machines', 'default'
-    #     )
-    #
-    #     tls_config = tls.TLSConfig(
-    #         client_cert=(
-    #             path.join(CERTS, 'cert.pem'), path.join(CERTS, 'key.pem')
-    #         ),
-    #         ca_cert=path.join(CERTS, 'ca.pem'),
-    #         assert_hostname=False,
-    #         verify=True
-    #     )
-    #
-    #     url = "https://" + DOCKER_IP + ":2376"
-    #     c = Client(base_url=url, tls=tls_config, version='1.20')
 
     elif _platform == "win32":
         exit
@@ -290,8 +266,8 @@ def test_influxdb_running_database_exist():
         else:
             continue
 
-# DROP CONTINUOUS QUERY "four_weeks" ON "juniper"
-# CREATE RETENTION POLICY open_nti_test ON juniper DURATION INF REPLICATION 1 DEFAULT
+    # DROP CONTINUOUS QUERY "four_weeks" ON "juniper"
+    # CREATE RETENTION POLICY open_nti_test ON juniper DURATION INF REPLICATION 1 DEFAULT
     if found_db:
         assert 1
     else:
@@ -304,52 +280,49 @@ def test_influxdb_create_default_RP():
     query = 'CREATE RETENTION POLICY "open_nti_test" ON juniper DURATION INF REPLICATION 1 DEFAULT;'
     result = db.query(query)
 
-# DROP CONTINUOUS QUERY "four_weeks" ON "juniper"
-# CREATE RETENTION POLICY open_nti_test ON juniper DURATION INF REPLICATION 1 DEFAULT
+    # DROP CONTINUOUS QUERY "four_weeks" ON "juniper"
+    # CREATE RETENTION POLICY open_nti_test ON juniper DURATION INF REPLICATION 1 DEFAULT
 
     if result:
         assert 1
 
-
-def test_collection_agent_01():
-    # Write datapoint using mocked Junos device
-    global OPENNTI_C
-
-    FIXTURES_DIR = "/opt/open-nti/tests/fixtures/test_collection_agent_01/"
-
-    exec_job_id = c.exec_create(
-        container=OPENNTI_C,
-        cmd='/usr/bin/python ' +
-        '/opt/open-nti/open-nti.py -s -t --tag test --input ' + FIXTURES_DIR
-    )
-
-    result = c.exec_start(exec_job_id, stream=True)
-    for line in result:
-        try:
-            stream_line = StreamLineBuildGenerator(line)
-            # Do something with your stream line
-            # ...
-        except ValueError:
-            # If we are not able to deserialize the received line
-            # as JSON object, just print it out
-            print(line)
-            continue
-
-    db = get_handle_db()
-
-    time.sleep(3)
-    query = 'select mean(value) from ' + \
-        '/P1-tf-mx960-1-re0.route-table.summary.inet.0.actives/;'
-    result = db.query(query)
-    points = list(result.get_points())
-
-    assert len(points) == 1 and points[0]['mean'] == 16
-
+# def test_collection_agent_01():
+#     # Write datapoint using mocked Junos device
+#     global OPENNTI_C
+#
+#     FIXTURES_DIR = "/opt/open-nti/tests/fixtures/test_collection_agent_01/"
+#
+#     exec_job_id = c.exec_create(
+#         container=OPENNTI_C,
+#         cmd='/usr/bin/python ' +
+#         '/opt/open-nti/open-nti.py -s -t --tag test --input ' + FIXTURES_DIR
+#     )
+#
+#     result = c.exec_start(exec_job_id, stream=True)
+#     for line in result:
+#         try:
+#             stream_line = StreamLineBuildGenerator(line)
+#             # Do something with your stream line
+#             # ...
+#         except ValueError:
+#             # If we are not able to deserialize the received line
+#             # as JSON object, just print it out
+#             print(line)
+#             continue
+#
+#     db = get_handle_db()
+#
+#     time.sleep(3)
+#     query = 'select mean(value) from ' + \
+#         '/P1-tf-mx960-1-re0.route-table.summary.inet.0.actives/;'
+#     result = db.query(query)
+#     points = list(result.get_points())
+#
+#     assert len(points) == 1 and points[0]['mean'] == 16
 
 def test_start_tcpreplay_container():
     global TCP_REPLAY_C
 
-
     # Force Stop and delete existing container if exist
     try:
         old_container_id = c.inspect_container(TCP_REPLAY_NAME)['Id']