diff --git a/ngsi_adapter/src/test/acceptance/README.rst b/ngsi_adapter/src/test/acceptance/README.rst index 82719e4..f505d3b 100644 --- a/ngsi_adapter/src/test/acceptance/README.rst +++ b/ngsi_adapter/src/test/acceptance/README.rst @@ -11,8 +11,8 @@ Gherkin has the purpose of serving documentation of test cases. Test case implementation has been performed using `Python `_ and `Lettuce `_. -Project Structure ------------------ +Acceptance Project Structure +---------------------------- :: ├───acceptance @@ -26,3 +26,71 @@ Project Structure │ │ └───probe_sample_data │ └───settings │ + + +FIWARE Monitoring Automation Framework +--------------------------------------- + +Features: + +- Lettuce-Tools support +- Settings using json files and Lettuce-Tools utility +- Test report using Lettuce-Tools XUnit output +- NGSI-Adapter Client +- Logging +- Remote NGSI-Adapter log capturing +- Test data management using templates (resources) + + +Acceptance test execution +------------------------- + +Execute the following command in the test project root directory: + +:: + + $> cd ngsi_adapter/src/test/acceptance + $> lettuce_tools -ft send_data_api_resource -ts comp -sd features/ --tags=-skip -en dev + +With this command, you will execute: + +- components Test Cases in the 'Development' environment configured in settings/dev-properties.json +- the send_data_api_resource feature +- Skipping all Scenarios with tagged with "skip" + + +**Prerequisites** + +- Python 2.7 or newer (2.x) (https://www.python.org/downloads/) +- pip (https://pypi.python.org/pypi/pip) +- virtualenv (https://pypi.python.org/pypi/virtualenv) +- Monitoring [NGSI-Adapter] (`Download NGSI-Adapter `_) + +**Test case execution using virtualenv** + +1. Create a virtual environment somewhere *(virtualenv $WORKON_HOME/venv)* +#. Activate the virtual environment *(source $WORKON_HOME/venv/bin/activate)* +#. Go to *ngsi_adapter/src/test/acceptance* folder in the project +#. Install the requirements for the acceptance tests in the virtual environment *(pip install -r requirements.txt --allow-all-external)* + +**Test case execution using Vagrant (optional)** + +Instead of using virtualenv, you can use the provided Vagrantfile to deploy a local VM using `Vagrant `_, +that will provide all environment configurations for launching test cases. + +1. Download and install Vagrant (https://www.vagrantup.com/downloads.html) +#. Go to *ngsi_adapter/src/test/acceptance* folder in the project +#. Execute *vagrant up* to launch a VM based on Vagrantfile provided. +#. After Vagrant provision, your VM is properly configured to launch acceptance tests. You have to access to the VM using +*vagrant ssh* and change to */vagrant* directory that will have mounted your workspace *(test/acceptance)*. + +If you need more information about how to use Vagrant, you can see +`Vagrant Getting Started `_ + +**Settings** + +Before executing the acceptance tests, you will need configure the properties file. To execute acceptance test on the +experimentation environment, you will have to configured the file *settings/dev-properties*. + +You will need a valid private key (*private_key_location*) to connect to NGSI-Adapter Host to capture remote logs. +In this way, you will be able to execute Scenarios that require the logs capturing for test validations. diff --git a/ngsi_adapter/src/test/acceptance/Vagrantfile b/ngsi_adapter/src/test/acceptance/Vagrantfile new file mode 100644 index 0000000..aef9292 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/Vagrantfile @@ -0,0 +1,46 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +#__author__ = 'jfernandez' + + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + # Every Vagrant virtual environment requires a box to build off of. + # Box: https://atlas.hashicorp.com/hashicorp/boxes/precise32 + config.vm.box = "hashicorp/precise32" + + # Provision + config.vm.provision "shell", inline: "cd /home/vagrant", privileged: true + config.vm.provision "shell", inline: "wget https://bootstrap.pypa.io/get-pip.py", privileged: true + config.vm.provision "shell", inline: "python get-pip.py", privileged: true + config.vm.provision "shell", inline: "apt-get update", privileged: true + config.vm.provision "shell", inline: "apt-get -y install python-dev", privileged: true + config.vm.provision "shell", inline: "apt-get -y install git", privileged: true + config.vm.provision "shell", inline: "apt-get -y install libxml2-dev libxslt1-dev", privileged: true + config.vm.provision "shell", inline: "pip install -r /vagrant/requirements.txt", privileged: true +end diff --git a/ngsi_adapter/src/test/acceptance/commons/constants.py b/ngsi_adapter/src/test/acceptance/commons/constants.py new file mode 100644 index 0000000..fb51e25 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/constants.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + + +# HEADERS +HEADER_CONTENT_TYPE = u'content-type' +HEADER_ACCEPT = u'accept' +HEADER_REPRESENTATION_JSON = u'application/json' +HEADER_REPRESENTATION_XML = u'application/xml' +HEADER_REPRESENTATION_TEXTPLAIN = u'text/plain' +HEADER_AUTH_TOKEN = u'X-Auth-Token' +HEADER_TENANT_ID = u'Tenant-Id' +HEADER_TRANSACTION_ID = u'txid' + +# HTTP VERBS +HTTP_VERB_POST = 'post' +HTTP_VERB_GET = 'get' +HTTP_VERB_PUT = 'put' +HTTP_VERB_DELETE = 'delete' +HTTP_VERB_UPDATE = 'update' + +# TRANSACTION ID +TRANSACTION_ID_PATTERN = "qa/{uuid}" + +# NGSI CLIENT +NGSI_ADAPTER_URI_BASE = "{api_root_url}" +NGSI_ADAPTER_URI_PARSER = NGSI_ADAPTER_URI_BASE + "/{parser_name}" +NGSI_ADAPTER_PARAMETER_ID = "id" +NGSI_ADAPTER_PARAMETER_TYPE = "type" + +# CONFIGURATION PROPERTIES +PROPERTIES_FILE = "properties.json" +PROPERTIES_CONFIG_ENV = "environment" +PROPERTIES_CONFIG_ENV_NAME = "name" +PROPERTIES_CONFIG_ENV_LOGS_PATH = "log_path" +PROPERTIES_CONFIG_ENV_LOCAL_PATH_REMOTE_LOGS = "local_path_remote_logs" +MONITORING_CONFIG_ENV_DEFAULT_PARSER = "default_parser" +MONITORING_CONFIG_ENV_DEFAULT_PARSER_DATA = "default_parser_data" +MONITORING_CONFIG_ENV_DEFAULT_PARSER_PARAMS = "default_parser_parameters" +MONITORING_CONFIG_SERVICE_ADAPTER = "monitoring_adapter_service" +MONITORING_CONFIG_SERVICE_PROTOCOL = "protocol" +MONITORING_CONFIG_SERVICE_HOST = "host" +MONITORING_CONFIG_SERVICE_PORT = "port" +MONITORING_CONFIG_SERVICE_RESOURCE = "resource" +MONITORING_CONFIG_SERVICE_PRIVATEKEY = "private_key_location" +MONITORING_CONFIG_SERVICE_HOSTUSER = "host_user" +MONITORING_CONFIG_SERVICE_HOSTPASSWORD = "host_password" +MONITORING_CONFIG_SERVICE_LOG_PATH = "service_log_path" +MONITORING_CONFIG_SERVICE_LOG_FILE_NAME = "service_log_file_name" + +# RESOURCES +RESOURCES_SAMPLEDATA_MODULE = "resources.probe_sample_data" +RESOURCES_PARAMETER_PATTERN = "${param_name}" diff --git a/ngsi_adapter/src/test/acceptance/commons/dataset_utils.py b/ngsi_adapter/src/test/acceptance/commons/dataset_utils.py new file mode 100644 index 0000000..3be4bc0 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/dataset_utils.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + + +from lettuce_tools.dataset_utils.dataset_utils import DatasetUtils + +dataset_utils = DatasetUtils() + + +def prepare_param(param): + """ + Generate a fixed length data for elements tagged with the text [LENGTH] in lettuce + Removes al the data elements tagged with the text [MISSING_PARAM] in lettuce + :param param: Lettuce parameter + :return data without not desired params + """ + + if "[MISSING_PARAM]" in param: + new_param = None + else: + new_param = dataset_utils.generate_fixed_length_param(param) + + return new_param diff --git a/ngsi_adapter/src/test/acceptance/commons/logger_utils.py b/ngsi_adapter/src/test/acceptance/commons/logger_utils.py new file mode 100644 index 0000000..46aa403 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/logger_utils.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + +import logging +import logging.config +import xml +import json +from constants import HEADER_CONTENT_TYPE, HEADER_REPRESENTATION_XML, HEADER_REPRESENTATION_JSON + +""" +Part of this code has been taken from: + https://pdihub.hi.inet/fiware/fiware-iotqaUtils/raw/develop/iotqautils/iotqaLogger.py +""" + +LOG_CONSOLE_FORMATTER = " %(asctime)s - %(name)s - %(levelname)s - %(message)s" +LOG_FILE_FORMATTER = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + +# Console logging level. By default: ERROR +logging.config.fileConfig("./settings/logging.conf") +logging_level = logging.ERROR + + +def configure_logging(level): + """ + Configures global log level to given one + :param level: Level (INFO | DEBUG | WARN | ERROR) + :return: + """ + + global logging_level + logging_level = logging.ERROR + if "info" == level.lower(): + logging_level = logging.INFO + elif "warn" == level.lower(): + logging_level = logging.WARNING + elif "debug" == level.lower(): + logging_level = logging.DEBUG + + +def get_logger(name): + """ + Creates new logger with the given name + :param name: Name of the logger + :return: Logger + """ + + #logging.config.fileConfig("logging.conf") + logger = logging.getLogger("testingLogger") + + # if not len(logger.handlers): + # # File handler + # file_hdlr = logging.FileHandler('logs/monitoring_tests.log') + # formatter = logging.Formatter(LOG_FILE_FORMATTER) + # file_hdlr.setFormatter(formatter) + # logger.addHandler(file_hdlr) + # logger.setLevel(logging.DEBUG) + # + # # Console hadler + # console_hdlr = logging.StreamHandler() + # formatter = logging.Formatter(LOG_CONSOLE_FORMATTER) + # console_hdlr.setFormatter(formatter) + # logger.addHandler(console_hdlr) + # logger.setLevel(logging_level) + + return logger + + +def _get_pretty_body(headers, body): + """ + Returns a pretty printed body using the Content-Type header information + :param headers: Headers for the request/response (dict) + :param body: Body to pretty print (string) + :return: Body pretty printed (string) + """ + + if HEADER_CONTENT_TYPE in headers: + if HEADER_REPRESENTATION_XML == headers[HEADER_CONTENT_TYPE]: + xml_parsed = xml.dom.minidom.parseString(body) + pretty_xml_as_string = xml_parsed.toprettyxml() + return pretty_xml_as_string + else: + if HEADER_REPRESENTATION_JSON in headers[HEADER_CONTENT_TYPE]: + parsed = json.loads(body) + return json.dumps(parsed, sort_keys=True, indent=4) + else: + return body + else: + return body + + +def log_print_request(logger, method, url, query_params=None, headers=None, body=None): + """ + Logs an HTTP request data. + :param logger: Logger to use + :param method: HTTP method + :param url: URL + :param query_params: Query parameters in the URL + :param headers: Headers (dict) + :param body: Body (raw body, string) + :return: None + """ + + log_msg = '>>>>>>>>>>>>>>>>>>>>> Request >>>>>>>>>>>>>>>>>>> \n' + log_msg += '\t> Method: %s\n' % method + log_msg += '\t> Url: %s\n' % url + if query_params is not None: + log_msg += '\t> Query params: {}\n'.format(str(query_params)) + if headers is not None: + log_msg += '\t> Headers: {}\n'.format(str(headers)) + if body is not None: + log_msg += '\t> Payload sent:\n {}\n'.format(_get_pretty_body(headers, body)) + + logger.debug(log_msg) + + +def log_print_response(logger, response): + """ + Logs an HTTP response data + :param logger: logger to use + :param response: HTTP response ('Requests' lib) + :return: None + """ + + log_msg = '<<<<<<<<<<<<<<<<<<<<<< Response <<<<<<<<<<<<<<<<<<\n' + log_msg += '\t< Response code: {}\n'.format(str(response.status_code)) + log_msg += '\t< Headers: {}\n'.format(str(dict(response.headers))) + try: + log_msg += '\t< Payload received:\n {}'.format(_get_pretty_body(dict(response.headers), response.content)) + except ValueError: + log_msg += '\t< Payload received:\n {}'.format(_get_pretty_body(dict(response.headers), response.content.text)) + + logger.debug(log_msg) diff --git a/ngsi_adapter/src/test/acceptance/commons/ngsi_adapter_api_utils/__init__.py b/ngsi_adapter/src/test/acceptance/commons/ngsi_adapter_api_utils/__init__.py new file mode 100644 index 0000000..53a9ce8 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/ngsi_adapter_api_utils/__init__.py @@ -0,0 +1 @@ +__author__ = 'jfernandez' diff --git a/ngsi_adapter/src/test/acceptance/commons/ngsi_adapter_api_utils/ngsi_adapter_client.py b/ngsi_adapter/src/test/acceptance/commons/ngsi_adapter_api_utils/ngsi_adapter_client.py new file mode 100644 index 0000000..945fd7e --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/ngsi_adapter_api_utils/ngsi_adapter_client.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FI-WARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + + +from commons.rest_client_utils import RestClient, API_ROOT_URL_ARG_NAME +from commons.constants import HEADER_REPRESENTATION_TEXTPLAIN, HEADER_CONTENT_TYPE, HEADER_TRANSACTION_ID, \ + HTTP_VERB_POST +from commons.utils import generate_transaction_id +from commons.logger_utils import get_logger + +NGSI_ADAPTER_URI_BASE = "{" + API_ROOT_URL_ARG_NAME + "}" +NGSI_ADAPTER_URI_PARSER = NGSI_ADAPTER_URI_BASE + "/{probe_name}" +NGSI_ADAPTER_PARAMETER_ID = "id" +NGSI_ADAPTER_PARAMETER_TYPE = "type" + +logger = get_logger("rest_client_utils") + + +class NgsiAdapterClient: + + headers = dict() + + def __init__(self, protocol, host, port, base_resource=None): + """ + Class constructor. Init default headers + :param protocol: API Protocol + :param host: API Host + :param port: API Port + :param base_resource: base uri resource (if exists) + :return: None + """ + + self.init_headers() + self.rest_client = RestClient(protocol, host, port, base_resource) + + def init_headers(self, content_type=HEADER_REPRESENTATION_TEXTPLAIN, transaction_id=generate_transaction_id()): + """ + Init header to values (or default values) + :param content_type: Content-Type header value. By default text/plain + :param transaction_id: txId header value. By default, generated value by Utils.generate_transaction_id() + :return: None + """ + + if content_type is None: + if HEADER_CONTENT_TYPE in self.headers: + del(self.headers[HEADER_CONTENT_TYPE]) + else: + self.headers.update({HEADER_CONTENT_TYPE: content_type}) + + if transaction_id is None: + if HEADER_TRANSACTION_ID in self.headers: + del(self.headers[HEADER_TRANSACTION_ID]) + else: + self.headers.update({HEADER_TRANSACTION_ID: transaction_id}) + + def set_headers(self, headers): + """ + Set header. + :param headers: Headers to be used by next request (dict) + :return: None + """ + + self.headers = headers + + def send_raw_data(self, raw_data, probe_name, entity_id, entity_type): + """ + Execute a well-formed POST request. All parameters are mandatory + :param raw_data: Raw probe data to send (string, text/plain) + :param probe_name: Parser to be used (string) + :param entity_id: Entity ID (string) + :param entity_type: Entity Type (string) + :return: HTTP Request response ('Requests' lib) + """ + + logger.info("Sending raw data to NGSI-Adapter [Probe: %s, EntityId: %, EntityType: %s", probe_name, + entity_id, entity_type) + parameters = dict() + parameters.update({NGSI_ADAPTER_PARAMETER_ID: entity_id}) + parameters.update({NGSI_ADAPTER_PARAMETER_TYPE: entity_type}) + return self.rest_client.post(uri_pattern=NGSI_ADAPTER_URI_PARSER, body=raw_data, headers=self.headers, + parametersn=parameters, probe_name=probe_name) + + def send_raw_data_custom(self, raw_data, probe_name=None, entity_id=None, entity_type=None, + http_method=HTTP_VERB_POST): + """ + Execute a 'send_data' request (POST request by default). Should support all testing cases. + The generated request could be malformed (Testing purpose) + Parameters with None value will not be in the generated request (missing parameter). + :param raw_data: Raw probe data to send (string, text/plain) + :param probe_name: Parser to be used (string) + :param entity_id: Entity ID (string) + :param entity_type: Entity Type (string) + :param http_method: send raw data is a HTTP POST request but, for testing purposes could be interesting to use + another HTTP verb. By default is defined to 'post' + :return: HTTP Request response ('Requests' lib) + """ + + logger.info("Sending raw data to NGSI-Adapter (custom operation for testing purpose)") + parameters = dict() + if entity_id is not None: + parameters.update({NGSI_ADAPTER_PARAMETER_ID: entity_id}) + + if entity_type is not None: + parameters.update({NGSI_ADAPTER_PARAMETER_TYPE: entity_type}) + + if probe_name is not None: + return self.rest_client.launch_request(uri_pattern=NGSI_ADAPTER_URI_PARSER, body=raw_data, + method=http_method, headers=self.headers, parameters=parameters, + probe_name=probe_name) + else: + return self.rest_client.launch_request(uri_pattern=NGSI_ADAPTER_URI_BASE, body=raw_data, + method=http_method, headers=self.headers, parameters=parameters) diff --git a/ngsi_adapter/src/test/acceptance/commons/remote_tail_utils.py b/ngsi_adapter/src/test/acceptance/commons/remote_tail_utils.py new file mode 100644 index 0000000..bfa2de7 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/remote_tail_utils.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + + +from sshtail import SSHTailer, load_dss_key +import time +import threading +from logger_utils import get_logger + + +logger = get_logger("remote_tail_utils") + +# Delay period just after starting remote tailers +TIMER_DELAY_PERIOD = 3 + +# Grace period when stopping thread. 3 seconds by default +TIMER_GRACE_PERIOD = 3 + +# Global flag +_tail_terminate_flag = False + + +class RemoteTail: + + def __init__(self, remote_host_ip, remote_host_user, remote_log_path, remote_log_file_name, local_log_target, + private_key): + """ + Inits RemoteTail class + :param remote_host_ip: Remote Host IP + :param remote_host_user: Remote host User name + :param remote_log_path: Remote log path location + :param remote_log_file_name: Remote log filename to be tailed + :param local_log_target: Local path where remote logs will be captured + :param private_key: Private key to use in the SSH connection. + If no path's specified for the private key file name, it automatically prepends /home//.ssh/ + and for RSA keys, import load_rsa_key instead. + :param service_name: Output log file naming (optional) + :return: None + """ + + self.tailer = None + self.tail_terminate_flag = False + self.thread = None + self.local_capture_file_descriptor = None + + self.remote_host_ip = remote_host_ip + self.remote_host_user = remote_host_user + self.remote_log_path = remote_log_path + self.remote_log_file_name = remote_log_file_name + self.local_log_target = local_log_target + self.private_key = private_key + + def init_tailer_connection(self): + """ + Creates ssh connection to host and init tail on the file. + :return: None + """ + + private_key_loaded = load_dss_key(self.private_key) + connection_host = self.remote_host_user + '@' + self.remote_host_ip + target_log_path = self.remote_log_path + self.remote_log_file_name + logger.info("Remote Tailer: Connecting to remote host [host: %s, path: %s", connection_host, + target_log_path) + self.tailer = SSHTailer(connection_host, target_log_path, private_key_loaded) + + # Open local output file + local_capture_path = self.local_log_target + self.remote_log_file_name + logger.debug("Remote Tailer: Opening local file to save the captured logs") + self.local_capture_file_descriptor = open(local_capture_path, 'w') + + def start_tailer(self): + """ + This method starts a new thread for execute a tailing on the remote log file + :return: None + """ + + logger.debug("Remote Tailer: Launching thread to capture logs") + self.thread = threading.Thread(target=_read_tailer, args=[self.tailer, self.local_capture_file_descriptor]) + self.thread.start() + logger.debug("Delay timer before starting: " + str(TIMER_DELAY_PERIOD)) + time.sleep(TIMER_DELAY_PERIOD) + + def stop_tailer(self): + """ + This method will stop the tailer process after a grace time period + :return: None + """ + + logger.info("Remote Tailer: Stopping tailers") + global _tail_terminate_flag + logger.debug("Grace period after stopping: " + str(TIMER_GRACE_PERIOD)) + time.sleep(TIMER_GRACE_PERIOD) + _tail_terminate_flag = True + + +def _read_tailer(tailer, local_capture_file_descriptor): + """ + Execute a 'tail' on remote log file until tail_terminate_flag will be True + :param tailer: Created and initialized sshtail connection + :param local_capture_file_descriptor: Opened descriptor to local file where remote logs will be captured + :return: None + """ + + global _tail_terminate_flag + _tail_terminate_flag = False + + try: + while not _tail_terminate_flag: + for line in tailer.tail(): + local_capture_file_descriptor.writelines(line + "\n") + local_capture_file_descriptor.flush() + + # wait a bit + time.sleep(0.5) + + logger.debug("Remote Tailer: Remote capture finished") + except: + logger.error("Remote Tailer: Error when reading remote log lines") + + logger.debug("Remote Tailer: Closing connections and file descriptors") + tailer.disconnect() + local_capture_file_descriptor.close() diff --git a/ngsi_adapter/src/test/acceptance/commons/rest_client_utils.py b/ngsi_adapter/src/test/acceptance/commons/rest_client_utils.py new file mode 100644 index 0000000..bdef1d4 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/rest_client_utils.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + + +import requests +import xmltodict +import xmldict +from json import JSONEncoder +from constants import HEADER_REPRESENTATION_JSON, HEADER_REPRESENTATION_XML, HTTP_VERB_POST, HTTP_VERB_DELETE, \ + HTTP_VERB_GET, HTTP_VERB_PUT, HTTP_VERB_UPDATE +from logger_utils import get_logger, log_print_request, log_print_response + +API_ROOT_URL_ARG_NAME = 'api_root_url' +URL_ROOT_PATTERN = "{protocol}://{host}:{port}" + +logger = get_logger("rest_client_utils") + + +class RestClient(object): + + api_root_url = None + + def __init__(self, protocol, host, port, resource=None): + """ + This method init the RestClient with an URL ROOT Pattern using the specified params + :param protocol: Web protocol [HTTP | HTTPS] (string) + :param host: Hostname or IP (string) + :param port: Service port (string) + :param resource: Base URI resource, if exists (string) + :return: None + """ + + self.api_root_url = self._generate_url_root(protocol, host, port) + if resource is not None: + self.api_root_url += "/" + resource + + @staticmethod + def _generate_url_root(protocol, host, port): + """ + Generates API root URL without resources + :param protocol: Web protocol [HTTP | HTTPS] (string) + :param host: Hostname or IP (string) + :param port: Service port (string) + :return: ROOT url + """ + return URL_ROOT_PATTERN.format(protocol=protocol, host=host, port=port) + + def _call_api(self, uri_pattern, method, body=None, headers=None, parameters=None, **kwargs): + """ + Launch HTTP request to the API with given arguments + :param uri_pattern: string pattern of the full API url with keyword arguments (format string syntax) + :param method: HTTP method to execute (string) [get | post | put | delete | update] + :param body: Raw Body content (string) (Plain/XML/JSON to be sent) + :param headers: HTTP header request (dict) + :param parameters: Query parameters for the URL. i.e. {'key1': 'value1', 'key2': 'value2'} + :param **kwargs: URL parameters (without API_ROOT_URL_ARG_NAME) to fill the patters + :returns: REST API response ('Requests' response) + """ + + logger.info("Executing API request [%s %s]", method, uri_pattern) + kwargs[API_ROOT_URL_ARG_NAME] = self.api_root_url + url = uri_pattern.format(**kwargs) + + log_print_request(logger, method, url, parameters, headers, body) + + try: + response = requests.request(method=method, url=url, data=body, headers=headers, params=parameters, + verify=False) + except Exception, e: + logger.error("Request {} to {} crashed: {}".format(method, url, str(e))) + raise e + + log_print_response(logger, response) + + return response + + def launch_request(self, uri_pattern, body, method, headers=None, parameters=None, **kwargs): + """ + Launch HTTP request to the API with given arguments + :param uri_pattern: string pattern of the full API url with keyword arguments (format string syntax) + :param body: Raw Body content (string) (Plain/XML/JSON to be sent) + :param method: HTTP ver to be used in the request [GET | POST | PUT | DELETE | UPDATE ] + :param headers: HTTP header (dict) + :param parameters: Query parameters for the URL. i.e. {'key1': 'value1', 'key2': 'value2'} + :param **kwargs: URL parameters (without url_root) to fill the patters + :returns: REST API response ('Requests' response) + """ + return self._call_api(uri_pattern, method, body, headers, parameters, **kwargs) + + def get(self, uri_pattern, headers=None, parameters=None, **kwargs): + """ + Launch HTTP GET request to the API with given arguments + :param uri_pattern: string pattern of the full API url with keyword arguments (format string syntax) + :param headers: HTTP header (dict) + :param parameters: Query parameters. i.e. {'key1': 'value1', 'key2': 'value2'} + :param **kwargs: URL parameters (without url_root) to fill the patters + :returns: REST API response ('Requests' response) + """ + return self._call_api(uri_pattern, HTTP_VERB_GET, headers=headers, parameters=parameters, **kwargs) + + def post(self, uri_pattern, body, headers=None, parameters=None, **kwargs): + """ + Launch HTTP POST request to the API with given arguments + :param uri_pattern: string pattern of the full API url with keyword arguments (format string syntax) + :param body: Raw Body content (string) (Plain/XML/JSON to be sent) + :param headers: HTTP header (dict) + :param parameters: Query parameters. i.e. {'key1': 'value1', 'key2': 'value2'} + :param **kwargs: URL parameters (without url_root) to fill the patters + :returns: REST API response ('Requests' response) + """ + return self._call_api(uri_pattern, HTTP_VERB_POST, body, headers, parameters, **kwargs) + + def put(self, uri_pattern, body, headers=None, parameters=None, **kwargs): + """ + Launch HTTP PUT request to the API with given arguments + :param uri_pattern: string pattern of the full API url with keyword arguments (format string syntax) + :param body: Raw Body content (string) (Plain/XML/JSON to be sent) + :param headers: HTTP header (dict) + :param parameters: Query parameters. i.e. {'key1': 'value1', 'key2': 'value2'} + :param **kwargs: URL parameters (without url_root) to fill the patters + :returns: REST API response ('Requests' response) + """ + return self._call_api(uri_pattern, HTTP_VERB_PUT, body, headers, parameters, **kwargs) + + def delete(self, uri_pattern, headers=None, parameters=None, **kwargs): + """ + Launch HTTP DELETE request to the API with given arguments + :param uri_pattern: string pattern of the full API url with keyword arguments (format string syntax) + :param headers: HTTP header (dict) + :param parameters: Query parameters. i.e. {'key1': 'value1', 'key2': 'value2'} + :param **kwargs: URL parameters (without url_root) to fill the patters + :returns: REST API response ('Requests' response) + """ + return self._call_api(uri_pattern, HTTP_VERB_DELETE, headers=headers, parameters=parameters, **kwargs) + + +def _xml_to_dict(xml_to_convert): + """ + Converts RAW XML string to Python dict + :param xml_to_convert: XML to convert (string/text) + :return: Python dict with all XML data + """ + + logger.debug("Converting XML to Python dict") + return xmltodict.parse(xml_to_convert, attr_prefix='') + + +def _dict_to_xml(dict_to_convert): + """ + Converts Python dict to XML + :param dict_to_convert: Python dict to be converted (dict) + :return: XML (string) + """ + + logger.debug("Converting Python dict to XML") + return xmldict.dict_to_xml(dict_to_convert) + + +def response_body_to_dict(http_requests_response, content_type, xml_root_element_name=None, is_list=False): + """ + Method to convert a XML or JSON response in a Python dict + :param http_requests_response: 'Requests (lib)' response + :param content_type: Expected content-type header value (Accept header value in the request) + :param xml_root_element_name: For XML requests. XML root element in response. + :param is_list: For XML requests. If response is a list, a True value will delete list node name + :return: Python dict with response. + """ + + logger.info("Converting response body from API (XML or JSON) to Python dict") + if HEADER_REPRESENTATION_JSON == content_type: + try: + return http_requests_response.json() + except Exception, e: + logger.error("Error parsing the response to JSON. Exception:" + str(e)) + raise e + else: + assert xml_root_element_name is not None,\ + "xml_root_element_name is a mandatory param when body is in XML" + + try: + response_body = _xml_to_dict(http_requests_response.content)[xml_root_element_name] + except Exception, e: + logger.error("Error parsing the response to XML. Exception: " + str(e)) + raise e + + if is_list and response_body is not None: + response_body = response_body.popitem()[1] + + return response_body + + +def model_to_request_body(body_model, content_type, body_model_root_element=None): + """ + Converts a Python dict (body model) to XML or JSON + :param body_model: Model to be parsed. This model should have a root element. + :param content_type: Target representation (Content-Type header value) + :param body_model_root_element: For XML requests. XML root element in the model (if exists). + :return: + """ + + logger.info("Converting body request model (Python dict) to JSON or XML") + if HEADER_REPRESENTATION_XML == content_type: + try: + return _dict_to_xml(body_model) + except Exception, e: + logger.error("Error parsing the body model to XML. Exception: " + str(e)) + raise e + else: + body_json = body_model[body_model_root_element] if body_model_root_element is not None else body_model + encoder = JSONEncoder() + + try: + return encoder.encode(body_json) + except Exception, e: + logger.error("Error parsing the body model to JSON. Exception:" + str(e)) + raise e diff --git a/ngsi_adapter/src/test/acceptance/commons/terrain_utils.py b/ngsi_adapter/src/test/acceptance/commons/terrain_utils.py new file mode 100644 index 0000000..4fc5f17 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/terrain_utils.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + + +from lettuce import world +from logger_utils import get_logger +import os +import sys +import json +from remote_tail_utils import RemoteTail +from constants import PROPERTIES_FILE, PROPERTIES_CONFIG_ENV, PROPERTIES_CONFIG_ENV_LOGS_PATH, \ + PROPERTIES_CONFIG_ENV_LOCAL_PATH_REMOTE_LOGS, MONITORING_CONFIG_SERVICE_PRIVATEKEY, \ + MONITORING_CONFIG_SERVICE_LOG_PATH, MONITORING_CONFIG_SERVICE_HOST, MONITORING_CONFIG_SERVICE_HOSTUSER, \ + MONITORING_CONFIG_SERVICE_ADAPTER, MONITORING_CONFIG_SERVICE_LOG_FILE_NAME + +logger = get_logger("terrain_utils") + + +def _load_project_properties(): + """ + Parse the JSON configuration file located in the src folder and + store the resulting dictionary in the lettuce world global variable. + """ + + logger.debug("Loading test properties") + with open(PROPERTIES_FILE) as config_file: + try: + world.config = json.load(config_file) + except Exception, e: + logger.error('Error parsing config file: %s' % e) + sys.exit(1) + + +def set_up(): + """ + Setup execution and configure global test parameters and environment. + Init the capture from remote logs + :return: None + """ + + logger.info("Setting up test execution") + _load_project_properties() + + """ + Make sure the logs path exists and create it otherwise. + """ + logger.debug("Generating log directories if not exist") + log_path = world.config[PROPERTIES_CONFIG_ENV][PROPERTIES_CONFIG_ENV_LOGS_PATH] + if not os.path.exists(log_path): + os.makedirs(log_path) + + log_path = world.config[PROPERTIES_CONFIG_ENV][PROPERTIES_CONFIG_ENV_LOCAL_PATH_REMOTE_LOGS] + if not os.path.exists(log_path): + os.makedirs(log_path) + + # Init remote logs capturing + logger.info("Initiating remote log capture") + remote_host_ip = world.config[MONITORING_CONFIG_SERVICE_ADAPTER][MONITORING_CONFIG_SERVICE_HOST] + remote_host_user = world.config[MONITORING_CONFIG_SERVICE_ADAPTER][MONITORING_CONFIG_SERVICE_HOSTUSER] + service_log_path = world.config[MONITORING_CONFIG_SERVICE_ADAPTER][MONITORING_CONFIG_SERVICE_LOG_PATH] + service_log_file_name = world.config[MONITORING_CONFIG_SERVICE_ADAPTER][MONITORING_CONFIG_SERVICE_LOG_FILE_NAME] + private_key = world.config[MONITORING_CONFIG_SERVICE_ADAPTER][MONITORING_CONFIG_SERVICE_PRIVATEKEY] + world.remote_tail_client = RemoteTail(remote_host_ip, remote_host_user, service_log_path, + service_log_file_name, log_path, private_key) + world.remote_tail_client.init_tailer_connection() + world.remote_tail_client.start_tailer() + + +def tear_down(): + """ + Tear down test execution process. + Stop the capture from remote logs + :return: + """ + + logger.info("Stopping remote log capture") + world.remote_tail_client.stop_tailer() diff --git a/ngsi_adapter/src/test/acceptance/commons/utils.py b/ngsi_adapter/src/test/acceptance/commons/utils.py new file mode 100644 index 0000000..f8c0506 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/commons/utils.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + + +import uuid +from pkg_resources import resource_string +from logger_utils import get_logger +from constants import TRANSACTION_ID_PATTERN, RESOURCES_SAMPLEDATA_MODULE, RESOURCES_PARAMETER_PATTERN + +logger = get_logger("utils") + + +def generate_transaction_id(): + """ + Generate a transaction ID value following defined pattern. + :return: New transactionId + """ + + return TRANSACTION_ID_PATTERN.format(uuid=uuid.uuid4()) + + +def get_probe_data_from_resource_file(filename, replacement_values=None): + """ + Get probe data from resource files. If replacement_values is not empty, + :param filename: Resource filename to be used for loading probe data + :param param_values: (key, value) pairs. (list of dict) + :return: File content with param value replacements + """ + + filename = filename + ".txt" if ".txt" not in filename else filename + logger.debug("Getting resource file content [Filename: %s]", filename) + file_content = resource_string(RESOURCES_SAMPLEDATA_MODULE, filename) + + if replacement_values is not None: + logger.debug("Configuring template [Params: %s]", str(replacement_values)) + for param in replacement_values: + file_content = file_content.replace(RESOURCES_PARAMETER_PATTERN.replace('param_name', param['key']), + param['value']) + + return file_content diff --git a/ngsi_adapter/src/test/acceptance/features/component/send_data/send_data_api_resource.feature b/ngsi_adapter/src/test/acceptance/features/component/send_data/send_data_api_resource.feature index a794e4b..70c1407 100644 --- a/ngsi_adapter/src/test/acceptance/features/component/send_data/send_data_api_resource.feature +++ b/ngsi_adapter/src/test/acceptance/features/component/send_data/send_data_api_resource.feature @@ -14,7 +14,7 @@ Feature: Sending probe data Scenario: Valid probe data is sent to CB using a not existing parser - Given the probe name "qa_probe" + Given the probe name "qa_probe_not_existing" And the monitored resource with id "qa:1234567890" and type "host" When I send raw data according to the selected probe Then the response status code is "404" @@ -80,10 +80,11 @@ Feature: Sending probe data | a | | B | | 12345678 | - | qa.parser | - | qa-parser | - | qa_parser | - | qa@parser | + | qa.probe | + | qa-probe | + | qa_probe | + | qa@probe | + @skip @CLAUDIA-4468 @CLAUDIA-4469 Scenario Outline: Valid probe data is sent to CB using an existing parser, with invalid entity ID values. @@ -99,6 +100,7 @@ Feature: Sending probe data | [MISSING_PARAM] | + @skip @CLAUDIA-4468 @CLAUDIA-4469 Scenario Outline: Valid probe data is sent to CB using an existing parser, with invalid entity TYPE values. Given the probe name "qa_probe" And the monitored resource with id "qa:1234567890" and type "" @@ -122,9 +124,8 @@ Feature: Sending probe data Scenario Outline: Valid probe data is sent to CB using an unsupported HTTP method Given the probe name "qa_probe" And the monitored resource with id "qa:1234567890" and type "host" - And http operation is "" - When I send raw data according to the selected probe - Then the response status code is "400" + When I send raw data according to the selected probe with "" HTTP operation + Then the response status code is "405" Examples: | http_verb | @@ -137,7 +138,7 @@ Feature: Sending probe data Given the probe name "qa_probe" And the monitored resource with id "qa:1234567890" and type "host" And the header Transaction-Id "" - When I send valid raw data according to the selected probe + When I send raw data according to the selected probe Then the response status code is "200" And the given Transaction-Id value is used in logs @@ -150,19 +151,15 @@ Feature: Sending probe data | 123-456 | | ABC_1av | + Scenario Outline: NGSI-Adapter generates new transaction-id value when header is missing or empty - Given the probe name "qa_probe" + Given the probe name "" And the monitored resource with id "qa:1234567890" and type "host" And the header Transaction-Id "" - When I send valid raw data according to the selected probe - Then the response status code is "200" - And new Transaction-Id value is used in logs + When I send raw data according to the selected probe + Then an auto-generated Transaction-Id value is used in logs Examples: - | transaction_id | - | 1 | - | 1231asdfgasd | - | a/12345.qa | - | ABCDEFG#123 | - | 123-456 | - | ABC_1av | + | transaction_id | probe_name | + | | no_transaction | + | [MISSING_PARAM] | no_transaction2 | diff --git a/ngsi_adapter/src/test/acceptance/features/component/send_data/steps.py b/ngsi_adapter/src/test/acceptance/features/component/send_data/steps.py index a000e2c..76fae07 100644 --- a/ngsi_adapter/src/test/acceptance/features/component/send_data/steps.py +++ b/ngsi_adapter/src/test/acceptance/features/component/send_data/steps.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -# Copyright 2014 Telefonica Investigación y Desarrollo, S.A.U +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U # -# This file is part of FI-WARE project. +# This file is part of FIWARE project. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,3 +24,100 @@ __author__ = 'jfernandez' +from lettuce import world, step + +from commons.utils import get_probe_data_from_resource_file +from nose.tools import assert_equal, assert_true +from commons.dataset_utils import prepare_param +from lettuce_tools.logs_checking.log_utils import LogUtils +import time +from commons.constants import PROPERTIES_CONFIG_ENV, \ + PROPERTIES_CONFIG_ENV_LOCAL_PATH_REMOTE_LOGS, MONITORING_CONFIG_ENV_DEFAULT_PARSER_PARAMS, \ + MONITORING_CONFIG_ENV_DEFAULT_PARSER_DATA, MONITORING_CONFIG_SERVICE_ADAPTER, \ + MONITORING_CONFIG_SERVICE_LOG_FILE_NAME + +# Wait X seconds for remote logging +WAIT_FOR_REMOTE_LOGGING = 5 + + +def _set_default_dataset(): + """ + Ser default dataset vars for testing when data is not specified in the Scenarios + :return None + """ + #default_parser = world.config[PROPERTIES_CONFIG_ENV][MONITORING_CONFIG_ENV_DEFAULT_PARSER] + world.raw_data_filename = world.config[PROPERTIES_CONFIG_ENV][MONITORING_CONFIG_ENV_DEFAULT_PARSER_DATA] + world.raw_data_params = world.config[PROPERTIES_CONFIG_ENV][MONITORING_CONFIG_ENV_DEFAULT_PARSER_PARAMS] + + +@step(u'the probe "(.*)" and its associated parser "(.*)"$') +def the_parser(step, probe_name, parser_name): + world.probe = prepare_param(probe_name) + world.parser = prepare_param(parser_name) + + +@step(u'the probe name "(.*)"') +def the_probe_name(step, probe_name): + world.probe = prepare_param(probe_name) + + +@step(u'the monitored resource with id "(.*)" and type "(.*)"$') +def the_monitored_resource_with_id_and_type(step, id, type): + world.entity_id = prepare_param(id) + world.entity_type = prepare_param(type) + + +@step(u'I send raw data according to the selected probe$') +def i_sed_raw_data_according_to_the_selected_parser(step): + if world.raw_data_filename is None: + _set_default_dataset() + + probe_data = get_probe_data_from_resource_file(world.raw_data_filename, world.raw_data_params) + world.response = world.ngsi_adapter_client.send_raw_data_custom(probe_data, world.probe, + world.entity_id, world.entity_type) + + +@step(u'I send raw data according to the selected probe with "(.*)" HTTP operation$') +def i_sed_raw_data_according_to_the_selected_parser_with_http_verb(step, http_verb): + if world.raw_data_filename is None: + _set_default_dataset() + + probe_data = get_probe_data_from_resource_file(world.raw_data_filename, world.raw_data_params) + world.response = world.ngsi_adapter_client.send_raw_data_custom(probe_data, world.probe, + world.entity_id, world.entity_type, + http_method=http_verb) + + +@step(u'the response status code is "(.*)"$') +def the_response_status_code_is(step, status_code): + assert_equal(str(world.response.status_code), status_code) + + +@step(u'the header Transaction-Id "(.*)"$') +def the_header_transaction_id(step, transaction_id): + world.transaction_id = prepare_param(transaction_id) + world.ngsi_adapter_client.init_headers(transaction_id=world.transaction_id) + + +@step(u'an auto-generated Transaction-Id value is used in logs') +@step(u'the given Transaction-Id value is used in logs') +def the_given_transaction_id_value_is_used_in_logs(step): + log_utils = LogUtils() + + remote_log_local_path = world.config[PROPERTIES_CONFIG_ENV][PROPERTIES_CONFIG_ENV_LOCAL_PATH_REMOTE_LOGS] + service_log_file_name = world.config[MONITORING_CONFIG_SERVICE_ADAPTER][MONITORING_CONFIG_SERVICE_LOG_FILE_NAME] + + # Wait for remote logging + time.sleep(WAIT_FOR_REMOTE_LOGGING) + + if world.transaction_id is not None and len(world.transaction_id) != 0: + log_value_transaction_id = {"TRANSACTION_ID": "trans={transaction_id}".format( + transaction_id=world.transaction_id)} + log_utils.search_in_log(remote_log_local_path, service_log_file_name, log_value_transaction_id) + else: + log_value_message = {"MESSAGE": "msg={probe}".format(probe=world.probe)} + log_line = log_utils.search_in_log(remote_log_local_path, service_log_file_name, log_value_message) + + transaction_id = log_line[log_utils.LOG_TAG["TRANSACTION_ID"].replace("=", "")] + assert_true(len(transaction_id) != 0, + "Transaction-ID not found in logs. Expected value. Value in logs: " + transaction_id) diff --git a/ngsi_adapter/src/test/acceptance/features/component/send_data/terrain.py b/ngsi_adapter/src/test/acceptance/features/component/send_data/terrain.py new file mode 100644 index 0000000..556548c --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/features/component/send_data/terrain.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U +# +# This file is part of FIWARE project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For those usages not covered by the Apache version 2.0 License please +# contact with opensource@tid.es + +__author__ = 'jfernandez' + + +from lettuce import before, after, world +from commons.terrain_utils import set_up, tear_down +from commons.ngsi_adapter_api_utils.ngsi_adapter_client import NgsiAdapterClient +from commons.logger_utils import get_logger +from commons.constants import MONITORING_CONFIG_SERVICE_ADAPTER, MONITORING_CONFIG_SERVICE_HOST, \ + MONITORING_CONFIG_SERVICE_PORT, MONITORING_CONFIG_SERVICE_PROTOCOL + +logger = get_logger("terrain_utils") + + +@before.all +def before_all(): + set_up() + + +@before.each_feature +def before_each_feature(feature): + world.ngsi_adapter_client = NgsiAdapterClient(world.config[MONITORING_CONFIG_SERVICE_ADAPTER] + [MONITORING_CONFIG_SERVICE_PROTOCOL], + world.config[MONITORING_CONFIG_SERVICE_ADAPTER] + [MONITORING_CONFIG_SERVICE_HOST], + world.config[MONITORING_CONFIG_SERVICE_ADAPTER] + [MONITORING_CONFIG_SERVICE_PORT]) + + +@before.each_scenario +def before_each_scenario(scenario): + world.parser = None + world.probe_id = "qa" + world.probe_type = "host" + world.raw_data_filename = None + world.raw_data_params = None + + logger.info("#######################################################") + logger.info("#######################################################") + + +@after.all +def after_all(total): + tear_down() diff --git a/ngsi_adapter/src/test/acceptance/requirements.txt b/ngsi_adapter/src/test/acceptance/requirements.txt index 87bf23e..97f9fa0 100644 --- a/ngsi_adapter/src/test/acceptance/requirements.txt +++ b/ngsi_adapter/src/test/acceptance/requirements.txt @@ -1,2 +1,8 @@ lettuce==0.2.19 -git+https://github.com/telefonicaid/lettuce-tools.git@v0.1#egg=lettuce_tools \ No newline at end of file +git+https://github.com/telefonicaid/lettuce-tools.git#egg=lettuce_tools +setuptools +nose +xmldict +xmltodict +python-sshtail +requests \ No newline at end of file diff --git a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_disk_grouping_valid_template.txt b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_disk_grouping_valid_template.txt index 9fed6c4..cc22438 100644 --- a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_disk_grouping_valid_template.txt +++ b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_disk_grouping_valid_template.txt @@ -1 +1 @@ -"DISK OK - free space: mygroup 25484 MB (${FREE_SPACE_VALUE}% inode=95%);| mygroup=4151MB;31071;31121;0;31171" \ No newline at end of file +"DISK OK - free space: mygroup 25484 MB (${FREE_SPACE_VALUE}% inode=95%);| mygroup=4151MB;31071;31121;0;31171" diff --git a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_disk_nogrouping_valid_template.txt b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_disk_nogrouping_valid_template.txt index 7c6161b..bd4b302 100644 --- a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_disk_nogrouping_valid_template.txt +++ b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_disk_nogrouping_valid_template.txt @@ -1 +1 @@ -"DISK OK - free space: / 1393 MB (${FREE_SPACE_VALUE_1}% inode=66%); /data 4195 MB ({FREE_SPACE_VALUE_2}% inode=99%);| /=3388MB;5023;5023;0;5038 /data=586MB;5022;5022;0;5037" \ No newline at end of file +"DISK OK - free space: / 1393 MB (${FREE_SPACE_VALUE_1}% inode=66%); /data 4195 MB ({FREE_SPACE_VALUE_2}% inode=99%);| /=3388MB;5023;5023;0;5038 /data=586MB;5022;5022;0;5037" diff --git a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_load_valid_template.txt b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_load_valid_template.txt index 9545e1d..4f73968 100644 --- a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_load_valid_template.txt +++ b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_load_valid_template.txt @@ -1 +1 @@ -"OK - load average: ${CPU_LOAD_VALUE}, 1.23, 3.45|load1=4.000;1.000;7.000;2; load5=1.000;5.000;5.000;2; load15=40.000;15.000;16.000;9;" \ No newline at end of file +"OK - load average: ${CPU_LOAD_VALUE}, 1.23, 3.45|load1=4.000;1.000;7.000;2; load5=1.000;5.000;5.000;2; load15=40.000;15.000;16.000;9;" diff --git a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_mem.sh_valid_template.txt b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_mem.sh_valid_template.txt index 5161563..e20ef3d 100644 --- a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_mem.sh_valid_template.txt +++ b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_mem.sh_valid_template.txt @@ -1 +1 @@ -"Memory: OK Total: 1877 MB - Used: 369 MB - ${USED_MEM_VALUE}% used|TOTAL=1969020928;;;; USED=386584576;;;; CACHE=999440384;;;; BUFFER=201584640;;;;" \ No newline at end of file +"Memory: OK Total: 1877 MB - Used: 369 MB - ${USED_MEM_VALUE}% used|TOTAL=1969020928;;;; USED=386584576;;;; CACHE=999440384;;;; BUFFER=201584640;;;;" diff --git a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_procs_valid_template.txt b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_procs_valid_template.txt index 675eb72..c56bbc2 100644 --- a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_procs_valid_template.txt +++ b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_procs_valid_template.txt @@ -1 +1 @@ -"PROCS OK: ${NUM_PROCESSES_VALUE} processes" \ No newline at end of file +"PROCS OK: ${NUM_PROCESSES_VALUE} processes" diff --git a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_users_valid_template.txt b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_users_valid_template.txt index 2786091..b349551 100644 --- a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_users_valid_template.txt +++ b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/check_users_valid_template.txt @@ -1 +1 @@ -"USERS OK - 1 users currently logged in |users=${NUM_USERS_VALUE};10;15;0" \ No newline at end of file +"USERS OK - 1 users currently logged in |users=${NUM_USERS_VALUE};10;15;0" diff --git a/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/qa_probe_valid_template.txt b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/qa_probe_valid_template.txt new file mode 100644 index 0000000..2786091 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/resources/probe_sample_data/qa_probe_valid_template.txt @@ -0,0 +1 @@ +"USERS OK - 1 users currently logged in |users=${NUM_USERS_VALUE};10;15;0" \ No newline at end of file diff --git a/ngsi_adapter/src/test/acceptance/settings/__init__.py b/ngsi_adapter/src/test/acceptance/settings/__init__.py new file mode 100644 index 0000000..53a9ce8 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/settings/__init__.py @@ -0,0 +1 @@ +__author__ = 'jfernandez' diff --git a/ngsi_adapter/src/test/acceptance/settings/dev-properties.json b/ngsi_adapter/src/test/acceptance/settings/dev-properties.json index 03146b8..49badc8 100644 --- a/ngsi_adapter/src/test/acceptance/settings/dev-properties.json +++ b/ngsi_adapter/src/test/acceptance/settings/dev-properties.json @@ -1,23 +1,31 @@ { "environment": { - "name": "experimentation" + "name": "experimentation", + "log_path": "./logs", + "local_path_remote_logs": "./logs/remote/", + "default_parser": "qa_probe", + "default_parser_data": "qa_probe_valid_template", + "default_parser_parameters": [{"key":"NUM_USERS_VALUE", "value":"5"}] }, "monitoring_adapter_service": { "protocol": "http", "host": "130.206.81.245", "port": "1337", "resource": "", - "host_user": "", - "host_password": "" + "host_user": "root", + "host_password": "", + "private_key_location": "./fiware_cloud_dsa", + "service_log_path": "/var/log/ngsi_adapter/", + "service_log_file_name": "ngsi_adapter.log" }, "monitoring_nagios": { "host": "130.206.81.243", - "host_user": "", + "host_user": "root", "host_password": "" }, "monitoring_remote_host": { "host": "130.206.81.246", - "host_user": "", + "host_user": "root", "host_password": "" } } diff --git a/ngsi_adapter/src/test/acceptance/settings/logging.conf b/ngsi_adapter/src/test/acceptance/settings/logging.conf new file mode 100644 index 0000000..2acc4b8 --- /dev/null +++ b/ngsi_adapter/src/test/acceptance/settings/logging.conf @@ -0,0 +1,38 @@ +[loggers] +keys=root,testingLogger + +[handlers] +keys=consoleHandler,fileHandler + +[formatters] +keys=consoleFormatter,fileFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler,fileHandler + +[logger_testingLogger] +level=DEBUG +handlers=consoleHandler,fileHandler +qualname=testingLogger +propagate=0 + +[handler_consoleHandler] +class=StreamHandler +level=ERROR +formatter=consoleFormatter +args=(sys.stdout,) + +[handler_fileHandler] +class=FileHandler +level=DEBUG +formatter=fileFormatter +args=('logs/monitoring_tests.log', 'w') + +[formatter_consoleFormatter] +format=- %(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= + +[formatter_fileFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt=