From 7524b9b04bd88a6dd49845c4fbc445f9cc9f00dd Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Mon, 24 Apr 2017 12:44:38 -0700 Subject: [PATCH 01/15] More tests for setup_resources --- .../environment/setup/setup_resources.py | 4 +-- .../setup/tests/test_setup_resources.py | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/sandbox_scripts/environment/setup/setup_resources.py b/sandbox_scripts/environment/setup/setup_resources.py index 00f8f7b..b01a7c8 100644 --- a/sandbox_scripts/environment/setup/setup_resources.py +++ b/sandbox_scripts/environment/setup/setup_resources.py @@ -1,11 +1,10 @@ # coding=utf-8 -#from sandbox_scripts.helpers.Networking.NetworkingHealthCheck import * from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers from sandbox_scripts.helpers.Networking.save_restore_mgr import SaveRestoreManager from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase from cloudshell.core.logger.qs_logger import get_qs_logger from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError -import os, sys +import os class EnvironmentSetupResources(object): @@ -45,6 +44,7 @@ def execute(self): write_to_output_window=True) # power on Vms that might be powered off because of the snapshot configuration + # TODO: get a list of vms that were restored from snapshot and only power on these ones sandbox.power_on_vms() # call activate_all_routes_and_connectors diff --git a/sandbox_scripts/environment/setup/tests/test_setup_resources.py b/sandbox_scripts/environment/setup/tests/test_setup_resources.py index d1c8b1f..000b009 100644 --- a/sandbox_scripts/environment/setup/tests/test_setup_resources.py +++ b/sandbox_scripts/environment/setup/tests/test_setup_resources.py @@ -1,6 +1,8 @@ import unittest from mock import patch, Mock, call from sandbox_scripts.environment.setup.setup_resources import EnvironmentSetupResources +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError +from cloudshell.api.common_cloudshell_api import CloudShellAPIError import json import os @@ -73,6 +75,38 @@ def test_flow_ok_with_no_storage_device(self, mock_save, mock_sandboxbase, mock_ call('Sandbox setup finished successfully')] mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + @patch('sandbox_scripts.environment.setup.setup_resources.SandboxBase') + @patch('sandbox_scripts.environment.setup.setup_resources.SaveRestoreManager') + def test_qualierror_exception(self, mock_save, mock_sandboxbase, mock_api_session): + mock_sandboxbase.return_value.get_storage_server_resource.return_value = False + mock_sandboxbase.return_value.activate_all_routes_and_connectors.side_effect = QualiError(name='a',message='b') + + self.setup_script.execute() + report_info_calls = [call('Beginning load configuration for resources'), + call('Skipping load configuration. No storage resource associated with the blueprint ', + write_to_output_window=True)] + mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) + + report_info_calls = [call('Sandbox setup finished successfully')] + mock_sandboxbase.return_value.report_info.asset_not_called(report_info_calls) + self.setup_script.logger.error.assert_called_with('Setup failed. CloudShell error at a. Error is: b') + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + @patch('sandbox_scripts.environment.setup.setup_resources.SandboxBase') + @patch('sandbox_scripts.environment.setup.setup_resources.SaveRestoreManager') + def test_general_exception(self, mock_save, mock_sandboxbase, mock_api_session): + mock_sandboxbase.return_value.get_storage_server_resource.side_effect = Exception('error') + self.setup_script.execute() + report_info_calls = [call('Beginning load configuration for resources')] + mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) + + report_info_calls = [call('Skipping load configuration. No storage resource associated with the blueprint ', + write_to_output_window=True), + call('Sandbox setup finished successfully')] + mock_sandboxbase.return_value.report_info.asset_not_called(report_info_calls) + + self.setup_script.logger.error.assert_called_with('Setup failed. Unexpected error:error') if __name__ == '__main__': unittest.main() From 54ba541258e59c81cb70d90b2be7d873726345d6 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Mon, 24 Apr 2017 12:45:26 -0700 Subject: [PATCH 02/15] Fix imports in some files --- sandbox_scripts/QualiEnvironmentUtils/Sandbox.py | 9 +++++++-- sandbox_scripts/environment/savesnapshot/SaveSnapshot.py | 3 +-- sandbox_scripts/environment/teardown/teardown_VM.py | 3 --- .../environment/teardown/teardown_resources.py | 2 +- .../helpers/Networking/NetworkingSaveNRestore.py | 5 +---- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py b/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py index efe0eb5..74ecb33 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py +++ b/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py @@ -2,6 +2,8 @@ from Resource import * from cloudshell.core.logger.qs_logger import * from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers +from cloudshell.api.cloudshell_api import * +from cloudshell.api.common_cloudshell_api import CloudShellAPIError from os.path import * @@ -395,7 +397,7 @@ def is_apps_in_reservation(self): if not apps or (len(apps) == 1 and not apps[0].Name): self.report_info("No apps found in reservation {0}".format(self.reservation_id)) self.api_session.WriteMessageToReservationOutput(reservationId=self.reservation_id, - message='No apps in reservation') + message='No apps in reservation') return False return True @@ -410,8 +412,11 @@ def power_on_vms(self, write_to_output=True): for resource in root_resources: if resource.is_app(): deployed_app_name = resource.name - self.api_session.WriteMessageToReservationOutput(reservationId=self.id, + if write_to_output: + self.api_session.WriteMessageToReservationOutput(reservationId=self.id, message='Power on Apps again') + write_to_output = False # to prevent more prints + self.api_session.ExecuteResourceConnectedCommand(self.id, deployed_app_name, "PowerOn", "power") diff --git a/sandbox_scripts/environment/savesnapshot/SaveSnapshot.py b/sandbox_scripts/environment/savesnapshot/SaveSnapshot.py index d26158c..80f073f 100644 --- a/sandbox_scripts/environment/savesnapshot/SaveSnapshot.py +++ b/sandbox_scripts/environment/savesnapshot/SaveSnapshot.py @@ -3,7 +3,7 @@ from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase from sandbox_scripts.helpers.Networking.save_restore_mgr import SaveRestoreManager from cloudshell.core.logger.qs_logger import get_qs_logger -from QualiUtils import QualiError +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError class EnvironmentSaveSnapshot: @@ -14,7 +14,6 @@ def __init__(self): log_category='EnvironmentCommands') - # @profileit(scriptName='SaveSnapshot') def execute(self): sandbox = SandboxBase(self.reservation_id, self.logger) saveNRestoreTool = SaveRestoreManager(sandbox) diff --git a/sandbox_scripts/environment/teardown/teardown_VM.py b/sandbox_scripts/environment/teardown/teardown_VM.py index 6b2651f..7caccb7 100644 --- a/sandbox_scripts/environment/teardown/teardown_VM.py +++ b/sandbox_scripts/environment/teardown/teardown_VM.py @@ -1,6 +1,3 @@ -from multiprocessing.pool import ThreadPool -from threading import Lock - from cloudshell.core.logger import qs_logger from sandbox_scripts.helpers.Networking.NetworkingSaveNRestore import * diff --git a/sandbox_scripts/environment/teardown/teardown_resources.py b/sandbox_scripts/environment/teardown/teardown_resources.py index 45d5416..309f7c9 100644 --- a/sandbox_scripts/environment/teardown/teardown_resources.py +++ b/sandbox_scripts/environment/teardown/teardown_resources.py @@ -3,7 +3,7 @@ from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase from sandbox_scripts.helpers.Networking.save_restore_mgr import SaveRestoreManager -from QualiUtils import QualiError +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError class EnvironmentTeardownResources: diff --git a/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py b/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py index 37d6d8a..3fceb7d 100644 --- a/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py +++ b/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py @@ -1,14 +1,11 @@ # coding=utf-8 import csv import tempfile -# import json -# import subprocess from multiprocessing.pool import ThreadPool from threading import Lock from sandbox_scripts.QualiEnvironmentUtils.ConfigFileManager import * from sandbox_scripts.QualiEnvironmentUtils.ConfigPoolManager import * -# from sandbox_scripts.QualiEnvironmentUtils.StorageManager import StorageManager from sandbox_scripts.helpers.Networking.base_save_restore import * from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError @@ -90,7 +87,7 @@ def load_config(self, config_stage, config_type, restore_method="Override", conf self.sandbox.report_error(res.message, raise_error=False) self.sandbox.api_session.SetResourceLiveStatus(res.resource_name, 'Error') elif res.message != '': - self.sandbox.report_info(res.resource_name + "\n" + res.message,write_to_output_window=True) + self.sandbox.report_info(res.resource_name + "\n" + res.message, write_to_output_window=True) if remove_temp_files: self._remove_temp_config_files() From 1260a476591787014709f2e6caaf259d5e49dba2 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Mon, 24 Apr 2017 19:20:50 -0700 Subject: [PATCH 03/15] Cleanups --- sandbox_scripts/QualiEnvironmentUtils/Sandbox.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py b/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py index 74ecb33..92486a0 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py +++ b/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py @@ -2,7 +2,6 @@ from Resource import * from cloudshell.core.logger.qs_logger import * from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers -from cloudshell.api.cloudshell_api import * from cloudshell.api.common_cloudshell_api import CloudShellAPIError from os.path import * @@ -372,15 +371,14 @@ def get_config_set_pool_resource(self): return resource return None - # ----------------------------------------- # Return the pool Apps of the sandbox, if found # ----------------------------------------- def get_Apps_resources(self): """ - Get the Apps resources - :rtype: list[ReservationAppResource] - """ + Get the Apps resources + :rtype: list[ReservationAppResource] + """ details = self.get_details() apps_resources = details.ReservationDescription.Apps From 3c3fb0a00128ecc070a9ee0fde9fffd90327e343 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Mon, 24 Apr 2017 22:51:27 -0500 Subject: [PATCH 04/15] setup_vm tests --- sandbox_scripts/environment/setup/setup_VM.py | 82 ++++---- .../environment/setup/tests/test_setup_VM.py | 175 ++++++++++++++++++ 2 files changed, 212 insertions(+), 45 deletions(-) create mode 100644 sandbox_scripts/environment/setup/tests/test_setup_VM.py diff --git a/sandbox_scripts/environment/setup/setup_VM.py b/sandbox_scripts/environment/setup/setup_VM.py index 5daa1cc..0d1878f 100644 --- a/sandbox_scripts/environment/setup/setup_VM.py +++ b/sandbox_scripts/environment/setup/setup_VM.py @@ -1,4 +1,10 @@ -from sandbox_scripts.helpers.Networking.NetworkingSaveNRestore import * +from sandbox_scripts.helpers.Networking.save_restore_mgr import SaveRestoreManager +from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase +from cloudshell.core.logger.qs_logger import get_qs_logger +from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError +from multiprocessing.pool import ThreadPool +from threading import Lock class EnvironmentSetupVM(object): @@ -16,32 +22,21 @@ def execute(self): self.sandbox.report_info('Beginning VMs power on') - reservation_details = self.sandbox.api_session.GetReservationDetails(self.reservation_id) - #TODO: don't use networking save and restore to figure if it's a snapshot setup - saveNRestoreTool = NetworkingSaveRestore(self.sandbox) - if saveNRestoreTool.get_storage_manager(): - if saveNRestoreTool.is_snapshot(): + save_n_restore_mgr = SaveRestoreManager(self.sandbox) + if save_n_restore_mgr.get_storage_manager(): + if save_n_restore_mgr.is_snapshot(): self.is_snapshot = True - self._run_async_power_on_refresh_ip(reservation_details) - - - def _run_async_power_on_refresh_ip(self, reservation_details): + self._run_async_power_on_refresh_ip() - """ - :param GetReservationDescriptionResponseInfo reservation_details: - :param BulkAppDeploymentyInfo deploy_results: - :param (dict of str: ResourceInfo) resource_details_cache: - :param str reservation_id: - :return: - """ + def _run_async_power_on_refresh_ip(self): # filter out resources not created in this reservation - resources = reservation_details.ReservationDescription.Resources + resources = self.sandbox.get_details().ReservationDescription.Resources if len(resources) == 0: - self.sandbox.report_info(message='No resources to power on ', write_to_output_window=True) + self.sandbox.report_info(message='No VMs to power on ', write_to_output_window=True) return pool = ThreadPool(len(resources)) @@ -60,7 +55,7 @@ def _run_async_power_on_refresh_ip(self, reservation_details): for async_result in async_results: res = async_result.get() if not res[0]: - raise Exception("Reservation is Active with Errors - " + res[1]) + raise QualiError("Reservation is Active with Errors - " + res[1]) def _power_on_refresh_ip(self, lock, message_status, resource): """ @@ -76,21 +71,19 @@ def _power_on_refresh_ip(self, lock, message_status, resource): resource_details = self.sandbox.api_session.GetResourceDetails(deployed_app_name) vm_details = resource_details.VmDetails - if not(vm_details and hasattr(vm_details, "UID") and vm_details.UID): - #self.logger.debug("Resource {0} is not a app, nothing to do with it".format(deployed_app_name)) + if vm_details is None or hasattr(vm_details, "UID") is False or vm_details.UID is None: + self.logger.debug("Skipping resource '{0}' - not an app, not powering on".format(deployed_app_name)) return True, "" power_on = "true" wait_for_ip = "true" - if resource.ResourceModelName.lower() =="vcenter static vm": + if resource.ResourceModelName.lower() == "vcenter static vm": self.logger.debug("Resource {0} is a static app".format(deployed_app_name)) wait_for_ip = "false" - elif not self.is_snapshot: return True, "" - try: self._power_on(deployed_app_name, power_on, lock, message_status) except Exception as exc: @@ -99,37 +92,36 @@ def _power_on_refresh_ip(self, lock, message_status, resource): return False, "Error powering on deployed app {0}".format(deployed_app_name) try: - if resource.ResourceModelName.lower() !="vCenter Static VM": - self._wait_for_ip(deployed_app_name, wait_for_ip, lock, message_status) + if wait_for_ip.lower() == "true": + self._wait_for_ip(deployed_app_name, lock, message_status) + else: + self.logger.info("Wait For IP is off for deployed app {0} in reservation {1}" + .format(deployed_app_name, self.reservation_id)) except Exception as exc: self.logger.error("Error refreshing IP on deployed app {0} in reservation {1}. Error: {2}" .format(deployed_app_name, self.reservation_id, str(exc))) return False, "Error refreshing IP deployed app {0}. Error: {1}".format(deployed_app_name, exc.message) - return True,"" + return True, "" - def _wait_for_ip(self, deployed_app_name, wait_for_ip, lock, message_status): + def _wait_for_ip(self, deployed_app_name, lock, message_status): + if not message_status['wait_for_ip']: + with lock: + if not message_status['wait_for_ip']: + message_status['wait_for_ip'] = True + self.sandbox.report_info(message='Waiting for apps IP addresses, this may take a while...', + write_to_output_window=True) - if wait_for_ip.lower() == "true": + self.logger.info("Executing 'Refresh IP' on deployed app {0} in reservation {1}" + .format(deployed_app_name, self.reservation_id)) - if not message_status['wait_for_ip']: - with lock: - if not message_status['wait_for_ip']: - message_status['wait_for_ip'] = True - self.sandbox.report_info(message='Waiting for apps IP addresses, this may take a while...', - write_to_output_window=True) + with lock: + self.sandbox.api_session.ExecuteResourceConnectedCommand(self.reservation_id, deployed_app_name, + "remote_refresh_ip", + "remote_connectivity") - self.logger.info("Executing 'Refresh IP' on deployed app {0} in reservation {1}" - .format(deployed_app_name, self.reservation_id)) - with lock: - self.sandbox.api_session.ExecuteResourceConnectedCommand(self.reservation_id, deployed_app_name, - "remote_refresh_ip", - "remote_connectivity") - else: - self.logger.info("Wait For IP is off for deployed app {0} in reservation {1}" - .format(deployed_app_name, self.reservation_id)) def _power_on(self, deployed_app_name, power_on, lock, message_status): diff --git a/sandbox_scripts/environment/setup/tests/test_setup_VM.py b/sandbox_scripts/environment/setup/tests/test_setup_VM.py new file mode 100644 index 0000000..90e3d41 --- /dev/null +++ b/sandbox_scripts/environment/setup/tests/test_setup_VM.py @@ -0,0 +1,175 @@ +import unittest +from mock import patch, Mock, call +from sandbox_scripts.environment.setup.setup_VM import EnvironmentSetupVM +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError +from cloudshell.api.common_cloudshell_api import CloudShellAPIError +from cloudshell.api.cloudshell_api import ResourceInfoVmDetails +import json +import os + + +resContext = '''{"id":"5487c6ce-d0b3-43e9-8ee7-e27af8406905", + "ownerUser":"bob", + "ownerPass":"nIqm+BG6ZGJjby5hUittVFFJASc=", + "domain":"Global", + "environmentName":"My environment", + "description":"New demo environment", + "parameters": + { "globalInputs": [], + "resourceRequirements":[], + "resourceAdditionalInfo":[]}}''' + +conContext = '''{"serverAddress": "localhost", +"adminAuthToken": "anAdminToken"}''' + +class SetupVMTests(unittest.TestCase): + + @patch('sandbox_scripts.environment.setup.setup_VM.get_qs_logger') + def setUp(self, mock_logger): + os.environ['reservationContext'] = resContext + os.environ['qualiConnectivityContext'] = conContext + self.setup_script = EnvironmentSetupVM() + + def tearDown(self): + pass + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + @patch('sandbox_scripts.environment.setup.setup_VM.SandboxBase') + @patch('sandbox_scripts.environment.setup.setup_VM.SaveRestoreManager') + def test_setup_vm_with_no_resources(self, mock_save, mock_sandboxbase, mock_api_session): + + self.setup_script.execute() + + report_info_calls = [call('Beginning VMs power on'), + call(message='No VMs to power on ', write_to_output_window=True)] + mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + @patch('sandbox_scripts.environment.setup.setup_VM.SandboxBase') + @patch('sandbox_scripts.environment.setup.setup_VM.SaveRestoreManager') + def test_setup_vm_with_regular_resource(self, mock_save, mock_sandboxbase, mock_api_session): + + rdi = Mock() + resource1 = Mock() + resource1.Name = "r1" + resource2 = Mock() + resource2.Name = "r2" + rdi.ReservationDescription.Resources = [resource1, resource2] + rdi.ReservationDescription.TopologiesReservedResources = [] + rdi.ReservationDescription.Apps = [] + + mock_sandboxbase.return_value.get_details.return_value = rdi + + def resource_details_mock_side_effect(name): + rd = Mock() + rd.details.Name = name + rd.details.Address = '' + if name == 'r1': + rd.VmDetails = None + elif name == 'r2': + rd.VmDetails = object() + return rd + + mock_sandboxbase.return_value.api_session.GetResourceDetails.side_effect = resource_details_mock_side_effect + + mock_save.return_value.is_snapshot.return_value = True + + self.setup_script.execute() + + report_info_calls = [call('Beginning VMs power on')] + mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) + + logger_debug_calls = [call("Skipping resource 'r1' - not an app, not powering on"), + call("Skipping resource 'r2' - not an app, not powering on")] + self.setup_script.logger.debug.assert_has_calls(logger_debug_calls) + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + @patch('sandbox_scripts.environment.setup.setup_VM.SandboxBase') + @patch('sandbox_scripts.environment.setup.setup_VM.SaveRestoreManager') + def test_setup_vm_with_static_app_resource_power_on_no_wait_for_ip(self, mock_save, mock_sandboxbase, mock_api_session): + rdi = Mock() + resource1 = Mock() + resource1.Name = "r2" + resource1.ResourceModelName = 'vcenter static vm' + rdi.ReservationDescription.Resources = [resource1] + rdi.ReservationDescription.TopologiesReservedResources = [] + rdi.ReservationDescription.Apps = [] + + mock_sandboxbase.return_value.get_details.return_value = rdi + + def resource_details_mock_side_effect(name): + rd = Mock() + rd.details.Name = name + rd.details.Address = '' + rd.VmDetails = Mock() + rd.VmDetails.UID = 'abcd' + return rd + + mock_sandboxbase.return_value.api_session.GetResourceDetails.side_effect = resource_details_mock_side_effect + + mock_save.return_value.is_snapshot.return_value = True + + self.setup_script.execute() + + report_info_calls = [call('Beginning VMs power on'), + call('Apps are powering on... '), + call(log_message="Executing 'Power On' on deployed app r2 in reservation " + "5487c6ce-d0b3-43e9-8ee7-e27af8406905", + message="Executing 'Power On' on deployed app r2 ", write_to_output_window=True)] + mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) + + logger_debug_calls = [call("Resource r2 is a static app")] + self.setup_script.logger.debug.assert_has_calls(logger_debug_calls) + + logger_info_calls = [call('Wait For IP is off for deployed app r2 in ' + 'reservation 5487c6ce-d0b3-43e9-8ee7-e27af8406905')] + self.setup_script.logger.info.assert_has_calls(logger_info_calls) + + api_calls = [call(u'5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'PowerOn', 'power')] + mock_sandboxbase.return_value.api_session.ExecuteResourceConnectedCommand.assert_has_calls(api_calls) + + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + @patch('sandbox_scripts.environment.setup.setup_VM.SandboxBase') + @patch('sandbox_scripts.environment.setup.setup_VM.SaveRestoreManager') + def test_setup_vm_with_deployed_app_resource(self, mock_save, mock_sandboxbase, mock_api_session): + rdi = Mock() + resource1 = Mock() + resource1.Name = "r2" + resource1.ResourceModelName = 'deployed app' + rdi.ReservationDescription.Resources = [resource1] + rdi.ReservationDescription.TopologiesReservedResources = [] + rdi.ReservationDescription.Apps = [] + + mock_sandboxbase.return_value.get_details.return_value = rdi + + def resource_details_mock_side_effect(name): + rd = Mock() + rd.details.Name = name + rd.details.Address = '' + rd.VmDetails = Mock() + rd.VmDetails.UID = 'abcd' + return rd + + mock_sandboxbase.return_value.api_session.GetResourceDetails.side_effect = resource_details_mock_side_effect + + mock_save.return_value.is_snapshot.return_value = True + + self.setup_script.execute() + + report_info_calls = [call('Beginning VMs power on'), + call('Apps are powering on... '), + call(log_message="Executing 'Power On' on deployed app r2 in reservation " + "5487c6ce-d0b3-43e9-8ee7-e27af8406905", + message="Executing 'Power On' on deployed app r2 ", write_to_output_window=True), + call(message='Waiting for apps IP addresses, this may take a while...', + write_to_output_window=True)] + mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) + + api_calls = [call(u'5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'PowerOn', 'power'), + call(u'5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'remote_refresh_ip', 'remote_connectivity')] + mock_sandboxbase.return_value.api_session.ExecuteResourceConnectedCommand.assert_has_calls(api_calls) + + +if __name__ == '__main__': + unittest.main() From c431b4b277ba5f466a7d27f88286b56802dfe70b Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Tue, 25 Apr 2017 00:25:14 -0500 Subject: [PATCH 05/15] Added some tests for Resource --- .../QualiEnvironmentUtils/Resource.py | 47 ++--- .../tests/test_Resource.py | 161 ++++++++++++++++++ .../environment/setup/tests/test_setup_VM.py | 2 +- 3 files changed, 188 insertions(+), 22 deletions(-) create mode 100644 sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py diff --git a/sandbox_scripts/QualiEnvironmentUtils/Resource.py b/sandbox_scripts/QualiEnvironmentUtils/Resource.py index c33c55f..26f92e1 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/Resource.py +++ b/sandbox_scripts/QualiEnvironmentUtils/Resource.py @@ -49,19 +49,18 @@ def attribute_exist(self, attribute_name): return True return False - # ----------------------------------------- # ----------------------------------------- def get_attribute(self, attribute_name): - attribute_name = attribute_name.lower() + attribute_name_lower = attribute_name.lower() for attribute in self.attributes: - if attribute.Name.lower() == attribute_name: + if attribute.Name.lower() == attribute_name_lower: if attribute.Type == 'Password': decrypted = self.api_session.DecryptPassword(attribute.Value) return decrypted.Value else: return attribute.Value - raise QualiError(self.name, "Attribute: " + attribute_name + " not found") + raise QualiError(self.name, "Attribute: '" + attribute_name + "' not found") # ----------------------------------------- # ----------------------------------------- @@ -70,7 +69,7 @@ def set_attribute_value(self, attribute_name, attribute_value): self.api_session.SetAttributeValue(resourceFullPath=self.name, attributeName=attribute_name, attributeValue=attribute_value) except CloudShellAPIError as error: - raise QualiError(self.name, "Attribute: " + attribute_name + " not found. " + error.message) + raise QualiError(self.name, "Failed to set attribute: '" + attribute_name + "'. " + error.message) # ----------------------------------------- # implement the command to get the neighbors and their ports @@ -99,9 +98,9 @@ def health_check(self,reservation_id): if self.has_command('health_check'): try: # Return a detailed description in case of a failure - out = self.execute_command(reservation_id, 'health_check', printOutput=False) #.Output() + out = self.execute_command(reservation_id, 'health_check', printOutput=False) if out.Output.find(' passed') == -1: - err = "Health check did not pass for device " + self.name + ". " +out.Output + err = "Health check did not pass for device " + self.name + ". " + out.Output return err except QualiError as qe: @@ -320,7 +319,7 @@ def execute_command(self, reservation_id, commandName, commandInputs=[], printOu :rtype: CommandExecutionCompletedResultInfo """ # check the command exists on the device - if self.commands.__sizeof__() > 0: + if len(self.commands) > 0: # Run executeCommand with the restore command and its params (ConfigPath,RestoreMethod) try: return self.api_session.ExecuteCommand(reservation_id, self.name, 'Resource', commandName, @@ -345,7 +344,7 @@ def execute_connected_command(self, reservation_id, commandName,tag,commandInput :rtype: CommandExecutionCompletedResultInfo """ # check the command exists on the device - if self.connected_commands.__sizeof__() > 0: + if len(self.connected_commands) > 0: # Run executeCommand with the restore command and its params (ConfigPath,RestoreMethod) try: return self.api_session.ExecuteResourceConnectedCommand(reservation_id,self.name, @@ -361,8 +360,10 @@ def execute_connected_command(self, reservation_id, commandName,tag,commandInput # ----------------------------------------- # ----------------------------------------- def set_address(self, address): - self.api_session.UpdateResourceAddress(resourceFullPath=self.name, resourceAddress=address) - + try: + self.api_session.UpdateResourceAddress(resourceFullPath=self.name, resourceAddress=address) + except CloudShellAPIError as error: + raise QualiError(self.name, error.message) # ----------------------------------------- # ----------------------------------------- @@ -389,16 +390,19 @@ def load_firmware(self, reservation_id, image_path): # ----------------------------------------- # ----------------------------------------- - def set_live_status(self,live_status_name, additional_info=''): - self.api_session.SetResourceLiveStatus(self.name,live_status_name, additional_info) + def set_live_status(self, live_status_name, additional_info=''): + try: + self.api_session.SetResourceLiveStatus(self.name, live_status_name, additional_info) + except CloudShellAPIError as error: + raise QualiError(self.name, error.message) # ----------------------------------------- # ----------------------------------------- def get_live_status(self): - self.api_session.GetResourceLiveStatus(self.name) - - - + try: + self.api_session.GetResourceLiveStatus(self.name) + except CloudShellAPIError as error: + raise QualiError(self.name, error.message) # ----------------------------------------- # ----------------------------------------- @@ -428,9 +432,10 @@ def create_artifact_info(self,config_path): # ----------------------------------------- def is_app(self): try: - if self.details.VmDetails.UID: + if self.details.VmDetails is not None and \ + hasattr(self.details.VmDetails, 'UID') and \ + self.details.VmDetails.UID: return True - except: return False @@ -445,9 +450,9 @@ def get_version(self, reservation_id): # Run executeCommand with the update_firmware command and its params (ConfigPath,RestoreMethod) try: version = self.execute_command(reservation_id, 'get_version', - printOutput=False).Output + printOutput=False).Output version = version.replace('\r\n', '') - return version + return version except QualiError as qerror: raise QualiError(self.name, "Failed to get the version: " + qerror.message) except Exception as ex: diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py b/sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py new file mode 100644 index 0000000..91620d0 --- /dev/null +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py @@ -0,0 +1,161 @@ +import unittest +from mock import patch, Mock,call +from sandbox_scripts.QualiEnvironmentUtils.Resource import ResourceBase +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError +from cloudshell.api.cloudshell_api import ReservationDescriptionInfo +from cloudshell.api.cloudshell_api import ResourceCommandInfo +import json +import os +from cloudshell.api.common_cloudshell_api import CloudShellAPIError + +resContext = '''{"id":"5487c6ce-d0b3-43e9-8ee7-e27af8406905", + "ownerUser":"bob", + "ownerPass":"nIqm+BG6ZGJjby5hUittVFFJASc=", + "domain":"Global", + "environmentName":"My environment", + "description":"New demo environment", + "parameters": + { "globalInputs": [], + "resourceRequirements":[], + "resourceAdditionalInfo":[]}}''' + +conContext = '''{"serverAddress": "localhost", +"adminAuthToken": "anAdminToken"}''' + + +class ResourceTests(unittest.TestCase): + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def setUp(self, mock_api_session): + os.environ['reservationContext'] = resContext + os.environ['qualiConnectivityContext'] = conContext + tli = Mock() + tli.Topologies = ["My environment"] + mock_api_session.return_value.GetActiveTopologyNames = Mock(return_value=tli) + + abstractinfo = Mock() + abstractinfo.Alias = "alias" + + topoinfo = Mock() + topoinfo.Name = "My environment" + topoinfo.AbstractResources = [abstractinfo] + mock_api_session.return_value.GetTopologyDetails = Mock(return_value=topoinfo) + mock_logger = Mock() + self.mock_api_session = mock_api_session + self.mock_logger = mock_logger + + rd = Mock() + rd.Name = 'r1' + rd.Address = '' + rd.ResourceAttributes = [] + self.mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + self.resource = ResourceBase(resource_name='r1') + + + + def tearDown(self): + pass + + # ================================================================ + # test has command function + def test_has_command_found(self): + command1 = Mock() + command1.Name = 'example_command' + self.resource.commands = [command1] + ret = self.resource.has_command('example_command') + + self.assertTrue(ret, "command was expected to be found but wasn't") + + def test_has_command_found_connected(self): + command1 = Mock() + command1.Name = 'example_command' + self.resource.connected_commands = [command1] + ret = self.resource.has_command('example_command') + + self.assertTrue(ret, "command was expected to be found but wasn't") + + def test_has_command_not_found(self): + command1 = Mock() + command1.Name = 'example_command' + self.resource.connected_commands = [command1] + ret = self.resource.has_command('example_command2') + + self.assertFalse(ret, "command was not expected to be found but was found") + + # ================================================================ + # test attribute exists + def test_attribute_found(self): + attr1 = Mock() + attr1.Name = 'Location' + self.resource.attributes = [attr1] + ret = self.resource.attribute_exist('Location') + + self.assertTrue(ret, "attribute was expected to be found but wasn't") + + def test_attribute_found_case_insensitive(self): + attr1 = Mock() + attr1.Name = 'locatioN' + self.resource.attributes = [attr1] + ret = self.resource.attribute_exist('loCation') + + self.assertTrue(ret, "attribute was expected to be found but wasn't") + + def test_attribute_not_found(self): + command1 = Mock() + command1.Name = 'Location' + self.resource.connected_commands = [command1] + ret = self.resource.has_command('LocationXYZ') + + self.assertFalse(ret, "attribute was not expected to be found but was found") + + # ================================================================ + # test get attribute + def test_get_attribute(self): + attr1 = Mock() + attr1.Name = 'Location' + attr1.Type = 'String' + attr1.Value = 'New York' + self.resource.attributes = [attr1] + ret = self.resource.get_attribute('Location') + + self.assertEqual(ret, "New York", "attribute value was not as expected") + + def test_get_password_attribute(self): + attr1 = Mock() + attr1.Name = 'MyPassword' + attr1.Type = 'Password' + attr1.Value = 'abcdefg' + self.resource.attributes = [attr1] + self.mock_api_session.return_value.DecryptPassword.return_value.Value = 'New York' + ret = self.resource.get_attribute('MyPassword') + self.assertEqual(ret, "New York", "attribute was expected to be found but wasn't") + + def test_get_attribute_not_found(self): + attr1 = Mock() + attr1.Name = 'Location' + attr1.Type = 'String' + attr1.Value = 'New York' + self.resource.attributes = [attr1] + with self.assertRaises(QualiError) as e: + ret = self.resource.get_attribute('LocationXYZ') + + the_exception = e.exception + self.assertEqual(str(the_exception), + "CloudShell error at r1. Error is: Attribute: 'LocationXYZ' not found") + + + # TODO: much more + # health_check + # load_network_config + # save_network_config + # orchestration_restore + # orchestration_save + # execute_command + # execute_connected_command + # load_firmware + # set_live_status + # get_live_status + + + +if __name__ == '__main__': + unittest.main() diff --git a/sandbox_scripts/environment/setup/tests/test_setup_VM.py b/sandbox_scripts/environment/setup/tests/test_setup_VM.py index 90e3d41..eec051f 100644 --- a/sandbox_scripts/environment/setup/tests/test_setup_VM.py +++ b/sandbox_scripts/environment/setup/tests/test_setup_VM.py @@ -107,7 +107,7 @@ def resource_details_mock_side_effect(name): mock_sandboxbase.return_value.api_session.GetResourceDetails.side_effect = resource_details_mock_side_effect - mock_save.return_value.is_snapshot.return_value = True + mock_save.return_value.is_snapshot.return_value = False self.setup_script.execute() From c6176bdcf9e307a9a6e48205aa6ecc90a389368f Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Tue, 25 Apr 2017 23:09:45 -0500 Subject: [PATCH 06/15] Few messages cleanups --- .../ConfigPoolManager.py | 6 ++--- .../QualiEnvironmentUtils/Sandbox.py | 11 ++++----- sandbox_scripts/environment/setup/setup_VM.py | 16 ++++++------- .../environment/setup/tests/test_setup_VM.py | 23 ++++++++++--------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py b/sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py index 9046798..d13d6dd 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py +++ b/sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py @@ -1,5 +1,7 @@ # coding=utf-8 -from sandbox_scripts.QualiEnvironmentUtils.Sandbox import * +from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase +from sandbox_scripts.QualiEnvironmentUtils.Resource import ResourceBase +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError import re @@ -63,9 +65,7 @@ def pool_data_to_dict(self): for resource_from_pool in self.pool_resource.details.ChildResources: split_name = resource_from_pool.Name.split('/') name_of_resource_from_pool = split_name[len(split_name)-1] - #resource_attributes_dict = dict() for attribute in resource_from_pool.ResourceAttributes: resource_dict_key = str('{ConfigPool:' + name_of_resource_from_pool + ':' + attribute.Name + '}').lower() - #resource_attributes_dict[resource_dict_key] = attribute.Value pool_data_dict[resource_dict_key] = attribute.Value return pool_data_dict diff --git a/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py b/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py index 92486a0..62845e2 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py +++ b/sandbox_scripts/QualiEnvironmentUtils/Sandbox.py @@ -393,9 +393,9 @@ def is_apps_in_reservation(self): apps = details.ReservationDescription.App if not apps or (len(apps) == 1 and not apps[0].Name): - self.report_info("No apps found in reservation {0}".format(self.reservation_id)) - self.api_session.WriteMessageToReservationOutput(reservationId=self.reservation_id, - message='No apps in reservation') + self.report_info(message='No apps in reservation', + log_message="No apps found in reservation {0}".format(self.reservation_id), + write_to_output_window=True) return False return True @@ -411,9 +411,8 @@ def power_on_vms(self, write_to_output=True): if resource.is_app(): deployed_app_name = resource.name if write_to_output: - self.api_session.WriteMessageToReservationOutput(reservationId=self.id, - message='Power on Apps again') - write_to_output = False # to prevent more prints + self.report_info(message='Power on Apps again', write_to_output_window=True) + write_to_output = False # to prevent more prints self.api_session.ExecuteResourceConnectedCommand(self.id, deployed_app_name, "PowerOn", "power") diff --git a/sandbox_scripts/environment/setup/setup_VM.py b/sandbox_scripts/environment/setup/setup_VM.py index 0d1878f..361dde0 100644 --- a/sandbox_scripts/environment/setup/setup_VM.py +++ b/sandbox_scripts/environment/setup/setup_VM.py @@ -87,20 +87,20 @@ def _power_on_refresh_ip(self, lock, message_status, resource): try: self._power_on(deployed_app_name, power_on, lock, message_status) except Exception as exc: - self.logger.error("Error powering on deployed app {0} in reservation {1}. Error: {2}" + self.logger.error("Error powering on deployed app '{0}' in reservation '{1}'. Error: {2}" .format(deployed_app_name, self.reservation_id, str(exc))) - return False, "Error powering on deployed app {0}".format(deployed_app_name) + return False, "Error powering on deployed app '{0}'".format(deployed_app_name) try: if wait_for_ip.lower() == "true": self._wait_for_ip(deployed_app_name, lock, message_status) else: - self.logger.info("Wait For IP is off for deployed app {0} in reservation {1}" + self.logger.info("Wait For IP is off for deployed app '{0}' in reservation '{1}'" .format(deployed_app_name, self.reservation_id)) except Exception as exc: - self.logger.error("Error refreshing IP on deployed app {0} in reservation {1}. Error: {2}" + self.logger.error("Error refreshing IP on deployed app '{0}' in reservation '{1}'. Error: {2}" .format(deployed_app_name, self.reservation_id, str(exc))) - return False, "Error refreshing IP deployed app {0}. Error: {1}".format(deployed_app_name, exc.message) + return False, "Error refreshing IP deployed app '{0}'. Error: {1}".format(deployed_app_name, exc.message) return True, "" @@ -113,7 +113,7 @@ def _wait_for_ip(self, deployed_app_name, lock, message_status): self.sandbox.report_info(message='Waiting for apps IP addresses, this may take a while...', write_to_output_window=True) - self.logger.info("Executing 'Refresh IP' on deployed app {0} in reservation {1}" + self.logger.info("Executing 'Refresh IP' on deployed app '{0}' in reservation '{1}'" .format(deployed_app_name, self.reservation_id)) with lock: @@ -131,9 +131,9 @@ def _power_on(self, deployed_app_name, power_on, lock, message_status): message_status['power_on'] = True self.sandbox.report_info('Apps are powering on... ') - self.sandbox.report_info(message="Executing 'Power On' on deployed app {0} " + self.sandbox.report_info(message="Executing 'Power On' on deployed app '{0}' " .format(deployed_app_name), - log_message="Executing 'Power On' on deployed app {0} in reservation {1}" + log_message="Executing 'Power On' on deployed app '{0}' in reservation '{1}'" .format(deployed_app_name, self.reservation_id), write_to_output_window=True) diff --git a/sandbox_scripts/environment/setup/tests/test_setup_VM.py b/sandbox_scripts/environment/setup/tests/test_setup_VM.py index eec051f..1837740 100644 --- a/sandbox_scripts/environment/setup/tests/test_setup_VM.py +++ b/sandbox_scripts/environment/setup/tests/test_setup_VM.py @@ -22,6 +22,7 @@ conContext = '''{"serverAddress": "localhost", "adminAuthToken": "anAdminToken"}''' + class SetupVMTests(unittest.TestCase): @patch('sandbox_scripts.environment.setup.setup_VM.get_qs_logger') @@ -113,19 +114,19 @@ def resource_details_mock_side_effect(name): report_info_calls = [call('Beginning VMs power on'), call('Apps are powering on... '), - call(log_message="Executing 'Power On' on deployed app r2 in reservation " - "5487c6ce-d0b3-43e9-8ee7-e27af8406905", - message="Executing 'Power On' on deployed app r2 ", write_to_output_window=True)] + call(log_message="Executing 'Power On' on deployed app 'r2' in reservation " + "'5487c6ce-d0b3-43e9-8ee7-e27af8406905'", + message="Executing 'Power On' on deployed app 'r2' ", write_to_output_window=True)] mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) logger_debug_calls = [call("Resource r2 is a static app")] self.setup_script.logger.debug.assert_has_calls(logger_debug_calls) - logger_info_calls = [call('Wait For IP is off for deployed app r2 in ' - 'reservation 5487c6ce-d0b3-43e9-8ee7-e27af8406905')] + logger_info_calls = [call("Wait For IP is off for deployed app 'r2' in reservation " + "'5487c6ce-d0b3-43e9-8ee7-e27af8406905'")] self.setup_script.logger.info.assert_has_calls(logger_info_calls) - api_calls = [call(u'5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'PowerOn', 'power')] + api_calls = [call('5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'PowerOn', 'power')] mock_sandboxbase.return_value.api_session.ExecuteResourceConnectedCommand.assert_has_calls(api_calls) @@ -159,15 +160,15 @@ def resource_details_mock_side_effect(name): report_info_calls = [call('Beginning VMs power on'), call('Apps are powering on... '), - call(log_message="Executing 'Power On' on deployed app r2 in reservation " - "5487c6ce-d0b3-43e9-8ee7-e27af8406905", - message="Executing 'Power On' on deployed app r2 ", write_to_output_window=True), + call(log_message="Executing 'Power On' on deployed app 'r2' in reservation " + "'5487c6ce-d0b3-43e9-8ee7-e27af8406905'", + message="Executing 'Power On' on deployed app 'r2' ", write_to_output_window=True), call(message='Waiting for apps IP addresses, this may take a while...', write_to_output_window=True)] mock_sandboxbase.return_value.report_info.assert_has_calls(report_info_calls) - api_calls = [call(u'5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'PowerOn', 'power'), - call(u'5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'remote_refresh_ip', 'remote_connectivity')] + api_calls = [call('5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'PowerOn', 'power'), + call('5487c6ce-d0b3-43e9-8ee7-e27af8406905', 'r2', 'remote_refresh_ip', 'remote_connectivity')] mock_sandboxbase.return_value.api_session.ExecuteResourceConnectedCommand.assert_has_calls(api_calls) From 41c2ae506c13c186f43505196ee3bd97e79b04ee Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Fri, 28 Apr 2017 11:19:08 -0700 Subject: [PATCH 07/15] Cleanups in ignore --- .gitignore | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index bf1ea97..33a540b 100644 --- a/.gitignore +++ b/.gitignore @@ -75,13 +75,10 @@ qpm.ini SandboxOrchestration/environment_scripts/env_setup/data.json SandboxOrchestrationPackage/Topology Scripts/Default Sandbox Setup.zip SandboxOrchestrationPackage/Topology Scripts/Default Sandbox Teardown.zip -SandboxOrchestrationPackage/Topology Scripts/Default Sandbox Teardown.zip +SandboxOrchestrationPackage/Topology Scripts/SaveSnapshot.zip sandbox_scripts/environment/.idea/.name sandbox_scripts/environment/.idea/encodings.xml sandbox_scripts/environment/.idea/misc.xml sandbox_scripts/environment/.idea/workspace.xml -*.zip sandbox_scripts/environment/.idea/environment.iml -SandboxOrchestrationPackage/Topology Scripts/Default Sandbox Setup.zip sandbox_scripts/environment/.idea/modules.xml -SandboxOrchestrationPackage/Topology Scripts/Default Sandbox Teardown.zip From eaba0af478e86ceb7019a4c13451e87d31c6bfde Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Fri, 28 Apr 2017 11:20:24 -0700 Subject: [PATCH 08/15] Code cleanups --- .../env_teardown/__main__.py | 1 + .../ConfigFileManager.py | 23 +++++++++++-------- .../QualiEnvironmentUtils/Resource.py | 17 ++++++-------- .../StorageClients/FTPClient.py | 20 ++++++++-------- .../tests/DebugInteractive_setup_resources.py | 2 +- .../helpers/Networking/base_save_restore.py | 6 ++--- 6 files changed, 34 insertions(+), 35 deletions(-) diff --git a/SandboxOrchestration/environment_scripts/env_teardown/__main__.py b/SandboxOrchestration/environment_scripts/env_teardown/__main__.py index c5a9bdb..baf10b7 100644 --- a/SandboxOrchestration/environment_scripts/env_teardown/__main__.py +++ b/SandboxOrchestration/environment_scripts/env_teardown/__main__.py @@ -2,6 +2,7 @@ from sandbox_scripts.environment.teardown.teardown_resources import EnvironmentTeardownResources from sandbox_scripts.environment.teardown.teardown_VM import EnvironmentTeardownVM + def main(): EnvironmentTeardown().execute() EnvironmentTeardownVM().execute() diff --git a/sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py b/sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py index 9a9cf5d..322f5fc 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py +++ b/sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py @@ -1,7 +1,8 @@ # coding=utf-8 -from sandbox_scripts.QualiEnvironmentUtils.Sandbox import * -from time import gmtime, strftime import traceback +import re +from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase +from time import gmtime, strftime # =================================== @@ -15,8 +16,7 @@ def __init__(self, sandbox): # ---------------------------------- # ---------------------------------- - def create_concrete_config_from_template(self, template_config_data, config_set_pool_data, - resource): + def create_concrete_config_from_template(self, template_config_data, config_set_pool_data, resource): """ Replace parameters in the template file with concrete values Parameters in the template file are marked with {} @@ -30,36 +30,39 @@ def create_concrete_config_from_template(self, template_config_data, config_set_ it = re.finditer(r"\{ConfigPool\:[^}]*\}", concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() - concrete_config_data = concrete_config_data.replace(param, config_set_pool_data[param.lower()]) + if param.lower() in config_set_pool_data: + concrete_config_data = concrete_config_data.replace(param, config_set_pool_data[param.lower()]) + else: + raise ('Could not find attribute ' + param.lower() + ' in the config pool') # Replace {QUALI-NOTATION} WITH A NOTE - it = re.finditer(r"\{QUALI NOTATION\}", concrete_config_data,flags=re.IGNORECASE) + it = re.finditer(r"\{QUALI NOTATION\}", concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() quali_note = "Built from template: " + strftime("%Y-%b-%d %H:%M:%S", gmtime()) concrete_config_data = concrete_config_data.replace(param, quali_note) # Replace {Device.Self.Name} with the resource's name - it = re.finditer(r"\{Device:Self:Name\}", concrete_config_data,flags=re.IGNORECASE) + it = re.finditer(r"\{Device:Self:Name\}", concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() concrete_config_data = concrete_config_data.replace(param, resource.name) # Replace {Device.Self.Address} with the resource's management ip - it = re.finditer(r"\{Device:Self:Address\}", concrete_config_data,flags=re.IGNORECASE) + it = re.finditer(r"\{Device:Self:Address\}", concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() concrete_config_data = concrete_config_data.replace(param, resource.address) # Replace {Device.Self.ATTRIBUTE_NAME} with the resource's attribute value # Need to decode password attributes: Password, Enable Password, and SNMP Read Community - it = re.finditer(r"\{Device:Self\:[^}]*\}", concrete_config_data,flags=re.IGNORECASE) + it = re.finditer(r"\{Device:Self\:[^}]*\}", concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() idx = param.rfind(':')+1 att_name = param[idx:len(param)-1] param_val = resource.get_attribute(att_name) - #param_val = resource.get_attribute(param) + # param_val = resource.get_attribute(param) concrete_config_data = concrete_config_data.replace(param, param_val) # Replacemant of params from types: {Device:ALIAS:Attribute_name} diff --git a/sandbox_scripts/QualiEnvironmentUtils/Resource.py b/sandbox_scripts/QualiEnvironmentUtils/Resource.py index 26f92e1..3e11584 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/Resource.py +++ b/sandbox_scripts/QualiEnvironmentUtils/Resource.py @@ -228,14 +228,13 @@ def orchestration_restore(self, reservation_id,config_path,artifact_info = None) if 'orchestration_restore' == command.Name: tag = command.Tag json_str = artifact_info - self.execute_connected_command(reservation_id, 'orchestration_restore',tag, - commandInputs=[json_str], - printOutput=True) + self.execute_connected_command(reservation_id, 'orchestration_restore', tag, + commandInputs=[json_str], printOutput=True) break else: if self.has_command('orchestration_restore'): - if(artifact_info == None): + if artifact_info is None: artifact_info = self.create_artifact_info(config_path) json_str = json.dumps(artifact_info) else: @@ -243,8 +242,7 @@ def orchestration_restore(self, reservation_id,config_path,artifact_info = None) command_inputs = [InputNameValue('saved_details', json_str)] self.execute_command(reservation_id, 'orchestration_restore', - commandInputs=command_inputs, - printOutput=True) + commandInputs=command_inputs, printOutput=True) except QualiError as qerror: raise QualiError(self.name, "Failed to load configuration: " + qerror.message) @@ -275,9 +273,8 @@ def orchestration_save(self, reservation_id, config_path, config_type): if 'orchestration_save' == command.Name: tag = command.Tag - config_name = self.execute_connected_command(reservation_id, 'orchestration_save',tag, - commandInputs=['shallow',''], - printOutput=False) + config_name = self.execute_connected_command(reservation_id, 'orchestration_save', tag, + commandInputs=['shallow',''], printOutput=False) return config_name.Output @@ -285,7 +282,7 @@ def orchestration_save(self, reservation_id, config_path, config_type): if self.has_command('orchestration_save'): config_name = self.execute_command(reservation_id, 'orchestration_save', - commandInputs=[InputNameValue('custom_params',json_str), + commandInputs=[InputNameValue('custom_params', json_str), InputNameValue('mode','shallow')], printOutput=False) diff --git a/sandbox_scripts/QualiEnvironmentUtils/StorageClients/FTPClient.py b/sandbox_scripts/QualiEnvironmentUtils/StorageClients/FTPClient.py index 541271b..68e1349 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/StorageClients/FTPClient.py +++ b/sandbox_scripts/QualiEnvironmentUtils/StorageClients/FTPClient.py @@ -10,15 +10,14 @@ class FTPClient(StorageClient): # ---------------------------------- # ---------------------------------- def __init__(self, sandbox, storage_resource): - super(FTPClient,self).__init__(sandbox, storage_resource) + super(FTPClient, self).__init__(sandbox, storage_resource) try: self.username = storage_resource.get_attribute("Storage username") self.password = storage_resource.get_attribute("Storage password") - #self.ftp = ftplib.FTP(self.address, self.username, self.password) + # self.ftp = ftplib.FTP(self.address, self.username, self.password) self.ftp = ftplib.FTP() - self.ftp.connect(self.address,self.port) - self.ftp.login(self.username,self.password) - + self.ftp.connect(self.address, self.port) + self.ftp.login(self.username, self.password) except Exception as e: self.sandbox.report_error("Failed to connect to the FTP server . Error is: " + str(e), raise_error=True) @@ -50,12 +49,11 @@ def download(self, source, destination): self.ftp.connect(self.address, self.port) self.ftp.login(self.username, self.password) source = self._remove_header(source) - self.ftp.retrbinary("RETR " + source ,open(destination, 'wb').write) + self.ftp.retrbinary("RETR " + source, open(destination, 'wb').write) except Exception as e: self.sandbox.report_error("Failed to download file " + source + " to " + destination + " from the FTP server. Error is: " + str(e), raise_error=True) - # ---------------------------------- # ---------------------------------- def upload(self, destination, source): @@ -67,11 +65,11 @@ def upload(self, destination, source): try: self.ftp.connect(self.address, self.port) self.ftp.login(self.username, self.password) - file_idx=destination.rfind('/') + file_idx = destination.rfind('/') destination_dir = destination[:(file_idx-len(destination))] destination_dir = self._remove_header(destination_dir) destination_file = destination[file_idx+1:] - #destination_dir = destination_dir.replace('//','/') + # destination_dir = destination_dir.replace('//','/') self.ftp.cwd(destination_dir) myfile = open(source, 'r') self.ftp.storlines('STOR ' + destination_file, myfile) @@ -80,8 +78,8 @@ def upload(self, destination, source): except Exception as e: self.sandbox.report_error("Failed to upload file " + source + " to " + destination_file + " on the FTP server. Error is: " + str(e), raise_error=True) - print "Failed to upload file " + source + " to " + destination_file + " on the FTP server. Error is: " + str(e) - + print "Failed to upload file " + source + " to " + destination_file + \ + " on the FTP server. Error is: " + str(e) # ---------------------------------- # ---------------------------------- diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py b/sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py index aeb2201..1a0d4ce 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py @@ -7,4 +7,4 @@ os.environ["environment_name"] = "Abstract-ALL" x = EnvironmentSetupResources() -x.execute() \ No newline at end of file +x.execute() diff --git a/sandbox_scripts/helpers/Networking/base_save_restore.py b/sandbox_scripts/helpers/Networking/base_save_restore.py index 173761f..3324744 100644 --- a/sandbox_scripts/helpers/Networking/base_save_restore.py +++ b/sandbox_scripts/helpers/Networking/base_save_restore.py @@ -15,9 +15,9 @@ def __init__(self, sandbox): self.storage_mgr = StorageManager(sandbox) self.config_files_root = self.storage_mgr.get_configs_root() else: - if self.is_resources_in_reservation_to_restore(ignore_models = None): + if self.is_resources_in_reservation_to_restore(ignore_models=None): self.sandbox.report_info("Failed to find a storage server resource (e.g. tftp) in the sandbox. ", - write_to_output_window=False) + write_to_output_window=False) # ---------------------------------- # ---------------------------------- @@ -27,7 +27,7 @@ def get_storage_manager(self): # ---------------------------------- # Is this Sandbox originates from a snapshot Blueprint? # ---------------------------------- - def is_snapshot(self,fileName = " "): + def is_snapshot(self, fileName=" "): # check if there is a directory with the Blueprint's name under the snapshots dir if fileName != " ": From 624a32a0a04b0c659dda12486289cafb7e27b370 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Fri, 28 Apr 2017 11:20:58 -0700 Subject: [PATCH 09/15] Refactor configpoolmanager and added tests --- .../ConfigPoolManager.py | 51 ++----- .../tests/test_ConfigPoolManager.py | 127 ++++++++++++++++++ .../Networking/NetworkingSaveNRestore.py | 13 +- .../helpers/Networking/vm_save_restore.py | 9 +- 4 files changed, 148 insertions(+), 52 deletions(-) create mode 100644 sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigPoolManager.py diff --git a/sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py b/sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py index d13d6dd..4ec74dc 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py +++ b/sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py @@ -13,52 +13,16 @@ def __init__(self, sandbox, pool_resource): :param ResourceBase pool_resource: The pool resource in the sandbox """ self.sandbox = sandbox - self.pool_resource = pool_resource - - # ----------------------------------------- - # Walk over the sub-resources of the pool resource, - # For each sub-resource in the pool, get the data from their attributes, and update it on the actual device - # The device's alias in the sandbox will appear as a pool sub-resource, - # optionally with a number concatenated at the end e.g. Gateway1 - # ----------------------------------------- - def push_data_from_pool_to_sandbox(self): - message = '' - sandbox_resources = self.sandbox.get_root_resources() - # a list to mark which resources in the sandbox were already updated - sandbox_resources_can_update = [True]*len(sandbox_resources) - - for resource_from_pool in self.pool_resource.details.ChildResources: - split_name = resource_from_pool.Name.split('/') - name_of_resource_from_pool = split_name[len(split_name)-1] - found_resource_in_sandbox = False - idx = 0 - # Find the resource_from_pool in the sandbox - for sandbox_resource in sandbox_resources: - # use a regular expression since the pool may contain resources with a running index at the - # end of the name, that should match a generic alias for multiple resources in the sandbox. - # This can be caused from an abstract with quantity>1 - # e.g. the pool has CC1, CC2. The sandbox has a CC X2 - - pattern = sandbox_resource.alias + '\d*' - found_pattern = re.search(pattern, name_of_resource_from_pool, flags=0) - if found_pattern: - if sandbox_resources_can_update[idx]: - found_resource_in_sandbox = True - sandbox_resources_can_update[idx] = False - for attribute in resource_from_pool.ResourceAttributes: - sandbox_resource.set_attribute_value(attribute_name=attribute.Name, - attribute_value=attribute.Value) - break - idx += 1 - if not found_resource_in_sandbox: - message += 'Resource ' + resource_from_pool.Name + ' is in the pool, but could not be found in the sandbox. Please check\n' - if message != '': - raise QualiError(name=self.sandbox.id, message=message) + if pool_resource: + self.pool_resource = pool_resource + else: + raise QualiError('ConfigPoolManager', 'Trying to use the ConfigPoolManager without a pool resource') + self.pool_data = self._pool_data_to_dict() # ----------------------------------------- # Create a dictionary that will hold the data from the pool # ----------------------------------------- - def pool_data_to_dict(self): + def _pool_data_to_dict(self): pool_data_dict = dict() for attribute in self.pool_resource.attributes: pool_data_dict[str('{ConfigPool:' + attribute.Name + '}').lower()] = attribute.Value @@ -66,6 +30,7 @@ def pool_data_to_dict(self): split_name = resource_from_pool.Name.split('/') name_of_resource_from_pool = split_name[len(split_name)-1] for attribute in resource_from_pool.ResourceAttributes: - resource_dict_key = str('{ConfigPool:' + name_of_resource_from_pool + ':' + attribute.Name + '}').lower() + resource_dict_key = str('{ConfigPool:' + name_of_resource_from_pool + ':' + + attribute.Name + '}').lower() pool_data_dict[resource_dict_key] = attribute.Value return pool_data_dict diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigPoolManager.py b/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigPoolManager.py new file mode 100644 index 0000000..3a3f2a1 --- /dev/null +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigPoolManager.py @@ -0,0 +1,127 @@ +import unittest +import json +import os +from mock import patch, Mock, call +from sandbox_scripts.QualiEnvironmentUtils.ConfigPoolManager import ConfigPoolManager +from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase +from sandbox_scripts.QualiEnvironmentUtils.Resource import ResourceBase +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError +from cloudshell.api.cloudshell_api import ResourceAttribute +from cloudshell.api.common_cloudshell_api import CloudShellAPIError + +resContext = '''{"id":"5487c6ce-d0b3-43e9-8ee7-e27af8406905", + "ownerUser":"bob", + "ownerPass":"nIqm+BG6ZGJjby5hUittVFFJASc=", + "domain":"Global", + "environmentName":"My environment", + "description":"New demo environment", + "parameters": + { "globalInputs": [], + "resourceRequirements":[], + "resourceAdditionalInfo":[]}}''' + +conContext = '''{"serverAddress": "localhost", +"adminAuthToken": "anAdminToken"}''' + + +class ConfigPoolManagerTests(unittest.TestCase): + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def setUp(self, mock_api_session): + os.environ['reservationContext'] = resContext + os.environ['qualiConnectivityContext'] = conContext + tli = Mock() + tli.Topologies = ["My environment"] + mock_api_session.return_value.GetActiveTopologyNames = Mock(return_value=tli) + + abstractinfo = Mock() + abstractinfo.Alias = "alias" + topoinfo = Mock() + + topoinfo.Name = "My environment" + topoinfo.AbstractResources = [abstractinfo] + mock_api_session.return_value.GetTopologyDetails = Mock(return_value=topoinfo) + mock_logger = Mock() + self.sandbox = SandboxBase(reservation_id="5487c6ce-d0b3-43e9-8ee7-e27af8406905", logger=mock_logger) + + def tearDown(self): + pass + + # ================================================================ + # test push_data_from_pool_to_sandbox function + def test_no_config_manager(self): + with self.assertRaises(QualiError) as e: + self.config_pool_mgr = ConfigPoolManager(sandbox=self.sandbox, pool_resource=None) + + the_exception = e.exception + self.assertEqual(str(the_exception), + 'CloudShell error at ConfigPoolManager. ' + 'Error is: Trying to use the ConfigPoolManager without a pool resource') + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_config_manager_without_attributes(self, mock_api_session): + rd = Mock() + rd.Name = 'pool manager' + rd.Address = '' + rd.ChildResources = [] + rd.ResourceAttributes = [] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='pool manager') + + config_pool_mgr = ConfigPoolManager(sandbox=self.sandbox, pool_resource=resource) + self.assertTrue(len(config_pool_mgr.pool_data) == 0) + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_config_manager_without_child_resources(self, mock_api_session): + rd = Mock() + rd.Name = 'pool manager' + rd.Address = '' + attr1 = Mock() + attr1.Name = 'Pool1' + attr1.Value = 'Pool1Val' + attr2 = Mock() + attr2.Name = 'Pool2' + attr2.Value = 'Pool2Val' + rd.ChildResources = [] + rd.ResourceAttributes = [attr1, attr2] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='pool manager') + + config_pool_mgr = ConfigPoolManager(sandbox=self.sandbox, pool_resource=resource) + self.assertTrue(len(config_pool_mgr.pool_data) == 2) + self.assertTrue('{configpool:pool1}' in config_pool_mgr.pool_data) + self.assertTrue('{configpool:pool2}' in config_pool_mgr.pool_data) + + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_config_manager_with_child_resources(self, mock_api_session): + rd = Mock() + rd.Name = 'pool manager' + rd.Address = '' + attr1 = Mock() + attr1.Name = 'Pool1' + attr1.Value = 'Pool1Val' + attr2 = Mock() + attr2.Name = 'Pool2' + attr2.Value = 'Pool2Val' + cr = Mock() + cr.Name = 'pool view' + cr.Address = '' + attr3 = Mock() + attr3.Name = 'Pool3' + attr3.Value = 'Pool3Val' + cr.ResourceAttributes = [attr3] + rd.ChildResources = [cr] + rd.ResourceAttributes = [attr1, attr2] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='pool manager') + + config_pool_mgr = ConfigPoolManager(sandbox=self.sandbox, pool_resource=resource) + self.assertTrue(len(config_pool_mgr.pool_data) == 3) + self.assertTrue('{configpool:pool1}' in config_pool_mgr.pool_data, "pool1 not found") + self.assertTrue('{configpool:pool2}' in config_pool_mgr.pool_data, "pool2 not found") + self.assertTrue('{configpool:pool view:pool3}' in config_pool_mgr.pool_data, "pool3 not found") + + +if __name__ == '__main__': + unittest.main() + diff --git a/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py b/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py index 3fceb7d..81c771d 100644 --- a/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py +++ b/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py @@ -1,11 +1,12 @@ # coding=utf-8 import csv +import os import tempfile from multiprocessing.pool import ThreadPool from threading import Lock -from sandbox_scripts.QualiEnvironmentUtils.ConfigFileManager import * -from sandbox_scripts.QualiEnvironmentUtils.ConfigPoolManager import * +from sandbox_scripts.QualiEnvironmentUtils.ConfigFileManager import ConfigFileManager +from sandbox_scripts.QualiEnvironmentUtils.ConfigPoolManager import ConfigPoolManager from sandbox_scripts.helpers.Networking.base_save_restore import * from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError @@ -22,9 +23,9 @@ def __init__(self, sandbox): self.storage_mgr = StorageManager(sandbox) self.config_files_root = self.storage_mgr.get_configs_root() else: - if self.is_resources_in_reservation_to_restore(ignore_models = None): + if self.is_resources_in_reservation_to_restore(ignore_models=None): self.sandbox.report_info("Failed to find a storage server resource (e.g. ftp) in the sandbox. ", - write_to_output_window=False) + write_to_output_window=False) # ---------------------------------- # load_network_config(ResourceName,config_type, RestoreMethod=Override) @@ -63,7 +64,7 @@ def load_config(self, config_stage, config_type, restore_method="Override", conf root_path = root_path + config_set_name.strip() + '/' root_path = root_path.replace(' ', '_') - self.sandbox.report_info("RootPath: " + root_path,write_to_output_window=True) + self.sandbox.report_info("RootPath: " + root_path, write_to_output_window=True) images_path_dict = self._get_images_path_dict(root_path) self.sandbox.report_info("\nLoading image and configuration on the devices. This action may take some time", write_to_output_window=True) @@ -249,7 +250,7 @@ def _get_concrete_config_file_path(self, root_path, resource, config_stage, writ config_set_pool_resource = self.sandbox.get_config_set_pool_resource() if config_set_pool_resource is not None: config_set_pool_manager = ConfigPoolManager(sandbox=self.sandbox, pool_resource=config_set_pool_resource) - config_set_pool_data = config_set_pool_manager.pool_data_to_dict() + config_set_pool_data = config_set_pool_manager.pool_data if config_stage == 'snapshots': config_path = root_path + resource.name + '_' + resource.model + '.cfg' else: diff --git a/sandbox_scripts/helpers/Networking/vm_save_restore.py b/sandbox_scripts/helpers/Networking/vm_save_restore.py index c4056e3..2ebd551 100644 --- a/sandbox_scripts/helpers/Networking/vm_save_restore.py +++ b/sandbox_scripts/helpers/Networking/vm_save_restore.py @@ -1,4 +1,8 @@ +import tempfile +import os from sandbox_scripts.helpers.Networking.base_save_restore import * +from sandbox_scripts.QualiEnvironmentUtils.ConfigPoolManager import ConfigPoolManager +from sandbox_scripts.QualiEnvironmentUtils.ConfigFileManager import ConfigFileManager from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError from multiprocessing.pool import ThreadPool from threading import Lock @@ -8,7 +12,6 @@ class VMsSaveRestore(BaseSaveRestore): def __init__(self, sandbox): super(VMsSaveRestore,self).__init__(sandbox) - # ---------------------------------- # ---------------------------------- def load_config(self, config_stage, config_set_name='', ignore_models=None, @@ -79,7 +82,7 @@ def _run_asynch_load(self, resource, root_path, ignore_models, in_teardown_mode) dest_name = resource.name + '_' + resource.model +'_artifact.txt' dest_name = dest_name.replace(' ','-') saved_artifact_info = self.storage_mgr.download_artifact_info(root_path, dest_name) - resource.orchestration_restore(self.sandbox.id,None,saved_artifact_info) + resource.orchestration_restore(self.sandbox.id, None, saved_artifact_info) health_check_result = resource.health_check(self.sandbox.id) if health_check_result != '': @@ -236,7 +239,7 @@ def _get_concrete_config_file_path(self, root_path, resource, config_stage, writ config_set_pool_resource = self.sandbox.get_config_set_pool_resource() if config_set_pool_resource is not None: config_set_pool_manager = ConfigPoolManager(sandbox=self.sandbox, pool_resource=config_set_pool_resource) - config_set_pool_data = config_set_pool_manager.pool_data_to_dict() + config_set_pool_data = config_set_pool_manager.pool_data if config_stage == 'snapshots': config_path = root_path + resource.name + '_' + resource.model + '.cfg' else: From d07ad795fe40563499b4f1872e27ba053de4651d Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Fri, 28 Apr 2017 14:25:23 -0700 Subject: [PATCH 10/15] Config file manager tests + refactoring --- .../ConfigFileManager.py | 53 ++--- .../tests/test_ConfigFileManager.py | 207 ++++++++++++++++++ .../tests/test_ConfigPoolManager.py | 42 +--- .../Networking/NetworkingSaveNRestore.py | 17 +- .../helpers/Networking/vm_save_restore.py | 17 +- test_requirements.txt | 1 + 6 files changed, 268 insertions(+), 69 deletions(-) create mode 100644 sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigFileManager.py diff --git a/sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py b/sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py index 322f5fc..4c5ba43 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py +++ b/sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py @@ -1,62 +1,60 @@ # coding=utf-8 import traceback import re -from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase from time import gmtime, strftime +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError # =================================== # =================================== class ConfigFileManager: - def __init__(self, sandbox): - """ - :param SandboxBase sandbox: The sandbox the config file mgr will work with - """ - self.sandbox = sandbox + def __init__(self): + pass # ---------------------------------- # ---------------------------------- - def create_concrete_config_from_template(self, template_config_data, config_set_pool_data, resource): + def create_concrete_config_from_template(self, template_config_data, config_set_pool_data, sandbox, resource): """ Replace parameters in the template file with concrete values Parameters in the template file are marked with {} :param str template_config_data: The data from the config template file :param dict config_set_pool_data: A dictionary with the data from the config set pool + :param SandboxBase sandbox: The sandbox to get other resources values from :param ResourceBase resource: The resource we want to create the config file for """ try: concrete_config_data = template_config_data # Replace {ConfigPool.PARAM} with PARAM's value from the pool - it = re.finditer(r"\{ConfigPool\:[^}]*\}", concrete_config_data, flags=re.IGNORECASE) + it = re.finditer(r'\{ConfigPool\:[^}]*\}', concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() if param.lower() in config_set_pool_data: concrete_config_data = concrete_config_data.replace(param, config_set_pool_data[param.lower()]) else: - raise ('Could not find attribute ' + param.lower() + ' in the config pool') + raise Exception('Could not find attribute ' + param.lower() + ' in the config pool') # Replace {QUALI-NOTATION} WITH A NOTE - it = re.finditer(r"\{QUALI NOTATION\}", concrete_config_data, flags=re.IGNORECASE) + it = re.finditer(r'\{QUALI NOTATION\}', concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() - quali_note = "Built from template: " + strftime("%Y-%b-%d %H:%M:%S", gmtime()) + quali_note = 'Built from template: ' + strftime('%Y-%b-%d %H:%M:%S', gmtime()) concrete_config_data = concrete_config_data.replace(param, quali_note) # Replace {Device.Self.Name} with the resource's name - it = re.finditer(r"\{Device:Self:Name\}", concrete_config_data, flags=re.IGNORECASE) + it = re.finditer(r'\{Device:Self:Name\}', concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() concrete_config_data = concrete_config_data.replace(param, resource.name) # Replace {Device.Self.Address} with the resource's management ip - it = re.finditer(r"\{Device:Self:Address\}", concrete_config_data, flags=re.IGNORECASE) + it = re.finditer(r'\{Device:Self:Address\}', concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() concrete_config_data = concrete_config_data.replace(param, resource.address) # Replace {Device.Self.ATTRIBUTE_NAME} with the resource's attribute value - # Need to decode password attributes: Password, Enable Password, and SNMP Read Community - it = re.finditer(r"\{Device:Self\:[^}]*\}", concrete_config_data, flags=re.IGNORECASE) + # TODO: Need to decode password attributes: Password, Enable Password, and SNMP Read Community + it = re.finditer(r'\{Device:Self\:[^}]*\}', concrete_config_data, flags=re.IGNORECASE) for match in it: param = match.group() idx = param.rfind(':')+1 @@ -65,25 +63,30 @@ def create_concrete_config_from_template(self, template_config_data, config_set_ # param_val = resource.get_attribute(param) concrete_config_data = concrete_config_data.replace(param, param_val) - # Replacemant of params from types: {Device:ALIAS:Attribute_name} - root_resources = self.sandbox.get_root_networking_resources() - it = re.finditer(r"\{Device:[^}]*\}", concrete_config_data, flags=re.IGNORECASE) + # Replacement of params from types: {Device:ALIAS:Attribute_name} + it = re.finditer(r'\{Device:[^}]*\}', concrete_config_data, flags=re.IGNORECASE) + root_resources = None for match in it: param = match.group() junk, sb_alias, alias_attribname = param.split(":") - alias_attribname = alias_attribname.replace("}","") + alias_attribname = alias_attribname.replace("}", "") concrete_name = '' + if root_resources is None: # fetch once the resources + root_resources = sandbox.get_root_networking_resources() for resource in root_resources: if resource.alias == sb_alias: concrete_name = resource.name - param_val = resource.get_attribute(alias_attribname) + if resource.attribute_exist(alias_attribname): + param_val = resource.get_attribute(alias_attribname) + else: + raise Exception("Could not find attribute '{0}' in resource '{1}'".format(alias_attribname, + resource.name)) concrete_config_data = concrete_config_data.replace(param, param_val) break if concrete_name <= ' ': - raise ('did not find concrete device with alias ' + sb_alias + '; likely missing from blueprint.') + raise Exception('Could not find a resource with alias ' + sb_alias + '; likely missing from blueprint.') return concrete_config_data - except: - print str(Exception.message) - self.sandbox.report_error("Failed to create a concrete config file from the template\'s data. " - "Unexpected error: " + traceback.format_exc()) + except Exception as ex: + raise QualiError('ConfigFileManager', "Failed to create a concrete config file from the template\'s data. " + "Unexpected error: " + ex.message) diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigFileManager.py b/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigFileManager.py new file mode 100644 index 0000000..29ce898 --- /dev/null +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigFileManager.py @@ -0,0 +1,207 @@ +import unittest +import time +from freezegun import freeze_time +from mock import patch, Mock, call +from sandbox_scripts.QualiEnvironmentUtils.Resource import ResourceBase +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError +from sandbox_scripts.QualiEnvironmentUtils.ConfigFileManager import ConfigFileManager + + +class ConfigFileManagerTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + # ================================================================ + # test create_concrete_config_from_template function + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_values_only_from_pool_when_pool_doesnt_have_the_attribute(self, mock_api_session): + config_file_mgr = ConfigFileManager() + + sandbox = None + resource = None + + tmp_template_config_file_data = """ + {ConfigPool:Pool1} + """ + + config_set_pool_data = {} + with self.assertRaises(QualiError) as e: + config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + the_exception = e.exception + self.assertEqual(str(the_exception), + "CloudShell error at ConfigFileManager. " + "Error is: Failed to create a concrete config file from the template's data. " + "Unexpected error: Could not find attribute {configpool:pool1} in the config pool") + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_values_only_from_pool(self, mock_api_session): + config_file_mgr = ConfigFileManager() + + sandbox = None + resource = None + + tmp_template_config_file_data = """{ConfigPool:Pool1}""" + + config_set_pool_data = { + '{configpool:pool1}': 'Pool1Val' + } + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + self.assertTrue(len(concrete_config_data) > 0) + self.assertEqual(concrete_config_data, "Pool1Val") + + def test_quali_notation(self): + config_file_mgr = ConfigFileManager() + + sandbox = None + resource = None + + tmp_template_config_file_data = """{QUALI NOTATION}""" + + config_set_pool_data = {} + with freeze_time("2017-01-17 12:00:01"): + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + self.assertTrue(len(concrete_config_data) > 0) + self.assertEqual(concrete_config_data, 'Built from template: 2017-Jan-17 12:00:01') + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_resource_name(self, mock_api_session): + config_file_mgr = ConfigFileManager() + + rd = Mock() + rd.Name = 'myresource' + rd.Address = '1.2.3.4' + rd.ChildResources = [] + rd.ResourceAttributes = [] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='myresource') + + sandbox = None + + tmp_template_config_file_data = """{Device:Self:Name}""" + + config_set_pool_data = {} + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + self.assertTrue(len(concrete_config_data) > 0) + self.assertEqual(concrete_config_data, "myresource") + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_resource_address(self, mock_api_session): + config_file_mgr = ConfigFileManager() + + rd = Mock() + rd.Name = 'myresource' + rd.Address = '1.2.3.4' + rd.ChildResources = [] + rd.ResourceAttributes = [] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='myresource') + + sandbox = None + + tmp_template_config_file_data = """{Device:Self:Address}""" + + config_set_pool_data = {} + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + self.assertTrue(len(concrete_config_data) > 0) + self.assertEqual(concrete_config_data, "1.2.3.4") + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_resource_attribute(self, mock_api_session): + config_file_mgr = ConfigFileManager() + + rd = Mock() + rd.Name = 'myresource' + rd.Address = '1.2.3.4' + attr1 = Mock() + attr1.Name = 'Pool1' + attr1.Value = 'Pool1Val' + rd.ChildResources = [] + rd.ResourceAttributes = [attr1] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='myresource') + + sandbox = None + + tmp_template_config_file_data = """{Device:Self:Pool1}""" + + config_set_pool_data = {} + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + self.assertTrue(len(concrete_config_data) > 0) + self.assertEqual(concrete_config_data, "Pool1Val") + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_attribute_from_other_resource_in_sandbox(self, mock_api_session): + config_file_mgr = ConfigFileManager() + + rd = Mock() + rd.Name = 'myresource' + rd.Address = '1.2.3.4' + attr1 = Mock() + attr1.Name = 'Pool1' + attr1.Value = 'Pool1Val' + rd.ChildResources = [] + rd.ResourceAttributes = [attr1] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='myresource', resource_alias='OtherDevice') + + sandbox = Mock() + sandbox.get_root_networking_resources.return_value = [resource] + + tmp_template_config_file_data = """{Device:OtherDevice:Pool1}""" + + config_set_pool_data = {} + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + self.assertTrue(len(concrete_config_data) > 0) + self.assertEqual(concrete_config_data, "Pool1Val") + + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_attribute_from_other_resource_in_sandbox_not_found(self, mock_api_session): + config_file_mgr = ConfigFileManager() + + rd = Mock() + rd.Name = 'myresource' + rd.Address = '1.2.3.4' + attr1 = Mock() + attr1.Name = 'Pool1' + attr1.Value = 'Pool1Val' + rd.ChildResources = [] + rd.ResourceAttributes = [attr1] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='myresource', resource_alias='OtherDevice') + + sandbox = Mock() + sandbox.get_root_networking_resources.return_value = [resource] + + tmp_template_config_file_data = """{Device:OtherDevice:Pool123}""" + + config_set_pool_data = {} + with self.assertRaises(QualiError) as e: + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + the_exception = e.exception + self.assertEqual(str(the_exception), + "CloudShell error at ConfigFileManager. " + "Error is: Failed to create a concrete config file from the template's data. " + "Unexpected error: Could not find attribute 'Pool123' in resource 'myresource'") + + +if __name__ == '__main__': + unittest.main() diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigPoolManager.py b/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigPoolManager.py index 3a3f2a1..3320beb 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigPoolManager.py +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigPoolManager.py @@ -1,53 +1,19 @@ import unittest -import json -import os -from mock import patch, Mock, call +from mock import patch, Mock from sandbox_scripts.QualiEnvironmentUtils.ConfigPoolManager import ConfigPoolManager -from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase from sandbox_scripts.QualiEnvironmentUtils.Resource import ResourceBase from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError -from cloudshell.api.cloudshell_api import ResourceAttribute -from cloudshell.api.common_cloudshell_api import CloudShellAPIError - -resContext = '''{"id":"5487c6ce-d0b3-43e9-8ee7-e27af8406905", - "ownerUser":"bob", - "ownerPass":"nIqm+BG6ZGJjby5hUittVFFJASc=", - "domain":"Global", - "environmentName":"My environment", - "description":"New demo environment", - "parameters": - { "globalInputs": [], - "resourceRequirements":[], - "resourceAdditionalInfo":[]}}''' - -conContext = '''{"serverAddress": "localhost", -"adminAuthToken": "anAdminToken"}''' class ConfigPoolManagerTests(unittest.TestCase): - @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') - def setUp(self, mock_api_session): - os.environ['reservationContext'] = resContext - os.environ['qualiConnectivityContext'] = conContext - tli = Mock() - tli.Topologies = ["My environment"] - mock_api_session.return_value.GetActiveTopologyNames = Mock(return_value=tli) - - abstractinfo = Mock() - abstractinfo.Alias = "alias" - topoinfo = Mock() - - topoinfo.Name = "My environment" - topoinfo.AbstractResources = [abstractinfo] - mock_api_session.return_value.GetTopologyDetails = Mock(return_value=topoinfo) - mock_logger = Mock() - self.sandbox = SandboxBase(reservation_id="5487c6ce-d0b3-43e9-8ee7-e27af8406905", logger=mock_logger) + def setUp(self): + self.sandbox = None def tearDown(self): pass # ================================================================ - # test push_data_from_pool_to_sandbox function + # test ConfigPoolManager/pool_data function def test_no_config_manager(self): with self.assertRaises(QualiError) as e: self.config_pool_mgr = ConfigPoolManager(sandbox=self.sandbox, pool_resource=None) diff --git a/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py b/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py index 81c771d..d5df163 100644 --- a/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py +++ b/sandbox_scripts/helpers/Networking/NetworkingSaveNRestore.py @@ -243,7 +243,7 @@ def _get_concrete_config_file_path(self, root_path, resource, config_stage, writ :rtype: str """ - config_file_mgr = ConfigFileManager(self.sandbox) + config_file_mgr = ConfigFileManager() #TODO - set the pool dictionary only once during the init of the class config_set_pool_data = dict() # If there is a pool resource, get the pool data @@ -277,8 +277,19 @@ def _get_concrete_config_file_path(self, root_path, resource, config_stage, writ self.storage_mgr.download(tftp_template_config_path, tmp_template_config_file.name) with open(tmp_template_config_file.name, 'r') as content_file: tmp_template_config_file_data = content_file.read() - concrete_config_data = config_file_mgr.create_concrete_config_from_template( - tmp_template_config_file_data, config_set_pool_data, resource) + + concrete_config_data = '' + try: + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, + config_set_pool_data, + self.sandbox, resource) + except QualiError as qe: + self.sandbox.report_error(error_message='Could not create a concrete config file ' + 'for resource {0}'.format(resource.name), + log_message=qe.message, + write_to_output_window=True) + tmp_concrete_config_file = tempfile.NamedTemporaryFile(delete=False) tf = file(tmp_concrete_config_file.name, 'wb+') tf.write(concrete_config_data) diff --git a/sandbox_scripts/helpers/Networking/vm_save_restore.py b/sandbox_scripts/helpers/Networking/vm_save_restore.py index 2ebd551..de1366e 100644 --- a/sandbox_scripts/helpers/Networking/vm_save_restore.py +++ b/sandbox_scripts/helpers/Networking/vm_save_restore.py @@ -232,7 +232,7 @@ def _get_concrete_config_file_path(self, root_path, resource, config_stage, writ :rtype: str """ - config_file_mgr = ConfigFileManager(self.sandbox) + config_file_mgr = ConfigFileManager() #TODO - set the pool dictionary only once during the init of the class config_set_pool_data = dict() # If there is a pool resource, get the pool data @@ -266,8 +266,19 @@ def _get_concrete_config_file_path(self, root_path, resource, config_stage, writ self.storage_mgr.download(tftp_template_config_path, tmp_template_config_file.name) with open(tmp_template_config_file.name, 'r') as content_file: tmp_template_config_file_data = content_file.read() - concrete_config_data = config_file_mgr.create_concrete_config_from_template( - tmp_template_config_file_data, config_set_pool_data, resource) + + concrete_config_data = '' + try: + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, + config_set_pool_data, + self.sandbox, resource) + except QualiError as qe: + self.sandbox.report_error(error_message='Could not create a concrete config file ' + 'for resource {0}'.format(resource.name), + log_message=qe.message, + write_to_output_window=True) + tmp_concrete_config_file = tempfile.NamedTemporaryFile(delete=False) tf = file(tmp_concrete_config_file.name, 'wb+') tf.write(concrete_config_data) diff --git a/test_requirements.txt b/test_requirements.txt index 4a6670f..375b29c 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -6,3 +6,4 @@ jsonpickle nose-exclude coveralls pyapi-gitlab +freezegun From a2f11f50b66cb01336a0607cdffc93d58d7c6ef3 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Fri, 28 Apr 2017 14:36:13 -0700 Subject: [PATCH 11/15] Added another test --- .../tests/test_ConfigFileManager.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigFileManager.py b/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigFileManager.py index 29ce898..3b5c528 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigFileManager.py +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/test_ConfigFileManager.py @@ -202,6 +202,38 @@ def test_attribute_from_other_resource_in_sandbox_not_found(self, mock_api_sessi "Error is: Failed to create a concrete config file from the template's data. " "Unexpected error: Could not find attribute 'Pool123' in resource 'myresource'") + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def test_attribute_from_other_resource_in_sandbox_resource_not_found(self, mock_api_session): + config_file_mgr = ConfigFileManager() + + rd = Mock() + rd.Name = 'myresource' + rd.Address = '1.2.3.4' + attr1 = Mock() + attr1.Name = 'Pool1' + attr1.Value = 'Pool1Val' + rd.ChildResources = [] + rd.ResourceAttributes = [attr1] + mock_api_session.return_value.GetResourceDetails = Mock(return_value=rd) + resource = ResourceBase(resource_name='myresource', resource_alias='OtherDevice') + + sandbox = Mock() + sandbox.get_root_networking_resources.return_value = [resource] + + tmp_template_config_file_data = """{Device:OtherDevice2:Pool1}""" + + config_set_pool_data = {} + with self.assertRaises(QualiError) as e: + concrete_config_data = config_file_mgr.create_concrete_config_from_template( + tmp_template_config_file_data, config_set_pool_data, sandbox, resource) + + the_exception = e.exception + self.assertEqual(str(the_exception), + "CloudShell error at ConfigFileManager. Error is: " + "Failed to create a concrete config file from the template's data. " + "Unexpected error: Could not find a resource with alias OtherDevice2; " + "likely missing from blueprint.") + if __name__ == '__main__': unittest.main() From fa218d993df662240614cd513107f2fbd172eada Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Fri, 28 Apr 2017 14:39:05 -0700 Subject: [PATCH 12/15] Omit the default setup/teardown scripts from the coverage --- .coveragerc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.coveragerc b/.coveragerc index 6fc93d8..4f90f1c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -11,3 +11,5 @@ omit = */profiler/* */tests/* */remap_child_resources_constants.py + */setup_script.py + */teardown_script.py From 7f421cc52391850e70f1c759259f1245797920ed Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Fri, 28 Apr 2017 14:42:24 -0700 Subject: [PATCH 13/15] Empty test file for storage manager --- .../tests/test_StorageManager.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 sandbox_scripts/QualiEnvironmentUtils/tests/test_StorageManager.py diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/test_StorageManager.py b/sandbox_scripts/QualiEnvironmentUtils/tests/test_StorageManager.py new file mode 100644 index 0000000..7b14ebf --- /dev/null +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/test_StorageManager.py @@ -0,0 +1,36 @@ +import unittest +from mock import patch, Mock,call +from sandbox_scripts.QualiEnvironmentUtils.StorageManager import StorageManager +from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError +from cloudshell.api.cloudshell_api import ReservationDescriptionInfo +import json +import os +from cloudshell.api.common_cloudshell_api import CloudShellAPIError + +resContext = '''{"id":"5487c6ce-d0b3-43e9-8ee7-e27af8406905", + "ownerUser":"bob", + "ownerPass":"nIqm+BG6ZGJjby5hUittVFFJASc=", + "domain":"Global", + "environmentName":"My environment", + "description":"New demo environment", + "parameters": + { "globalInputs": [], + "resourceRequirements":[], + "resourceAdditionalInfo":[]}}''' + +conContext = '''{"serverAddress": "localhost", +"adminAuthToken": "anAdminToken"}''' + + +class StorageManagerTests(unittest.TestCase): + @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') + def setUp(self, mock_api_session): + pass + + def tearDown(self): + pass + +# TODO: implement tests + +if __name__ == '__main__': + unittest.main() From fc53857a53c0ec8be38c7818a2d5b9a7a9674aa9 Mon Sep 17 00:00:00 2001 From: ayeleta Date: Thu, 4 May 2017 10:53:46 -0700 Subject: [PATCH 14/15] tests for load_network_config + health_check --- .../QualiEnvironmentUtils/Resource.py | 16 +- .../tests/DebugInteractive_setup_resources.py | 8 +- .../tests/test_Resource.py | 160 +++++++++++++++++- 3 files changed, 168 insertions(+), 16 deletions(-) diff --git a/sandbox_scripts/QualiEnvironmentUtils/Resource.py b/sandbox_scripts/QualiEnvironmentUtils/Resource.py index 3e11584..e5dc14d 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/Resource.py +++ b/sandbox_scripts/QualiEnvironmentUtils/Resource.py @@ -123,19 +123,22 @@ def load_network_config(self, reservation_id, config_path, config_type, restore_ the_path = "undef" the_cfgtype = "undef" the_restoremeth = "undef" + found_cmd = False for command in self.commands: if command.Name == 'restore': + found_cmd = True for parm in command.Parameters: if parm.Name in ["path", "src_Path"]: the_path = parm.Name - if parm.Name in ["configuration_type","config_type"]: + elif parm.Name in ["configuration_type","config_type"]: the_cfgtype = parm.Name - if parm.Name in ["restore_method"]: + elif parm.Name in ["restore_method"]: the_restoremeth = parm.Name - + if not found_cmd: + raise QualiError(self.name, "Restore command does not exist. Check driver installation.") if the_path == "undef" or the_cfgtype == "undef" or the_restoremeth == "undef": raise QualiError(self.name, "Failed to find viable restore command for " + self.name \ - + " : " + the_path + ", " + the_cfgtype + ", " + the_restoremeth) + + " - config_path: " + the_path + ", config_type: " + the_cfgtype + ", restore_method: " + the_restoremeth) except QualiError as qerror: raise QualiError(self.name, "Failed building restore command input parm names." + qerror.message) @@ -157,8 +160,8 @@ def load_network_config(self, reservation_id, config_path, config_type, restore_ except QualiError as qerror: raise QualiError(self.name, "Failed to load configuration: " + qerror.message) - except: - raise "Failed to load configuration. Unexpected error:" + str(sys.exc_info()[0]) + except Exception as e: + raise QualiError(self.name, "Failed to load configuration. Unexpected error:" + e.message) # ----------------------------------------- # ----------------------------------------- @@ -169,6 +172,7 @@ def save_network_config(self, reservation_id, config_path, config_type): :param config_path: The path where to save the config file :param config_type: StartUp or Running """ + #TODO modify the function to identify the command name and its params (similar behavior as in load_network_config) # Run executeCommand with the restore command and its params (ConfigPath,RestoreMethod) try: command_inputs = [InputNameValue('source_filename', str(config_type)), diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py b/sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py index 1a0d4ce..af9f368 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/DebugInteractive_setup_resources.py @@ -1,10 +1,10 @@ import cloudshell.helpers.scripts.cloudshell_dev_helpers as dev_helpers from sandbox_scripts.environment.setup.setup_resources import * -dev_helpers.attach_to_cloudshell_as(user="admin", password="dev", domain="Global", - reservation_id="d7be79c2-57d0-4063-9ada-e27cd3c608a7", - server_address="svl-dev-quali") -os.environ["environment_name"] = "Abstract-ALL" +dev_helpers.attach_to_cloudshell_as(user="admin", password="admin", domain="Global", + reservation_id="c04d3da4-8025-4efe-9f4d-820ba19d20af", + server_address="localhost") +#os.environ["environment_name"] = "Abstract-ALL" x = EnvironmentSetupResources() x.execute() diff --git a/sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py b/sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py index 91620d0..86e4c63 100644 --- a/sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py +++ b/sandbox_scripts/QualiEnvironmentUtils/tests/test_Resource.py @@ -2,6 +2,7 @@ from mock import patch, Mock,call from sandbox_scripts.QualiEnvironmentUtils.Resource import ResourceBase from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError +from cloudshell.api.cloudshell_api import CommandParameter from cloudshell.api.cloudshell_api import ReservationDescriptionInfo from cloudshell.api.cloudshell_api import ResourceCommandInfo import json @@ -100,10 +101,10 @@ def test_attribute_found_case_insensitive(self): self.assertTrue(ret, "attribute was expected to be found but wasn't") def test_attribute_not_found(self): - command1 = Mock() - command1.Name = 'Location' - self.resource.connected_commands = [command1] - ret = self.resource.has_command('LocationXYZ') + attr1 = Mock() + attr1.Name = 'Location' + self.resource.attributes = [attr1] + ret = self.resource.attribute_exist('LocationXYZ') self.assertFalse(ret, "attribute was not expected to be found but was found") @@ -142,10 +143,157 @@ def test_get_attribute_not_found(self): self.assertEqual(str(the_exception), "CloudShell error at r1. Error is: Attribute: 'LocationXYZ' not found") + # ================================================================ + # test health_check + def test_health_check_passed(self): + command1 = Mock() + command1.Name = 'health_check' + self.resource.commands = [command1] + + rd = Mock() + rd.Output = "Health check passed" + self.mock_api_session.return_value.ExecuteCommand = Mock(return_value=rd) + ret = self.resource.health_check("5487c6ce-d0b3-43e9-8ee7-e27af8406905") + self.assertEqual('',ret, "command was expected to be pass but wasn't") + + def test_health_check_fail_w_exception(self): + command1 = Mock() + command1.Name = 'health_check' + self.resource.commands = [command1] + self.mock_api_session.return_value.ExecuteCommand.side_effect = CloudShellAPIError(100,"Invalid command","") + + ret = self.resource.health_check("5487c6ce-d0b3-43e9-8ee7-e27af8406905") + self.assertEqual('Health check did not pass for device r1. CloudShell error at r1. Error is: Invalid command',ret) + + def test_health_check_failed(self): + command1 = Mock() + command1.Name = 'health_check' + self.resource.commands = [command1] + + rd = Mock() + rd.Output = "Health check failed" + self.mock_api_session.return_value.ExecuteCommand = Mock(return_value=rd) + ret = self.resource.health_check("5487c6ce-d0b3-43e9-8ee7-e27af8406905") + self.assertEqual('Health check did not pass for device r1. Health check failed',ret) + + def test_health_check_not_found(self): + ret = self.resource.health_check("5487c6ce-d0b3-43e9-8ee7-e27af8406905") + self.assertEqual('',ret, "command was expected to be found but wasn't") + + # ================================================================ + # test load_network_config + def test_load_network_config_restore_not_exist(self): + try: + self.resource.load_network_config("5487c6ce-d0b3-43e9-8ee7-e27af8406905",'tftp://www.cfg','running') + self.fail("Didn't get an exception while expecting to. load_network_config does not exist." ) + except QualiError as e: + self.assertEqual('Failed building restore command input parm names.Restore command does not exist. Check driver installation.', + e.message) + + def test_load_network_config_wrong_param(self): + command1 = Mock() + command1.Name = 'restore' + param1 = Mock() + param1.Name = 'path123' + param2 = Mock() + param2.Name = 'configuration_type' + param3 = Mock() + param3.Name = 'restore_method' + command1.Parameters = [param1, param2, param3] + self.resource.commands = [command1] + with self.assertRaises(Exception) as e: + self.resource.load_network_config("5487c6ce-d0b3-43e9-8ee7-e27af8406905",'tftp://www.cfg','running') + + the_exception = e.exception + self.assertEqual(str(the_exception), + 'CloudShell error at r1. Error is: Failed building restore command input parm names.Failed to find viable restore command for r1 - config_path: undef, config_type: configuration_type, restore_method: restore_method') + + def test_load_network_config_passed(self): + command1 = Mock() + command1.Name = 'restore' + param1 = Mock() + param1.Name = 'path' + param2 = Mock() + param2.Name = 'configuration_type' + param3 = Mock() + param3.Name = 'restore_method' + command1.Parameters = [param1, param2, param3] + self.resource.commands = [command1] + + try: + self.resource.load_network_config("5487c6ce-d0b3-43e9-8ee7-e27af8406905",'tftp://www.cfg','running') + except Exception as e: + self.fail('Got an unexpected exception: ' + e.message) + + def test_load_network_config_failed(self): + command1 = Mock() + command1.Name = 'restore' + param1 = Mock() + param1.Name = 'path' + param2 = Mock() + param2.Name = 'configuration_type' + param3 = Mock() + param3.Name = 'restore_method' + command1.Parameters = [param1, param2, param3] + self.resource.commands = [command1] + self.mock_api_session.return_value.ExecuteCommand.side_effect = CloudShellAPIError(100,"Device rejected connection","") + with self.assertRaises(QualiError) as e: + self.resource.load_network_config("5487c6ce-d0b3-43e9-8ee7-e27af8406905",'tftp://www.cfg','running') + + the_exception = e.exception + self.assertEqual(str(the_exception), + 'CloudShell error at r1. Error is: Failed to load configuration: Device rejected connection') + + def test_load_network_config_w_vrf(self): + command1 = Mock() + command1.Name = 'restore' + param1 = Mock() + param1.Name = 'path' + param2 = Mock() + param2.Name = 'configuration_type' + param3 = Mock() + param3.Name = 'restore_method' + command1.Parameters = [param1, param2, param3] + self.resource.commands = [command1] + + attr1 = Mock() + attr1.Name = 'VRF Management Name' + attr1.Value = '123' + self.resource.attributes = [attr1] + try: + self.resource.load_network_config("5487c6ce-d0b3-43e9-8ee7-e27af8406905",'tftp://www.cfg','running') + #todo: check command_inputs size is 4 + call_args = self.mock_api_session.return_value.ExecuteCommand.call_args[0] + self.assertEqual(call_args[0], '5487c6ce-d0b3-43e9-8ee7-e27af8406905') + self.assertEqual(call_args[1], 'r1') + self.assertEqual(call_args[2], 'Resource') + self.assertEqual(call_args[3], 'restore') + command_inputs = call_args[4] + self.assertEqual(command_inputs[0].Name, 'path') + self.assertEqual(command_inputs[0].Value, 'tftp://www.cfg') + self.assertEqual(command_inputs[1].Name, 'configuration_type') + self.assertEqual(command_inputs[1].Value, 'running') + self.assertEqual(command_inputs[2].Name, 'restore_method') + self.assertEqual(command_inputs[2].Value, 'Override') + self.assertEqual(command_inputs[3].Name, 'vrf_management_name') + self.assertEqual(command_inputs[3].Value, '123') + except Exception as e: + self.fail('Got an unexpected exception: ' + e.message) + + # ================================================================ + # test save_network_config + ''' + def test_save_network_config_restore_not_exist(self): + try: + self.resource.save_network_config("5487c6ce-d0b3-43e9-8ee7-e27af8406905",'tftp://www.cfg','running') + self.fail("Didn't get an exception while expecting to. save_network_config does not exist." ) + except QualiError as e: + self.assertEqual('Failed to save configuration: No commands were found', + e.message) + + ''' # TODO: much more - # health_check - # load_network_config # save_network_config # orchestration_restore # orchestration_save From 301cedd829bb5af256ae86a9625b1adcac1582b7 Mon Sep 17 00:00:00 2001 From: Yaniv Kalsky Date: Fri, 5 May 2017 14:53:13 -0700 Subject: [PATCH 15/15] Fix test (parallel may cause other order of calls) --- sandbox_scripts/environment/setup/tests/test_setup_VM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox_scripts/environment/setup/tests/test_setup_VM.py b/sandbox_scripts/environment/setup/tests/test_setup_VM.py index 1837740..513a248 100644 --- a/sandbox_scripts/environment/setup/tests/test_setup_VM.py +++ b/sandbox_scripts/environment/setup/tests/test_setup_VM.py @@ -82,7 +82,7 @@ def resource_details_mock_side_effect(name): logger_debug_calls = [call("Skipping resource 'r1' - not an app, not powering on"), call("Skipping resource 'r2' - not an app, not powering on")] - self.setup_script.logger.debug.assert_has_calls(logger_debug_calls) + self.setup_script.logger.debug.assert_has_calls(logger_debug_calls, any_order=True) @patch('cloudshell.helpers.scripts.cloudshell_scripts_helpers.get_api_session') @patch('sandbox_scripts.environment.setup.setup_VM.SandboxBase')