From 8c9f259d361e19850ead75efefa053113d4efbd9 Mon Sep 17 00:00:00 2001 From: Alex Azarh Date: Wed, 22 Feb 2017 18:02:22 +0200 Subject: [PATCH 01/14] fixing descriptions and error messages --- drivers/openstack_shellPackage/DataModel/datamodel.xml | 4 ++-- .../cp/openstack/command/operations/deploy_operation.py | 3 +-- .../openstack/domain/services/cp_validators/cp_validator.py | 2 +- .../openstack/domain/services/nova/nova_instance_service.py | 2 ++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/openstack_shellPackage/DataModel/datamodel.xml b/drivers/openstack_shellPackage/DataModel/datamodel.xml index 0ba6fc8..fcfc9a2 100644 --- a/drivers/openstack_shellPackage/DataModel/datamodel.xml +++ b/drivers/openstack_shellPackage/DataModel/datamodel.xml @@ -42,7 +42,7 @@ - + @@ -95,7 +95,7 @@ - + diff --git a/package/cloudshell/cp/openstack/command/operations/deploy_operation.py b/package/cloudshell/cp/openstack/command/operations/deploy_operation.py index 398c58d..6361125 100644 --- a/package/cloudshell/cp/openstack/command/operations/deploy_operation.py +++ b/package/cloudshell/cp/openstack/command/operations/deploy_operation.py @@ -38,10 +38,9 @@ def deploy(self, os_session, name, reservation, cp_resource_model, deploy_req_mo logger.info("Inside Deploy Operation.") # First create instance instance = None - floating_ip_dict = None floating_ip_str = '' try: - # Look for right at the beginning -- ??? + # Check for cancellation right at the beginning self.cancellation_service.check_if_cancelled(cancellation_context=cancellation_context) instance = self.instance_service.create_instance(openstack_session=os_session, diff --git a/package/cloudshell/cp/openstack/domain/services/cp_validators/cp_validator.py b/package/cloudshell/cp/openstack/domain/services/cp_validators/cp_validator.py index 04530df..8460f71 100644 --- a/package/cloudshell/cp/openstack/domain/services/cp_validators/cp_validator.py +++ b/package/cloudshell/cp/openstack/domain/services/cp_validators/cp_validator.py @@ -36,7 +36,7 @@ def validate_openstack_username(self, os_username, logger): def validate_openstack_password(self, os_password, logger): - return self._is_not_empty(os_password, "OpenStack Passwrd") + return self._is_not_empty(os_password, "OpenStack Password") def _is_not_empty(self, value, err_value): diff --git a/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py b/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py index 84f78cd..32b6c1e 100644 --- a/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py +++ b/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py @@ -38,6 +38,8 @@ def create_instance(self, openstack_session, name, reservation, if not openstack_session or not name or not reservation or \ not deploy_req_model: return None + if not deploy_req_model.instance_flavor: + raise ValueError("Instance flavor cannot be empty.") client = novaclient.Client(self.API_VERSION, session=openstack_session) From b69ed2a93e1a7e4fff5d87e4b2902a2cae5b6596 Mon Sep 17 00:00:00 2001 From: Alex Azarh Date: Wed, 22 Feb 2017 18:56:46 +0200 Subject: [PATCH 02/14] refactored instance rollback --- .../drivermetadata.xml | 2 +- .../openstack_shell/src/drivermetadata.xml | 2 +- drivers/version.txt | 2 +- .../command/operations/deploy_operation.py | 57 +++++++++++++------ package/version.txt | 2 +- version.txt | 2 +- 6 files changed, 45 insertions(+), 22 deletions(-) diff --git a/drivers/deployment_drivers/openstack_nova_image_instance/drivermetadata.xml b/drivers/deployment_drivers/openstack_nova_image_instance/drivermetadata.xml index 407140c..b77e9e3 100644 --- a/drivers/deployment_drivers/openstack_nova_image_instance/drivermetadata.xml +++ b/drivers/deployment_drivers/openstack_nova_image_instance/drivermetadata.xml @@ -1,4 +1,4 @@ - + diff --git a/drivers/openstack_shell/src/drivermetadata.xml b/drivers/openstack_shell/src/drivermetadata.xml index 0c9aa49..b022856 100644 --- a/drivers/openstack_shell/src/drivermetadata.xml +++ b/drivers/openstack_shell/src/drivermetadata.xml @@ -1,4 +1,4 @@ - + diff --git a/drivers/version.txt b/drivers/version.txt index e484aaf..927734f 100644 --- a/drivers/version.txt +++ b/drivers/version.txt @@ -1 +1 @@ -0.0.16 \ No newline at end of file +0.0.17 \ No newline at end of file diff --git a/package/cloudshell/cp/openstack/command/operations/deploy_operation.py b/package/cloudshell/cp/openstack/command/operations/deploy_operation.py index 6361125..227e297 100644 --- a/package/cloudshell/cp/openstack/command/operations/deploy_operation.py +++ b/package/cloudshell/cp/openstack/command/operations/deploy_operation.py @@ -32,7 +32,7 @@ def deploy(self, os_session, name, reservation, cp_resource_model, deploy_req_mo :param DeployDataHolder deploy_req_model: :param OpenStackResourceModel cp_resource_model: :param cloudshell.shell.core.context.CancellationContext cancellation_context: - :param LoggingSessionContext logger: + :param logging.Logger logger: :rtype DeployResultModel: """ logger.info("Inside Deploy Operation.") @@ -73,9 +73,10 @@ def deploy(self, os_session, name, reservation, cp_resource_model, deploy_req_mo else: floating_ip_subnet_uuid = cp_resource_model.floating_ip_subnet_uuid - floating_ip_dict = self.network_service.create_floating_ip(floating_ip_subnet_id=floating_ip_subnet_uuid, - openstack_session=os_session, - logger=logger) + floating_ip_dict = self.network_service.create_floating_ip( + floating_ip_subnet_id=floating_ip_subnet_uuid, + openstack_session=os_session, + logger=logger) if floating_ip_dict: floating_ip_str = floating_ip_dict['floating_ip_address'] if floating_ip_str: @@ -104,18 +105,40 @@ def deploy(self, os_session, name, reservation, cp_resource_model, deploy_req_mo # If any Exception is raised during deploy or assign floating IP - clean up OpenStack side and re-raise except Exception as e: logger.error(traceback.format_exc()) - if instance: - instance_id = instance.id - # This calls detach and delete floating IP and instance terminate (handles empty floating IP) - self.instance_service.detach_floating_ip(openstack_session=os_session, - instance=instance, - floating_ip=floating_ip_str, - logger=logger) - self.network_service.delete_floating_ip(openstack_session=os_session, - floating_ip=floating_ip_str, - logger=logger) - self.instance_service.terminate_instance(openstack_session=os_session, - instance_id=instance_id, - logger=logger) + + self._rollback_failed_instance(logger=logger, + os_session=os_session, + floating_ip=floating_ip_str, + instance=instance) + # Re-raise for the UI raise + + def _rollback_failed_instance(self, logger, os_session, floating_ip, instance): + """ + + :param logging.Logger logger: + :param keystoneauth1.session.Session os_session: + :param str floating_ip: + :param novaclient.Client.servers.Server instance: + :return: + """ + if not instance: + return + instance_id = instance.id + + # This calls detach and delete floating IP and instance terminate (handles empty floating IP) + try: + self.instance_service.detach_floating_ip(openstack_session=os_session, + instance=instance, + floating_ip=floating_ip, + logger=logger) + self.network_service.delete_floating_ip(openstack_session=os_session, + floating_ip=floating_ip, + logger=logger) + except Exception: + logger.exception("Failed to remove floating ip {}. ".format(floating_ip)) + + self.instance_service.terminate_instance(openstack_session=os_session, + instance_id=instance_id, + logger=logger) diff --git a/package/version.txt b/package/version.txt index e484aaf..927734f 100644 --- a/package/version.txt +++ b/package/version.txt @@ -1 +1 @@ -0.0.16 \ No newline at end of file +0.0.17 \ No newline at end of file diff --git a/version.txt b/version.txt index e484aaf..927734f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.16 \ No newline at end of file +0.0.17 \ No newline at end of file From aa9f5dfd66ed47ec1940e8767a3aaac746c6a27f Mon Sep 17 00:00:00 2001 From: Abhijit Gadgil Date: Thu, 23 Feb 2017 18:21:03 +0530 Subject: [PATCH 03/14] #75 Fix - A much simpler and robust algorithm for subnet allocation - Quick testing with multiple create/delete subnets --- .../neutron/neutron_network_service.py | 105 ++++++++++-------- .../test_neutron_network_service.py | 12 +- 2 files changed, 71 insertions(+), 46 deletions(-) diff --git a/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py b/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py index 7204d75..73f035b 100644 --- a/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py +++ b/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py @@ -3,7 +3,7 @@ import traceback import time - +import ipaddress class NeutronNetworkService(object): """ @@ -155,57 +155,74 @@ def _get_unused_cidr(self, client, cp_resvd_cidrs, logger): :return str: """ - # Algorithm below is a very simplistic one where we choose one of the three prefixes and then use - # /24 networks starting with that prefix. This algorithm will break if all three 10.X, 192.168.X and 172.X - # networks are used in a given On Prem Network. - + # We basically start with a 10.0. network to find a subnet that does not overlap with + # either the reserved_cidrs or currently allocated CIDRs + # currently supports /24 subnets logger.debug("reserved CIDRs: {0}".format(cp_resvd_cidrs)) - candidate_prefixes = {'10': '10.0', '192.168': '192.168', '172': '172.0'} - cp_resvd_cidrs = cp_resvd_cidrs.split(",") - possible_prefixes = filter(lambda x: any(map(lambda y: not y.strip().startswith(x), cp_resvd_cidrs)), - candidate_prefixes.keys()) - logger.debug("Possible Prefixes that can be used: {0}".format(possible_prefixes)) - if not possible_prefixes: - return None - - prefix = possible_prefixes[0] - subnet_prefix = candidate_prefixes[prefix] - - # Get all subnets that start with 'our prefix' - subnets = client.list_subnets(fields=['cidr', 'id'])['subnets'] - subnet_cidrs = map(lambda x: x.get('cidr'), subnets) - - allocated_subnets = [] - for subnet in subnets: - if subnet['cidr'].startswith(prefix): - allocated_subnets.append(subnet['cidr']) - - allocated_subnets.sort() - logger.debug("Allocated Subnets: {0}".format(",".join(allocated_subnets))) - - if not allocated_subnets: - subnet_num = 0 - else: - last_subnet = allocated_subnets[-1] - subnet_num = int(last_subnet.split("/")[0].split(".")[2]) - subnet_num += 1 - if subnet_num == 255: - subnet_num = 0 - cidr = ".".join([subnet_prefix, str(subnet_num), "0/24"]) - while cidr in subnet_cidrs: - subnet_num += 1 - cidr = ".".join([subnet_prefix, str(subnet_num), "0/24"]) - else: - cidr = ".".join([subnet_prefix,str(subnet_num), "0/24"]) + blacklist_cidrs = map(lambda x: x.strip(), cp_resvd_cidrs.split(",")) + + current_subnets = client.list_subnets(fields=['cidr', 'id'])['subnets'] + current_subnets_cidrs = map(lambda x: unicode(x.get('cidr')), current_subnets) + + # Total CIDRs we don't care about are - reserved + currently allocated + + blacklist_cidrs += current_subnets_cidrs + blacklist_cidrs = map(lambda x: unicode(x), blacklist_cidrs) + blacklist_subnets = map(lambda x: ipaddress.IPv4Network(x), blacklist_cidrs) + + # start with a 10 subnet + found_subnet = None + first_octet = 10 + for i in range(256): + second_octet = i + for j in range(256): + third_octet = j + subnet_str = '{0}.{1}.{2}.0/24'.format(first_octet, second_octet, third_octet) + u_subnet_str = unicode(subnet_str) + u_subnet = ipaddress.IPv4Network(u_subnet_str) + # print u_subnet, blacklist_subnets + if not any(map(lambda x: u_subnet.overlaps(x), blacklist_subnets)): + found_subnet = u_subnet + break + if found_subnet: + break + + if not found_subnet: + first_octet = 172 + for i in range(16, 32): + second_octet = i + for j in range(256): + third_octet = j + subnet_str = '{0}.{1}.{2}.0/24'.format(first_octet, second_octet, third_octet) + u_subnet_str = unicode(subnet_str) + u_subnet = ipaddress.IPv4Network(u_subnet_str) + if not any(map(lambda x: u_subnet.overlaps(x), blacklist_subnets)): + found_subnet = u_subnet + break + if found_subnet: + break - logger.debug("Found {0} CIDR".format(cidr)) + if not found_subnet: + first_octet = 192 + second_octet = 168 + for j in range(256): + third_octet = j + subnet_str = '{0}.{1}.{2}.0/24'.format(first_octet, second_octet, third_octet) + u_subnet_str = unicode(subnet_str) + u_subnet = ipaddress.IPv4Network(u_subnet_str) + if not any(map(lambda x: u_subnet.overlaps(x), blacklist_subnets)): + found_subnet = u_subnet + break - if subnet_num == 255: + if not found_subnet: return None + cidr = str(found_subnet) + return cidr + def create_floating_ip(self, openstack_session, floating_ip_subnet_id, logger): """ diff --git a/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py b/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py index 7688f3d..ca751e3 100644 --- a/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py +++ b/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py @@ -77,10 +77,18 @@ def test_create_and_attach_subnet_to_net_success(self): test_neutron_network_service.neutron_client.Client = Mock(return_value=mock_client) mock_client.create_subnet = Mock(return_value={'subnet':'subnet success'}) - self.network_service._get_unused_cidr = Mock(return_value = '10.0.0.0/24') + mock_return_subnets = {'subnets':[{'cidr': '192.168.1.0/24', 'id':'test-id-1'}, + {'cidr': '192.168.1.0/24', 'id': 'test-id-2'}]} + + test_reserved_subnets = '10.0.0.0/16, 172.16.0.0/12, 192.168.0.0/24' + mock_client.list_subnets = Mock(return_value=mock_return_subnets) + + cp_resource_model = Mock() + cp_resource_model.reserved_networks = test_reserved_subnets + # self.network_service._get_unused_cidr = Mock(return_value = '10.0.0.0/24') result = self.network_service.create_and_attach_subnet_to_net(openstack_session=self.openstack_session, - cp_resource_model=Mock(), + cp_resource_model=cp_resource_model, net_id=test_net_id, logger=self.mock_logger) self.assertEqual(result, 'subnet success') From 304a3f1bb66c0e24656c884e34e96a08a493063b Mon Sep 17 00:00:00 2001 From: Abhijit Gadgil Date: Thu, 23 Feb 2017 23:16:29 +0530 Subject: [PATCH 04/14] #75 - Added more unit tests for algorithm verification --- .../test_neutron_network_service.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py b/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py index ca751e3..e30add3 100644 --- a/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py +++ b/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py @@ -70,6 +70,37 @@ def test_get_network_with_segmentation_id_no_network(self): logger=self.mock_logger) self.assertEqual(result, None) + def test_valid_cidr_returned(self): + + mock_client = Mock() + test_neutron_network_service.neutron_client.Client = Mock(return_value=mock_client) + mock_client.create_subnet = Mock(return_value={'subnet': 'subnet success'}) + + mock_return_subnets = {'subnets': [{'cidr': '10.0.0.0/24', 'id': 'test-id-1'}, + {'cidr': '10.0.1.0/24', 'id': 'test-id-2'}]} + + test_reserved_subnets = '172.0.0.0/8, 192.168.0.0/24' + mock_client.list_subnets = Mock(return_value=mock_return_subnets) + result = self.network_service._get_unused_cidr(client=mock_client, + cp_resvd_cidrs=test_reserved_subnets, + logger=self.mock_logger) + self.assertEqual(result, '10.0.2.0/24') + + def none_cidr_returned(self): + mock_client = Mock() + test_neutron_network_service.neutron_client.Client = Mock(return_value=mock_client) + mock_client.create_subnet = Mock(return_value={'subnet': 'subnet success'}) + + mock_return_subnets = {'subnets': [{'cidr': '10.0.0.0/24', 'id': 'test-id-1'}, + {'cidr': '10.0.1.0/24', 'id': 'test-id-2'}]} + + test_reserved_subnets = '10.0.0.0/8, 172.16.0.0/12 , 192.168.0.0/24' + mock_client.list_subnets = Mock(return_value=mock_return_subnets) + result = self.network_service._get_unused_cidr(client=mock_client, + cp_resvd_cidrs=test_reserved_subnets, + logger=self.mock_logger) + self.assertEqual(result, None) + def test_create_and_attach_subnet_to_net_success(self): test_net_id = 'test-net-id' @@ -80,7 +111,7 @@ def test_create_and_attach_subnet_to_net_success(self): mock_return_subnets = {'subnets':[{'cidr': '192.168.1.0/24', 'id':'test-id-1'}, {'cidr': '192.168.1.0/24', 'id': 'test-id-2'}]} - test_reserved_subnets = '10.0.0.0/16, 172.16.0.0/12, 192.168.0.0/24' + test_reserved_subnets = '10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/24' mock_client.list_subnets = Mock(return_value=mock_return_subnets) cp_resource_model = Mock() From 7f56455073405d5a3bba99a63c5812cf2f8cbd7b Mon Sep 17 00:00:00 2001 From: Abhijit Gadgil Date: Sun, 26 Feb 2017 14:51:43 +0530 Subject: [PATCH 05/14] #201 Fix Fixed if optional reserved_networks parameter was left empty --- .../neutron/neutron_network_service.py | 3 +- .../test_neutron_network_service.py | 34 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py b/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py index 73f035b..a81c482 100644 --- a/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py +++ b/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py @@ -160,7 +160,8 @@ def _get_unused_cidr(self, client, cp_resvd_cidrs, logger): # currently supports /24 subnets logger.debug("reserved CIDRs: {0}".format(cp_resvd_cidrs)) - blacklist_cidrs = map(lambda x: x.strip(), cp_resvd_cidrs.split(",")) + # Empty reserved_addresses generates a list with single empty string + blacklist_cidrs = filter(lambda x: len(x) > 0, map(lambda x: x.strip(), cp_resvd_cidrs.split(","))) current_subnets = client.list_subnets(fields=['cidr', 'id'])['subnets'] current_subnets_cidrs = map(lambda x: unicode(x.get('cidr')), current_subnets) diff --git a/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py b/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py index e30add3..c8b8e86 100644 --- a/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py +++ b/package/tests/test_cp/test_openstack/test_domain/test_services/test_neutron/test_neutron_network_service.py @@ -86,7 +86,7 @@ def test_valid_cidr_returned(self): logger=self.mock_logger) self.assertEqual(result, '10.0.2.0/24') - def none_cidr_returned(self): + def test_none_cidr_returned(self): mock_client = Mock() test_neutron_network_service.neutron_client.Client = Mock(return_value=mock_client) mock_client.create_subnet = Mock(return_value={'subnet': 'subnet success'}) @@ -94,13 +94,43 @@ def none_cidr_returned(self): mock_return_subnets = {'subnets': [{'cidr': '10.0.0.0/24', 'id': 'test-id-1'}, {'cidr': '10.0.1.0/24', 'id': 'test-id-2'}]} - test_reserved_subnets = '10.0.0.0/8, 172.16.0.0/12 , 192.168.0.0/24' + test_reserved_subnets = '10.0.0.0/8, 172.16.0.0/12 , 192.168.0.0/16' mock_client.list_subnets = Mock(return_value=mock_return_subnets) result = self.network_service._get_unused_cidr(client=mock_client, cp_resvd_cidrs=test_reserved_subnets, logger=self.mock_logger) self.assertEqual(result, None) + def test_empty_reserved_networks(self): + mock_client = Mock() + test_neutron_network_service.neutron_client.Client = Mock(return_value=mock_client) + mock_client.create_subnet = Mock(return_value={'subnet': 'subnet success'}) + + mock_return_subnets = {'subnets': [{'cidr': '10.0.0.0/24', 'id': 'test-id-1'}, + {'cidr': '10.0.1.0/24', 'id': 'test-id-2'}]} + + test_reserved_subnets = '' + mock_client.list_subnets = Mock(return_value=mock_return_subnets) + result = self.network_service._get_unused_cidr(client=mock_client, + cp_resvd_cidrs=test_reserved_subnets, + logger=self.mock_logger) + self.assertEqual(result, '10.0.2.0/24') + + def test_reserved_networks_one_empty_entry(self): + mock_client = Mock() + test_neutron_network_service.neutron_client.Client = Mock(return_value=mock_client) + mock_client.create_subnet = Mock(return_value={'subnet': 'subnet success'}) + + mock_return_subnets = {'subnets': [{'cidr': '10.0.0.0/24', 'id': 'test-id-1'}, + {'cidr': '10.0.1.0/24', 'id': 'test-id-2'}]} + + test_reserved_subnets = '172.16.0.0/12,,192.168.0.0/16' + mock_client.list_subnets = Mock(return_value=mock_return_subnets) + result = self.network_service._get_unused_cidr(client=mock_client, + cp_resvd_cidrs=test_reserved_subnets, + logger=self.mock_logger) + self.assertEqual(result, '10.0.2.0/24') + def test_create_and_attach_subnet_to_net_success(self): test_net_id = 'test-net-id' From cc3a72fb556149c57a540dc9fef6cece71839762 Mon Sep 17 00:00:00 2001 From: "evgeny.k" Date: Mon, 27 Feb 2017 11:52:00 +0200 Subject: [PATCH 06/14] installation-removal --- .../DataModel/datamodel.xml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/drivers/openstack_shellPackage/DataModel/datamodel.xml b/drivers/openstack_shellPackage/DataModel/datamodel.xml index fcfc9a2..513a9b6 100644 --- a/drivers/openstack_shellPackage/DataModel/datamodel.xml +++ b/drivers/openstack_shellPackage/DataModel/datamodel.xml @@ -215,29 +215,6 @@ - - - - - - - - - - - - - - - - Generic Driver - - - - - - - From c1818f659e672b949784a28d0d303ada5e77f93e Mon Sep 17 00:00:00 2001 From: Abhijit Gadgil Date: Mon, 27 Feb 2017 18:44:18 +0530 Subject: [PATCH 07/14] #159 reopen - fix Now added rules that work for Ubuntu and CentOS. Also work if Python 2 is not installed by default. Verified on Ubuntu 16.04 and CentOS 7.2 Cloud images. --- .../services/nova/nova_instance_service.py | 4 +- .../domain/services/nova/udev_rules.py | 53 +++++++++++-------- .../test_nova/test_nova_instance_service.py | 2 +- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py b/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py index 32b6c1e..57e982d 100644 --- a/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py +++ b/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py @@ -9,7 +9,7 @@ from cloudshell.cp.openstack.common.driver_helper import CloudshellDriverHelper from cloudshell.cp.openstack.models.exceptions import * -from cloudshell.cp.openstack.domain.services.nova.udev_rules import udev_rules_str +from cloudshell.cp.openstack.domain.services.nova.udev_rules import udev_rules_sh_str class NovaInstanceService(object): @@ -69,7 +69,7 @@ def create_instance(self, openstack_session, name, reservation, # user data if deploy_req_model.auto_udev: - server_create_args.update({'userdata':udev_rules_str}) + server_create_args.update({'userdata':udev_rules_sh_str}) instance = client.servers.create(**server_create_args) diff --git a/package/cloudshell/cp/openstack/domain/services/nova/udev_rules.py b/package/cloudshell/cp/openstack/domain/services/nova/udev_rules.py index d5200be..3c17758 100644 --- a/package/cloudshell/cp/openstack/domain/services/nova/udev_rules.py +++ b/package/cloudshell/cp/openstack/domain/services/nova/udev_rules.py @@ -1,35 +1,44 @@ -udev_rules_str = \ -'''#!/usr/bin/env python +udev_rules_sh_str = '''#!/bin/sh -import sys -import os +PYTHON= +if [ -e '/usr/bin/python' ]; then + PYTHON=/usr/bin/python +elif [ -e '/usr/bin/python3' ]; then + PYTHON=/usr/bin/python3 +fi -NETDEV_RULES_FILE = "/etc/udev/rules.d/70-attach-interface-qs-cloudshell.rules" +OS_TYPE=`$PYTHON -c 'import sys;print(sys.platform)'` -if sys.platform.lower().find("linux") < 0: - print "Non Linux OS. Doing Nothing..." - sys.exit(0) +LOGGER_TAG="ADD_QS_TAG" +if [ $OS_TYPE != 'linux2' -a $OS_TYPE != 'linux' ]; then + logger -i -s --tag ${LOGGER_TAG} "Platform $OS_TYPE is not supported for Udev rules." + exit 1 +fi -def generate_udev_rules_str(): - if not os.path.exists("/sbin/ifconfig"): - print "Ifconfig does not exist. Doing nothing..." - sys.exit(0) +QS_UDEV_FILE='/etc/udev/rules.d/70-a-qs-connectivity.rules' - if not os.path.exists("/sbin/dhclient"): - print "dhclient does not exist or not in standard path. Doing Noting." - sys.exit(0) +logger -i -s --tag ${LOGGER_TAG} "Creating UDEV Entries in file $QS_UDEV_FILE" +cat < $QS_UDEV_FILE +# Udev Rules for Quali Systems Apply Connectivity - etc_net_file_str = '\\n'.join(['KERNEL=="eth0", GOTO="no_eth0_config"', 'SUBSYSTEM=="net", ACTION=="add", KERNEL=="eth*", RUN+="/sbin/ifconfig %k up", RUN+="/sbin/dhclient -v %k -lf /var/run/dhclient--%.lease -pf /var/run/dhclient--%k.pid"', '', 'LABEL="no_eth0_config"', '']) +# KERNEL=="eth0", GOTO="no_eth0_config" - print etc_net_file_str - with open(NETDEV_RULES_FILE, "w") as f: - f.write(etc_net_file_str) +SUBSYSTEM=="net", ACTION=="add", KERNEL=="eth*", RUN+="/sbin/ifconfig %k up", RUN+="/sbin/dhclient -v %k -lf /var/run/dhclient-qs-%k.lease -pf /var/run/dhclient-qs-%k.pid"' +SUBSYSTEM=="net", ACTION=="add", KERNEL=="en*", RUN+="/sbin/ifconfig %k up", RUN+="/sbin/dhclient -v %k -lf /var/run/dhclient-qs-%k.lease -pf /var/run/dhclient-qs-%k.pid"' - result = os.system("udevadm control --reload") +# LABEL="no_eth0_config" - print result +TOEND + +# We are going to stop renaming of the devices - this messes things up. Everything will be eth0->ethN +logger -i -s --tag $LOGGER_TAG "Disabling device name change." +/bin/ln -s /dev/null /etc/udev/rules.d/80-net-name-slot.rules +/bin/ln -s /dev/null /etc/udev/rules.d/80-net-setup-link.rules + + +logger -i -s --tag $LOGGER_TAG "Calling udevadm control" +/sbin/udevadm control --reload -generate_udev_rules_str() ''' \ No newline at end of file diff --git a/package/tests/test_cp/test_openstack/test_domain/test_services/test_nova/test_nova_instance_service.py b/package/tests/test_cp/test_openstack/test_domain/test_services/test_nova/test_nova_instance_service.py index 6fd1227..600d7ff 100644 --- a/package/tests/test_cp/test_openstack/test_domain/test_services/test_nova/test_nova_instance_service.py +++ b/package/tests/test_cp/test_openstack/test_domain/test_services/test_nova/test_nova_instance_service.py @@ -45,7 +45,7 @@ def test_instance_create_success(self): mock_deploy_req_model.affinity_group_uuid = '' mock_deploy_req_model.cp_avail_zone = 'test-avail-zone' - test_nova_instance_service.udev_rules_str = 'test_userdata' + test_nova_instance_service.udev_rules_sh_str = 'test_userdata' mock_cp_resource_model = Mock() mock_cp_resource_model.qs_mgmt_os_net_uuid = '1234' From ef730289582d9694d1778d80e9fc262f14ab7c3f Mon Sep 17 00:00:00 2001 From: Alex Azarh Date: Mon, 27 Feb 2017 18:19:32 +0200 Subject: [PATCH 08/14] updated version to 1.0.0 --- drivers/version.txt | 2 +- package/version.txt | 2 +- version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/version.txt b/drivers/version.txt index 927734f..afaf360 100644 --- a/drivers/version.txt +++ b/drivers/version.txt @@ -1 +1 @@ -0.0.17 \ No newline at end of file +1.0.0 \ No newline at end of file diff --git a/package/version.txt b/package/version.txt index 927734f..afaf360 100644 --- a/package/version.txt +++ b/package/version.txt @@ -1 +1 @@ -0.0.17 \ No newline at end of file +1.0.0 \ No newline at end of file diff --git a/version.txt b/version.txt index 927734f..afaf360 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.17 \ No newline at end of file +1.0.0 \ No newline at end of file From 3de0a1c20396af66b0afdf9f1af707dd2f77a27e Mon Sep 17 00:00:00 2001 From: Alex Azarh Date: Mon, 27 Feb 2017 18:20:24 +0200 Subject: [PATCH 09/14] updated version --- .../openstack_nova_image_instance/drivermetadata.xml | 2 +- drivers/openstack_shell/src/drivermetadata.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/deployment_drivers/openstack_nova_image_instance/drivermetadata.xml b/drivers/deployment_drivers/openstack_nova_image_instance/drivermetadata.xml index b77e9e3..6ebf2c3 100644 --- a/drivers/deployment_drivers/openstack_nova_image_instance/drivermetadata.xml +++ b/drivers/deployment_drivers/openstack_nova_image_instance/drivermetadata.xml @@ -1,4 +1,4 @@ - + diff --git a/drivers/openstack_shell/src/drivermetadata.xml b/drivers/openstack_shell/src/drivermetadata.xml index b022856..5108578 100644 --- a/drivers/openstack_shell/src/drivermetadata.xml +++ b/drivers/openstack_shell/src/drivermetadata.xml @@ -1,4 +1,4 @@ - + From 17977e2f9043d6c7cd6afd2dee9f0388675afe3d Mon Sep 17 00:00:00 2001 From: Alex Azarh Date: Mon, 27 Feb 2017 18:22:08 +0200 Subject: [PATCH 10/14] fixing requirements --- .../openstack_nova_image_instance/requirements.txt | 2 +- drivers/openstack_shell/src/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/deployment_drivers/openstack_nova_image_instance/requirements.txt b/drivers/deployment_drivers/openstack_nova_image_instance/requirements.txt index 5f920cb..be0cf81 100644 --- a/drivers/deployment_drivers/openstack_nova_image_instance/requirements.txt +++ b/drivers/deployment_drivers/openstack_nova_image_instance/requirements.txt @@ -1,3 +1,3 @@ cloudshell-shell-core>=2.2.0,<2.3.0 -cloudshell-cp-openstack<0.1.0 +cloudshell-cp-openstack>=1.0.0,<1.1.0 jsonpickle==0.9.3 diff --git a/drivers/openstack_shell/src/requirements.txt b/drivers/openstack_shell/src/requirements.txt index 5f920cb..be0cf81 100644 --- a/drivers/openstack_shell/src/requirements.txt +++ b/drivers/openstack_shell/src/requirements.txt @@ -1,3 +1,3 @@ cloudshell-shell-core>=2.2.0,<2.3.0 -cloudshell-cp-openstack<0.1.0 +cloudshell-cp-openstack>=1.0.0,<1.1.0 jsonpickle==0.9.3 From e70d732f750d0bd8aa3989c1adbda5c49871ef40 Mon Sep 17 00:00:00 2001 From: Alex Azarh Date: Tue, 28 Feb 2017 12:28:58 +0200 Subject: [PATCH 11/14] adding logs --- .../operations/connectivity_operation.py | 2 +- .../connectivity/vlan_connectivity_service.py | 40 +++++++++++++------ .../neutron/neutron_network_service.py | 22 ++++++---- .../services/nova/nova_instance_service.py | 6 +-- .../connectivity_action_resource_info.py | 6 ++- .../cp/openstack/openstack_shell.py | 1 + 6 files changed, 52 insertions(+), 25 deletions(-) diff --git a/package/cloudshell/cp/openstack/command/operations/connectivity_operation.py b/package/cloudshell/cp/openstack/command/operations/connectivity_operation.py index fa6f161..9952fef 100644 --- a/package/cloudshell/cp/openstack/command/operations/connectivity_operation.py +++ b/package/cloudshell/cp/openstack/command/operations/connectivity_operation.py @@ -17,7 +17,7 @@ def apply_connectivity(self, openstack_session, cp_resource_model, conn_request, :param keystoneauth1.session.Session openstack_session: :param OpenStackResourceModel cp_resource_model: :param str conn_request: Connectivty Request JSON - :param LoggingSessionContext logger: + :param logging.Logger logger: :return DriverResponseRoot: """ diff --git a/package/cloudshell/cp/openstack/domain/services/connectivity/vlan_connectivity_service.py b/package/cloudshell/cp/openstack/domain/services/connectivity/vlan_connectivity_service.py index 4a9cc9f..d6a8cf3 100644 --- a/package/cloudshell/cp/openstack/domain/services/connectivity/vlan_connectivity_service.py +++ b/package/cloudshell/cp/openstack/domain/services/connectivity/vlan_connectivity_service.py @@ -32,7 +32,7 @@ def perform_apply_connectivity(self, openstack_session, cp_resource_model, conne :param keystoneauth1.session.Session openstack_session: :param OpenStackResourceModel cp_resource_model: :param str connection_request: - :param LoggingSessionContext logger: + :param logging.Logger logger: :return: """ @@ -48,8 +48,9 @@ def perform_apply_connectivity(self, openstack_session, cp_resource_model, conne set_vlan_actions_dict = {} remove_vlan_actions_dict = {} + logger.info("Processing {0} actions and creating mapping between vlan ids to actions".format(len(actions))) + # Add more description - # TODO : implement remove actions dict for action in actions: curr_dict = self._get_curr_actions_dict(action_type=action.type, @@ -58,9 +59,11 @@ def perform_apply_connectivity(self, openstack_session, cp_resource_model, conne if curr_dict is None: raise ValueError("Unknown action: Action not one of 'setVlan' or 'removeVlan'.") - actionid = action.actionId + action_id = action.actionId deployed_app_res_name = action.actionTarget.fullName - action_resource_info = self.get_action_resource_info(deployed_app_res_name, actionid, action) + logger.debug("Processing action id {0} for target {1}".format(action_id, deployed_app_res_name)) + action_resource_info = self.get_action_resource_info(deployed_app_res_name, action_id, action) + logger.debug("Action resource info: {}".format(action_resource_info)) action_vlanid = action.connectionParams.vlanId if action_vlanid in curr_dict.keys(): @@ -106,17 +109,20 @@ def set_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions, l :param keystoneauth1.session.Session openstack_session: :param OpenStackResourceModel cp_resource_model: :param dict vlan_actions: - :param LoggingSessionContext logger: - :return ConnectivityActionResult List : + :param logging.Logger logger: + :rtype: ConnectivityActionResult[] """ # For each VLAN ID (create VLAN network) results = [] + logger.debug("We have {0} 'set vlan' actions to process".format(len(vlan_actions))) + for vlan_id, values in vlan_actions.iteritems(): net = None net_err_msg = '' try: + logger.debug("creating or getting network with segmentation id {}".format(vlan_id)) net = self.network_service.create_or_get_network_with_segmentation_id( openstack_session=openstack_session, cp_resource_model=cp_resource_model, @@ -125,6 +131,7 @@ def set_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions, l except Exception as e: logger.error(traceback.format_exc()) net_err_msg = self._format_err_msg_for_exception(e) + if not net: fail_results = self.set_fail_results(values=values, action_type='setVlan', @@ -138,6 +145,7 @@ def set_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions, l net_id = net['id'] subnet = net['subnets'] if not subnet: + logger.info("Its a new network without a subnet so will allocate new CIDR") with self.subnet_lock: subnet = self.network_service.create_and_attach_subnet_to_net( openstack_session=openstack_session, @@ -150,6 +158,7 @@ def set_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions, l except Exception as e: logger.error(traceback.format_exc()) subnet_err_msg = self._format_err_msg_for_exception(e) + if not subnet: fail_results = self.set_fail_results(values=values, action_type='setVlan', @@ -158,6 +167,7 @@ def set_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions, l results += fail_results else: attach_results = [] + logger.info("Attaching nics to instances") for val in values: action_result = self.attach_nic_to_instance_action_result(openstack_session=openstack_session, action_resource_info=val, @@ -173,24 +183,28 @@ def remove_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions :param keystoneauth1.session.Session openstack_session: :param OpenStckResourceModel cp_resource_model: :param dict vlan_actions: - :param LoggingSessionContext logger: + :param logging.Logger logger: :return: """ + logger.debug("We have {0} 'remove vlan' actions to process".format(len(vlan_actions))) + results = [] - for k, values in vlan_actions.iteritems(): + for vlan_id, values in vlan_actions.iteritems(): + logger.debug("Finding network with segmentation id {}".format(vlan_id)) net = self.network_service.get_network_with_segmentation_id(openstack_session=openstack_session, - segmentation_id=int(k), logger=logger) + segmentation_id=int(vlan_id), logger=logger) if not net: fail_results = self.set_fail_results(values=values, action_type='removeVlan', - failure_text="Failed to get Network with VLAN ID {0}".format(k)) + failure_text="Failed to get Network with VLAN ID {0}".format(vlan_id)) results += fail_results else: net_id = net['id'] remove_results = [] + logger.debug("Detaching nics from network {}".format(net_id)) for val in values: action_result = self.detach_nic_from_instance_action_result(openstack_session=openstack_session, action_resource_info=val, @@ -250,7 +264,7 @@ def get_action_resource_info(self, deployed_app_resource_name, actionid, action) :param str deployed_app_resource_name: :param str actionid: :param action: action obtained from JSON - :return ConnectivityActionResourceInfo: + :rtype: ConnectivityActionResourceInfo """ @@ -288,7 +302,7 @@ def attach_nic_to_instance_action_result(self, openstack_session, action_resourc :param keystoneauth1.session.Session openstack_session: :param ConnectivityActionResourceInfo action_resource_info: :param str net_id: - :param LoggingSessionContext logger: + :param logging.Logger logger: :return ConnectivityActionResultModel: """ action_result = ConnectivityActionResultModel() @@ -297,11 +311,13 @@ def attach_nic_to_instance_action_result(self, openstack_session, action_resourc result_err_msg = '' try: instance_id = action_resource_info.vm_uuid + logger.debug("Attaching instance {0} to net {1}".format(instance_id, net_id)) result = self.instance_service.attach_nic_to_net(openstack_session=openstack_session, instance_id=instance_id, net_id=net_id, logger=logger) except Exception as e: result_err_msg = self._format_err_msg_for_exception(e) logger.error(result_err_msg) + if not result: action_result.success = "False" action_result.actionId = action_resource_info.actionid diff --git a/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py b/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py index a81c482..6635aeb 100644 --- a/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py +++ b/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py @@ -21,7 +21,7 @@ def create_or_get_network_with_segmentation_id(self, openstack_session, cp_resou :param keystoneauth1.session.Session openstack_session: :param OpenStackResourceModel cp_resource_model: :param int segmentation_id: - :param LoggingSessionContext logger: + :param logging.Logger logger: :return dict : """ @@ -40,16 +40,21 @@ def create_or_get_network_with_segmentation_id(self, openstack_session, cp_resou create_nw_json.update({'provider:physical_network': interface_name}) try: - new_net = client.create_network({'network': create_nw_json}) + request = {'network': create_nw_json} + logger.debug("Calling neutron client create_network with request: {}".format(request)) + new_net = client.create_network(request) new_net = new_net['network'] except NetCreateConflict as e: logger.error(traceback.format_exc()) networks_res = client.list_networks(**{'provider:segmentation_id': segmentation_id}) networks = networks_res['networks'] if not networks: + logger.error("Network with segmentation id {0} not found and couldnt be created".format(segmentation_id)) raise new_net = networks_res['networks'][0] + logger.debug("Got network: {}".format(new_net)) + return new_net def get_network_with_segmentation_id(self, openstack_session, segmentation_id, logger): @@ -76,6 +81,7 @@ def create_and_attach_subnet_to_net(self, openstack_session, cp_resource_model, :param keystoneauth1.session.Session openstack_session: :param OpenStackResourceModel cp_resource_model: :param str net_id: UUID string + :param logging.Logger logger: :return dict: """ @@ -93,8 +99,11 @@ def create_and_attach_subnet_to_net(self, openstack_session, cp_resource_model, 'name': subnet_name, 'gateway_ip': None} - new_subnet = client.create_subnet({'subnet':create_subnet_json}) + request = {'subnet': create_subnet_json} + logger.debug("Calling neutron client create_subnet with request: {}".format(request)) + new_subnet = client.create_subnet(request) new_subnet = new_subnet['subnet'] + logger.debug("Created new subnet: {}".format(new_subnet)) return new_subnet @@ -103,16 +112,13 @@ def remove_subnet_and_net(self, openstack_session, network, logger): :param keystoneauth1.session.Session openstack_session: :param dict network: - :param LoggingSessionContext logger: + :param logging.Logger logger: :return: """ - # FIXME: What happens if multiple threads call this? client = neutron_client.Client(session=openstack_session) try: - # FIXME: This whole block should be synchronized. - # Get a list of all ports for this network. If there's any port with device owner other than DHCP, # You won't be able to delete the network or subnet. Retry it a few times (sometimes seen that when # this call happens, some 'ports' are still there. @@ -170,6 +176,7 @@ def _get_unused_cidr(self, client, cp_resvd_cidrs, logger): blacklist_cidrs += current_subnets_cidrs blacklist_cidrs = map(lambda x: unicode(x), blacklist_cidrs) + logger.debug("blacklist CIDRs: {0}".format(blacklist_cidrs)) blacklist_subnets = map(lambda x: ipaddress.IPv4Network(x), blacklist_cidrs) # start with a 10 subnet @@ -220,6 +227,7 @@ def _get_unused_cidr(self, client, cp_resvd_cidrs, logger): return None cidr = str(found_subnet) + logger.debug("Resolved CIDR: {}".format(cidr)) return cidr diff --git a/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py b/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py index 32b6c1e..c31cf7a 100644 --- a/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py +++ b/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py @@ -229,14 +229,14 @@ def attach_nic_to_net(self, openstack_session, instance_id, net_id, logger): :param openstack_session: :param instance_id: :param net_id: - :param logger: + :param logging.Logger logger: :return: """ instance = self.get_instance_from_instance_id(openstack_session=openstack_session, instance_id=instance_id, logger=logger) - if instance is None : + if instance is None: return None try: @@ -251,8 +251,6 @@ def attach_nic_to_net(self, openstack_session, instance_id, net_id, logger): logger.error("Exception: {0} during interface attach.".format(e)) raise - return None - def detach_nic_from_instance(self, openstack_session, instance_id, port_id, logger): """ diff --git a/package/cloudshell/cp/openstack/models/connectivity_action_resource_info.py b/package/cloudshell/cp/openstack/models/connectivity_action_resource_info.py index 1c51ee8..29a2eef 100644 --- a/package/cloudshell/cp/openstack/models/connectivity_action_resource_info.py +++ b/package/cloudshell/cp/openstack/models/connectivity_action_resource_info.py @@ -1,4 +1,3 @@ - class ConnectivityActionResourceInfo: def __init__(self, deployed_app_resource_name, actionid, vm_uuid, interface_ip, interface_port_id, interface_mac): self.deployed_app_resource_name = deployed_app_resource_name @@ -7,3 +6,8 @@ def __init__(self, deployed_app_resource_name, actionid, vm_uuid, interface_ip, self.iface_ip = interface_ip self.interface_port_id = interface_port_id self.interface_mac = interface_mac + + def __str__(self): + return "deployed_app_resource_name: {0}, vm_uuid: {1}, actionid: {2}, iface_ip: {3}, interface_port_id: {4}, " \ + "interface_mac: {5}".format(self.deployed_app_resource_name, self.vm_uuid, self.actionid, self.iface_ip, + self.interface_port_id, self.interface_mac) diff --git a/package/cloudshell/cp/openstack/openstack_shell.py b/package/cloudshell/cp/openstack/openstack_shell.py index 7750c8f..fb9fecc 100644 --- a/package/cloudshell/cp/openstack/openstack_shell.py +++ b/package/cloudshell/cp/openstack/openstack_shell.py @@ -261,6 +261,7 @@ def apply_connectivity(self, command_context, connectivity_request): cp_resource_model=cp_resource_model, conn_request=connectivity_request, logger=logger) + logger.info(connectivity_result) return self.command_result_parser.set_command_result(connectivity_result) From 8c08038143f43163e432f69b24b1983c8a8cb176 Mon Sep 17 00:00:00 2001 From: Abhijit Gadgil Date: Tue, 28 Feb 2017 16:08:04 +0530 Subject: [PATCH 12/14] #159 Fix Fixed udev rules for Ubuntu 14.04 --- .../cloudshell/cp/openstack/domain/services/nova/udev_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/cloudshell/cp/openstack/domain/services/nova/udev_rules.py b/package/cloudshell/cp/openstack/domain/services/nova/udev_rules.py index 3c17758..04a8d1c 100644 --- a/package/cloudshell/cp/openstack/domain/services/nova/udev_rules.py +++ b/package/cloudshell/cp/openstack/domain/services/nova/udev_rules.py @@ -39,6 +39,6 @@ logger -i -s --tag $LOGGER_TAG "Calling udevadm control" -/sbin/udevadm control --reload +/sbin/udevadm control --reload && /sbin/udevadm trigger --subsystem-match=net ''' \ No newline at end of file From 595c7201bbc575c12e3661a8f518457206c11bb0 Mon Sep 17 00:00:00 2001 From: Alex Azarh Date: Tue, 28 Feb 2017 14:58:57 +0200 Subject: [PATCH 13/14] added more logs --- .../openstack_nova_image_instance/driver.py | 10 +++++----- .../connectivity/vlan_connectivity_service.py | 18 ++++++++++-------- .../neutron/neutron_network_service.py | 18 ++++++++++-------- .../services/nova/nova_instance_service.py | 5 +++-- .../cloudshell/cp/openstack/openstack_shell.py | 6 +++--- 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/drivers/deployment_drivers/openstack_nova_image_instance/driver.py b/drivers/deployment_drivers/openstack_nova_image_instance/driver.py index 8e2a659..553485f 100644 --- a/drivers/deployment_drivers/openstack_nova_image_instance/driver.py +++ b/drivers/deployment_drivers/openstack_nova_image_instance/driver.py @@ -35,9 +35,9 @@ def Deploy(self, context, Name=None): with LoggingSessionContext(context) as logger: with ErrorHandlingContext(logger): with CloudShellSessionContext(context) as session: - logger.debug("Deploy Called for Reservation: {0}".format(context.reservation.reservation_id)) + logger.info("Deploy Called for Reservation: {0}".format(context.reservation.reservation_id)) # Get CS Session we are going to make an API call using this session - logger.debug("creating session: {0}, {1}, {2}".format(context.connectivity.server_address, + logger.info("creating session: {0}, {1}, {2}".format(context.connectivity.server_address, context.connectivity.admin_auth_token, context.reservation.domain)) @@ -48,7 +48,7 @@ def Deploy(self, context, Name=None): app_name = context_json_decoded['name'] cloud_provider_name = context_json_decoded["deploymentService"].get("cloudProviderName") - logger.debug("cloud_provider_name from context = {0}".format(cloud_provider_name)) + logger.info("cloud_provider_name from context = {0}".format(cloud_provider_name)) if cloud_provider_name: deploy_service_res_model.cloud_provider = str(cloud_provider_name) @@ -56,9 +56,9 @@ def Deploy(self, context, Name=None): deploy_req = DeployDataHolder({self.APP_NAME: app_name, self.IMAGE_PARAM: deploy_service_res_model}) - logger.debug("Calling the Shell Driver's Deploy method for app: {0}".format(app_name)) + logger.info("Calling the Shell Driver's Deploy method for app: {0}".format(app_name)) - logger.debug("cloud_provider = {0}".format(deploy_service_res_model.cloud_provider)) + logger.info("cloud_provider = {0}".format(deploy_service_res_model.cloud_provider)) # Calls command on the OpenStack cloud provider result = session.ExecuteCommand(context.reservation.reservation_id, deploy_service_res_model.cloud_provider, diff --git a/package/cloudshell/cp/openstack/domain/services/connectivity/vlan_connectivity_service.py b/package/cloudshell/cp/openstack/domain/services/connectivity/vlan_connectivity_service.py index d6a8cf3..65a5294 100644 --- a/package/cloudshell/cp/openstack/domain/services/connectivity/vlan_connectivity_service.py +++ b/package/cloudshell/cp/openstack/domain/services/connectivity/vlan_connectivity_service.py @@ -61,9 +61,9 @@ def perform_apply_connectivity(self, openstack_session, cp_resource_model, conne action_id = action.actionId deployed_app_res_name = action.actionTarget.fullName - logger.debug("Processing action id {0} for target {1}".format(action_id, deployed_app_res_name)) + logger.info("Processing action id {0} for target {1}".format(action_id, deployed_app_res_name)) action_resource_info = self.get_action_resource_info(deployed_app_res_name, action_id, action) - logger.debug("Action resource info: {}".format(action_resource_info)) + logger.info("Action resource info: {}".format(action_resource_info)) action_vlanid = action.connectionParams.vlanId if action_vlanid in curr_dict.keys(): @@ -92,6 +92,8 @@ def perform_apply_connectivity(self, openstack_session, cp_resource_model, conne driver_response_root = DriverResponseRoot() driver_response_root.driverResponse = driver_response + logger.info(jsonpickle.dumps(driver_response, unpicklable=False)) + return driver_response_root def _format_err_msg_for_exception(self, e): @@ -116,13 +118,13 @@ def set_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions, l # For each VLAN ID (create VLAN network) results = [] - logger.debug("We have {0} 'set vlan' actions to process".format(len(vlan_actions))) + logger.info("We have {0} 'set vlan' actions to process".format(len(vlan_actions))) for vlan_id, values in vlan_actions.iteritems(): net = None net_err_msg = '' try: - logger.debug("creating or getting network with segmentation id {}".format(vlan_id)) + logger.info("creating or getting network with segmentation id {}".format(vlan_id)) net = self.network_service.create_or_get_network_with_segmentation_id( openstack_session=openstack_session, cp_resource_model=cp_resource_model, @@ -187,12 +189,12 @@ def remove_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions :return: """ - logger.debug("We have {0} 'remove vlan' actions to process".format(len(vlan_actions))) + logger.info("We have {0} 'remove vlan' actions to process".format(len(vlan_actions))) results = [] for vlan_id, values in vlan_actions.iteritems(): - logger.debug("Finding network with segmentation id {}".format(vlan_id)) + logger.info("Finding network with segmentation id {}".format(vlan_id)) net = self.network_service.get_network_with_segmentation_id(openstack_session=openstack_session, segmentation_id=int(vlan_id), logger=logger) if not net: @@ -204,7 +206,7 @@ def remove_vlan_actions(self, openstack_session, cp_resource_model, vlan_actions net_id = net['id'] remove_results = [] - logger.debug("Detaching nics from network {}".format(net_id)) + logger.info("Detaching nics from network {}".format(net_id)) for val in values: action_result = self.detach_nic_from_instance_action_result(openstack_session=openstack_session, action_resource_info=val, @@ -311,7 +313,7 @@ def attach_nic_to_instance_action_result(self, openstack_session, action_resourc result_err_msg = '' try: instance_id = action_resource_info.vm_uuid - logger.debug("Attaching instance {0} to net {1}".format(instance_id, net_id)) + logger.info("Attaching instance {0} to net {1}".format(instance_id, net_id)) result = self.instance_service.attach_nic_to_net(openstack_session=openstack_session, instance_id=instance_id, net_id=net_id, logger=logger) except Exception as e: diff --git a/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py b/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py index 6635aeb..ddbddce 100644 --- a/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py +++ b/package/cloudshell/cp/openstack/domain/services/neutron/neutron_network_service.py @@ -41,7 +41,7 @@ def create_or_get_network_with_segmentation_id(self, openstack_session, cp_resou try: request = {'network': create_nw_json} - logger.debug("Calling neutron client create_network with request: {}".format(request)) + logger.info("Calling neutron client create_network with request: {}".format(request)) new_net = client.create_network(request) new_net = new_net['network'] except NetCreateConflict as e: @@ -53,7 +53,7 @@ def create_or_get_network_with_segmentation_id(self, openstack_session, cp_resou raise new_net = networks_res['networks'][0] - logger.debug("Got network: {}".format(new_net)) + logger.info("Got network: {}".format(new_net)) return new_net @@ -100,10 +100,10 @@ def create_and_attach_subnet_to_net(self, openstack_session, cp_resource_model, 'gateway_ip': None} request = {'subnet': create_subnet_json} - logger.debug("Calling neutron client create_subnet with request: {}".format(request)) + logger.info("Calling neutron client create_subnet with request: {}".format(request)) new_subnet = client.create_subnet(request) new_subnet = new_subnet['subnet'] - logger.debug("Created new subnet: {}".format(new_subnet)) + logger.info("Created new subnet: {}".format(new_subnet)) return new_subnet @@ -134,11 +134,13 @@ def remove_subnet_and_net(self, openstack_session, network, logger): else: break - logger.debug("Found {0} ports".format(network_ports)) + logger.info("Found {0} ports: {1}".format(len(network_ports), network_ports)) if len(network_ports) <= 1: for subnet in network['subnets']: + logger.info("Deleting subnet {}".format(subnet)) client.delete_subnet(subnet) + logger.info("Deleting network {}".format(network)) client.delete_network(network['id']) else: @@ -164,7 +166,7 @@ def _get_unused_cidr(self, client, cp_resvd_cidrs, logger): # We basically start with a 10.0. network to find a subnet that does not overlap with # either the reserved_cidrs or currently allocated CIDRs # currently supports /24 subnets - logger.debug("reserved CIDRs: {0}".format(cp_resvd_cidrs)) + logger.info("reserved CIDRs: {0}".format(cp_resvd_cidrs)) # Empty reserved_addresses generates a list with single empty string blacklist_cidrs = filter(lambda x: len(x) > 0, map(lambda x: x.strip(), cp_resvd_cidrs.split(","))) @@ -176,7 +178,7 @@ def _get_unused_cidr(self, client, cp_resvd_cidrs, logger): blacklist_cidrs += current_subnets_cidrs blacklist_cidrs = map(lambda x: unicode(x), blacklist_cidrs) - logger.debug("blacklist CIDRs: {0}".format(blacklist_cidrs)) + logger.info("blacklist CIDRs: {0}".format(blacklist_cidrs)) blacklist_subnets = map(lambda x: ipaddress.IPv4Network(x), blacklist_cidrs) # start with a 10 subnet @@ -227,7 +229,7 @@ def _get_unused_cidr(self, client, cp_resvd_cidrs, logger): return None cidr = str(found_subnet) - logger.debug("Resolved CIDR: {}".format(cidr)) + logger.info("Resolved CIDR: {}".format(cidr)) return cidr diff --git a/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py b/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py index c31cf7a..ad69734 100644 --- a/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py +++ b/package/cloudshell/cp/openstack/domain/services/nova/nova_instance_service.py @@ -207,7 +207,8 @@ def get_instance_from_instance_id(self, openstack_session, instance_id, logger, :param str instance_id: :param LoggingSessionContext logger: :param novaclient.Client client: client (optional) - :rtype novaclient.Client.servers.Server instance: + :return: instance + :rtype: novaclient.Client.servers.Server """ if client is None: client = novaclient.Client(self.API_VERSION, session=openstack_session) @@ -265,7 +266,7 @@ def detach_nic_from_instance(self, openstack_session, instance_id, port_id, logg instance = self.get_instance_from_instance_id(openstack_session=openstack_session, instance_id=instance_id, logger=logger) - logger.info("Returned instance {0}".format(instance)) + logger.info("Returned instance {0}".format(instance.name)) if instance is None: return False diff --git a/package/cloudshell/cp/openstack/openstack_shell.py b/package/cloudshell/cp/openstack/openstack_shell.py index fb9fecc..61cbe50 100644 --- a/package/cloudshell/cp/openstack/openstack_shell.py +++ b/package/cloudshell/cp/openstack/openstack_shell.py @@ -1,3 +1,5 @@ +import jsonpickle + from cloudshell.cp.openstack.common.driver_helper import CloudshellDriverHelper # Model @@ -89,7 +91,7 @@ def power_on(self, command_context): deployed_app_resource = self.model_parser.deployed_app_resource_from_context_remote(context_remote) deployed_app_fullname = context_remote.fullname - logger.info(deployed_app_resource) + logger.debug(jsonpickle.dumps(deployed_app_resource, unpicklable=False)) os_session = self.os_session_provider.get_openstack_session(cs_session, resource_model, logger) self.power_operation.power_on(openstack_session=os_session, @@ -261,8 +263,6 @@ def apply_connectivity(self, command_context, connectivity_request): cp_resource_model=cp_resource_model, conn_request=connectivity_request, logger=logger) - logger.info(connectivity_result) - return self.command_result_parser.set_command_result(connectivity_result) # Connectivity Operations End From 2a5764dfca53304add4e61e479781ff32f9410a2 Mon Sep 17 00:00:00 2001 From: Alex Azarh Date: Tue, 28 Feb 2017 15:33:56 +0200 Subject: [PATCH 14/14] fix tests --- package/tests/test_cp/test_openstack/test_openstack_shell.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package/tests/test_cp/test_openstack/test_openstack_shell.py b/package/tests/test_cp/test_openstack/test_openstack_shell.py index a48a6bb..fc35b30 100644 --- a/package/tests/test_cp/test_openstack/test_openstack_shell.py +++ b/package/tests/test_cp/test_openstack/test_openstack_shell.py @@ -29,7 +29,8 @@ def setUp(self): self.os_shell_api.model_parser.get_resource_model_from_context = Mock(return_value=OpenStackResourceModel) self.os_shell_api.os_session_provider.get_openstack_session = Mock(return_value=Mock()) - def test_power_on(self): + @patch('cloudshell.cp.openstack.openstack_shell.jsonpickle.dumps') + def test_power_on(self, json_dumps): """ :return: