update 2.4.X with: support for 2nd gen shells and apps, ami lookup, s…

…upport for both subnet name or cidr block

.gitignore

@@ -99,3 +99,4 @@ cloudformation/deploy_execution_server_by_selected_version
 cloudformation/deploy_execution_server_by_selected_version/*
 */.aws/*
 .aws/*
+/local/

cloudformation/AMI_Lookup.json

@@ -61,18 +61,20 @@
               [
                 "// Map display OS names to AMI name patterns",
                 "var osNameToPattern = {",
-                "  'Windows Server 2012 R2 English 64-bit': 'Windows_Server-2012-R2_RTM-English-64Bit-Base-*',",
-                "  'CentOS 7.4 Latest': 'centos-7.4-plain-x86_64*'",
+                "  'Windows Server 2012 R2 English 64-bit': 'Windows_Server-2012*English*Base*',",
+                "   'CentOS 7.4 Latest': 'CentOS Linux 7 x86_64 HVM EBS ENA*'",
                 "};",
                 "// Map display OS names to owner",
-                "var osNameToOwner = {",
+                "var osNameToFilterData = {",
                 "  'Windows Server 2012 R2 English 64-bit': {",
                 "                                            'owner': 'amazon', ",
-                "                                            'ownerId': '801119661308' // amazon web services",
+                "                                            'filterValue': '801119661308', // amazon web services",
+                "                                            'filterName': 'owner-id' // amazon web services",
                 "                                            },",
                 "  'CentOS 7.4 Latest': {",
                 "                        'owner': 'aws-marketplace', ",
-                "                        'ownerId': '679593333241' // centos.org",
+                "                        'filterName': 'product-code',",
+                "                        'filterValue': 'aw0evgkw8e5c1q413zgy5pjce',",
                 "                        }",
                 "};",
                 "var aws = require('aws-sdk');",
@@ -86,14 +88,15 @@
                 "  var responseStatus = 'FAILED';",
                 "  var responseData = {};",
                 "  var osBaseName = osNameToPattern[event.ResourceProperties.OSName];",
-                "  var osOwner = osNameToOwner[event.ResourceProperties.OSName]['owner'];",
-                "  var osOwnerId = osNameToOwner[event.ResourceProperties.OSName]['ownerId'];",
-                "  console.log('OS: ' + event.ResourceProperties.OSName + ' -> ' + osBaseName + ' owned by: ' + osOwner + ' owner-id: ' + osOwnerId);",
+                "  var osOwner = osNameToFilterData[event.ResourceProperties.OSName]['owner'];",
+                "  var filterValue = osNameToFilterData[event.ResourceProperties.OSName]['filterValue'];",
+                "  var filterName = osNameToFilterData[event.ResourceProperties.OSName]['filterName'];",
+                "  console.log('OS: ' + event.ResourceProperties.OSName + ' -> ' + osBaseName + ' owned by: ' + osOwner + ' ' + filterName+ ' ' + filterValue);",
                 "  var ec2 = new aws.EC2({region: event.ResourceProperties.Region});",
                 "  var describeImagesParams = {",
                 "    Filters: [",
-                "    { Name: 'name', Values: [osBaseName]},",
-                "    { Name: 'owner-id', Values: [osOwnerId]}],",
+                "    { Name: filterName, Values: [filterValue]},",
+                "    { Name: 'name', Values: [osBaseName]}],",
                 "    Owners: [osOwner]",
                 "  };",
                 "  console.log('Calling describeImages...');",

drivers/aws_shell/src/drivermetadata.xml

@@ -1,4 +1,4 @@
-<Driver Description="This driver orchestrate all the command that will be executed on AWS" MainClass="driver.AWSShellDriver" Name="AWS Shell Driver" Version="2.4.1">
+<Driver Description="This driver orchestrate all the command that will be executed on AWS" MainClass="driver.AWSShellDriver" Name="AWS Shell Driver" Version="2.4.2">
     <Layout>
         <Category Name="Deployment">
             <Command Description="" DisplayName="Deploy From AMI" EnableCancellation="true" Name="deploy_ami" Tags="allow_unreserved" />

drivers/version.txt

@@ -1 +1,2 @@
-2.4.1
+2.4.2
+

package/cloudshell/cp/aws/domain/ami_management/operations/deploy_operation.py

@@ -282,11 +282,24 @@ def _get_ami_credentials(self, s3_session, key_pair_location, reservation, wait_
                     "Timeout when waiting for windows credentials. Traceback: {0}".format(traceback.format_exc()))
                 return None
         else:
-            return None if ami_deploy_action.actionParams.appResource.attributes[
-                "User"] else self.credentials_service.get_default_linux_credentials()
+            return None if self._get_deployed_app_resource_user_attribute(ami_deploy_action) else \
+                self.credentials_service.get_default_linux_credentials()
 
         return result
 
+    @staticmethod
+    def _get_deployed_app_resource_user_attribute(ami_deploy_action):
+        """
+        check if deployed app resource has a user attribute, while respecting 2nd gen shell namespaces.
+        2nd gen shells are a kind of resource whose attributes are namespaced,
+        i.e. User is namespace.User in 2nd gen shells
+
+        :param ami_deploy_action: cloudshell.cp.core.models.DeployApp
+        :return:
+        """
+        attribute_names_in_deployed_resource = ami_deploy_action.actionParams.appResource.attributes.keys()
+        return next((attr for attr in attribute_names_in_deployed_resource if attr.split('.')[-1]=='User'))
+
     @staticmethod
     def _get_name_from_tags(result):
         return [tag['Value'] for tag in result.tags if tag['Key'] == 'Name'][0]

package/cloudshell/cp/aws/domain/deployed_app/operations/set_app_security_groups.py

@@ -32,6 +32,12 @@ def set_apps_security_groups(self, reservation, app_security_group_models, ec2_s
         :return:
         """
 
+        # security groups are associated with network interfaces;
+        # we identify the security groups that need to be created or updated, by finding the interfaces
+        # interfaces are identified by comparing the subnet id of the interface, with the requested subnet
+        # however, when working with "Default Subnet" mode, which is when no subnet was specified in sandbox
+        # we instead provide the server will send a subnet id in the form of the vpc CIDR block.
+
         result = []
 
         for app_security_group_model in app_security_group_models:
@@ -46,7 +52,10 @@ def set_apps_security_groups(self, reservation, app_security_group_models, ec2_s
 
                 for security_group_configuration in security_group_configurations:
                     subnet_id = security_group_configuration.subnet_id
-                    network_interfaces = filter(lambda x: x.subnet_id == subnet_id, instance.network_interfaces)
+
+                    network_interfaces = filter(lambda x: x.subnet_id == subnet_id or x.vpc.cidr_block == subnet_id,
+                                                instance.network_interfaces)
+
                     for network_interface in network_interfaces:
                         custom_security_group = self.security_group_service.get_or_create_custom_security_group(
                             ec2_session=ec2_session,

package/cloudshell/cp/aws/domain/services/ec2/security_group.py

@@ -184,7 +184,7 @@ def get_ip_permission_object(port_data):
                 }
             ]}
 
-    def get_inbound_ports_security_group(self, ec2_session,  network_interface):
+    def get_inbound_ports_security_group(self, ec2_session, network_interface):
         """
         Returns an inbound ports security group for the nic
         Inbound ports security group is defined by the following attributes and their values:

package/tests/test_ami_management/test_operations/test_deploy_operation.py

@@ -99,36 +99,24 @@ def test_extract_instance_id_on_cancellation(self):
 
     def test_deploy(self):
         # prepare
-        ami_datamodel = Mock()
-        ami_datamodel.storage_size = 30
-        ami_datamodel.inbound_ports = "80"
-        ami_datamodel.outbound_ports = "20"
-        ami_datamodel.add_public_ip = None
-        ami_datamodel.allocate_elastic_ip = True
-        ami_deploy_action = Mock()
-        ami_deploy_action.actionParms = Mock()
-        ami_deploy_action.actionParams.deployment = Mock()
-        ami_deploy_action.actionParams.deployment.customModel = ami_datamodel
-        instance = Mock()
-        instance.network_interfaces = []
-        instance.tags = [{'Key': 'Name', 'Value': 'my name'}]
+        ami_datamodel = self._create_ami_datamodel()
+        ami_deploy_action = self._create_ami_deploy_action(ami_datamodel)
+        ami_deployment_info = Mock()
+        instance = self._create_instance()
+        network_config_results = [Mock(device_index=0, public_ip=instance.public_ip_address)]
         self.instance_service.create_instance = Mock(return_value=instance)
         sg = Mock()
         self.security_group_service.create_security_group = Mock(return_value=sg)
-        self.deploy_operation._get_block_device_mappings = Mock()
+
+        self._mock_deploy_operation(ami_deployment_info, network_config_results)
+
         cancellation_context = Mock()
         inst_name = 'my name'
         reservation = Mock()
-        ami_deployment_info = Mock()
         network_actions = None
-        self.deploy_operation._create_deployment_parameters = Mock(return_value=ami_deployment_info)
-        self.deploy_operation._populate_network_config_results_with_interface_data = Mock()
-        network_config_results_dto = Mock()
-        network_config_results = [Mock(device_index=0, public_ip=instance.public_ip_address)]
-        self.deploy_operation._prepare_network_result_models = Mock(return_value=network_config_results)
-        self.deploy_operation._prepare_network_config_results_dto = Mock(return_value=[])
 
         # act
+
         res = self.deploy_operation.deploy(ec2_session=self.ec2_session,
                                            s3_session=self.s3_session,
                                            name=inst_name,
@@ -183,6 +171,107 @@ def test_deploy(self):
                 network_config_results=network_config_results,
                 logger=self.logger)
 
+    def test_2ndGen_deployed_app_with_namespaced_user(self):
+        # prepare
+        ami_datamodel = self._create_ami_datamodel()
+        ami_deploy_action = self._create_ami_deploy_action(ami_datamodel)
+        ami_deployment_info = Mock()
+        instance = self._create_instance()
+        instance.platform = None
+        network_config_results = [Mock(device_index=0, public_ip=instance.public_ip_address)]
+        self.instance_service.create_instance = Mock(return_value=instance)
+        sg = Mock()
+        self.security_group_service.create_security_group = Mock(return_value=sg)
+
+        self._mock_deploy_operation(ami_deployment_info, network_config_results)
+
+        cancellation_context = Mock()
+        inst_name = 'my name'
+        reservation = Mock()
+        network_actions = None
+
+        # act
+
+        res = self.deploy_operation.deploy(ec2_session=self.ec2_session,
+                                           s3_session=self.s3_session,
+                                           name=inst_name,
+                                           reservation=reservation,
+                                           aws_ec2_cp_resource_model=self.ec2_datamodel,
+                                           ami_deploy_action=ami_deploy_action,
+                                           network_actions=network_actions,
+                                           ec2_client=self.ec2_client,
+                                           cancellation_context=cancellation_context,
+                                           logger=self.logger)
+
+        ami_credentials = self.credentials_manager.get_windows_credentials()
+
+        # assert
+        self.assertEqual(res[0].vmName, 'my name')
+        self.assertEqual(res[0].deployedAppAdditionalData["inbound_ports"], ami_datamodel.inbound_ports)
+        self.assertEqual(res[0].vmUuid, instance.instance_id)
+        self.assertTrue(self.tag_service.get_security_group_tags.called)
+        self.assertTrue(self.security_group_service.create_security_group.called)
+        self.assertTrue(self.instance_service.set_ec2_resource_tags.called_with(
+                self.security_group_service.create_security_group()),
+                self.tag_service.get_security_group_tags())
+
+        self.assertTrue(self.key_pair.load.called_with(self.ec2_datamodel.key_pair_location,
+                                                       instance.key_pair.key_name,
+                                                       self.key_pair.FILE_SYSTEM))
+
+        self.assertTrue(self.security_group_service.set_security_group_rules.called_with(
+                ami_datamodel, self.security_group_service.create_security_group()))
+
+        self.security_group_service.remove_allow_all_outbound_rule.assert_called_with(security_group=sg)
+
+        self.instance_service.create_instance.assert_called_once_with(ec2_session=self.ec2_session,
+                                                                      name=inst_name,
+                                                                      reservation=reservation,
+                                                                      ami_deployment_info=ami_deployment_info,
+                                                                      ec2_client=self.ec2_client,
+                                                                      wait_for_status_check=ami_datamodel.wait_for_status_check,
+                                                                      cancellation_context=cancellation_context,
+                                                                      logger=self.logger)
+
+        self.deploy_operation.elastic_ip_service.set_elastic_ips.assert_called_once_with(
+                ec2_session=self.ec2_session,
+                ec2_client=self.ec2_client,
+                instance=instance,
+                ami_deployment_model=ami_datamodel,
+                network_actions=None,
+                network_config_results=network_config_results,
+                logger=self.logger)
+
+    def _mock_deploy_operation(self, ami_deployment_info, network_config_results):
+        self.deploy_operation._get_block_device_mappings = Mock()
+        self.deploy_operation._create_deployment_parameters = Mock(return_value=ami_deployment_info)
+        self.deploy_operation._populate_network_config_results_with_interface_data = Mock()
+        self.deploy_operation._prepare_network_result_models = Mock(return_value=network_config_results)
+        self.deploy_operation._prepare_network_config_results_dto = Mock(return_value=[])
+
+    def _create_instance(self):
+        instance = Mock()
+        instance.network_interfaces = []
+        instance.tags = [{'Key': 'Name', 'Value': 'my name'}]
+        return instance
+
+    def _create_ami_deploy_action(self, ami_datamodel):
+        ami_deploy_action = Mock()
+        ami_deploy_action.actionParms = Mock()
+        ami_deploy_action.actionParams.deployment = Mock()
+        ami_deploy_action.actionParams.deployment.customModel = ami_datamodel
+        ami_deploy_action.actionParams.appResource.attributes = {"gen2.User": "lala"}
+        return ami_deploy_action
+
+    def _create_ami_datamodel(self):
+        ami_datamodel = Mock()
+        ami_datamodel.storage_size = 30
+        ami_datamodel.inbound_ports = "80"
+        ami_datamodel.outbound_ports = "20"
+        ami_datamodel.add_public_ip = None
+        ami_datamodel.allocate_elastic_ip = True
+        return ami_datamodel
+
     def test_get_block_device_mappings_throws_max_storage_error(self):
         ec_model = Mock()
         ec_model.max_storage_size = 30

package/version.txt

@@ -1 +1,2 @@
-2.4.1
+2.4.2
+

version.txt

@@ -1 +1,2 @@
-2.4.1
+2.4.2
+