Merge branch 'release-1.7.3'

* release-1.7.3: Bumping version to 1.7.3 Update Changlog [EMR] Added support for Security Groups.
aws · Jan 20, 2015 · 4fa4590 · 4fa4590
2 parents da51de9 + 45ab73f
commit 4fa4590
Show file tree

Hide file tree

Showing 14 changed files with 205 additions and 14 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -2,6 +2,14 @@
 CHANGELOG
 =========
 
+1.7.3
+=====
+
+* feature:``aws emr``: Add support for security groups.
+* feature:``aws cognitio-identity``: Enhance authentication flow by being able
+  to save associations of IAM roles with identity pools.
+
+
 1.7.2
 =====
 

diff --git a/awscli/__init__.py b/awscli/__init__.py
@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.7.2'
+__version__ = '1.7.3'
 
 #
 # Get our data path to be added to botocore's search path

diff --git a/awscli/customizations/emr/applicationutils.py b/awscli/customizations/emr/applicationutils.py
@@ -22,7 +22,7 @@ def build_applications(session,
     step_list = []
     ba_list = []
     region = parsed_globals.region if parsed_globals.region \
-                 else session.get_config_variable('region')
+        else session.get_config_variable('region')
 
     for app_config in parsed_applications:
         app_name = app_config['Name'].lower()

diff --git a/awscli/customizations/emr/argumentschema.py b/awscli/customizations/emr/argumentschema.py
@@ -87,8 +87,29 @@
                 EC2_ROLE_NAME + ". In order to use the default"
                 " role, you must have already created it using the "
                 "<code>create-default-roles</code> command. "
+        },
+        "EmrManagedMasterSecurityGroup": {
+            "type": "string",
+            "description": helptext.EMR_MANAGED_MASTER_SECURITY_GROUP
+        },
+        "EmrManagedSlaveSecurityGroup": {
+            "type": "string",
+            "description": helptext.EMR_MANAGED_SLAVE_SECURITY_GROUP
+        },
+        "AdditionalMasterSecurityGroups": {
+            "type": "array",
+            "description": helptext.ADDITIONAL_MASTER_SECURITY_GROUPS,
+            "items": {
+                "type": "string"
+            }
+        },
+        "AdditionalSlaveSecurityGroups": {
+            "type": "array",
+            "description": helptext.ADDITIONAL_SLAVE_SECURITY_GROUPS,
+            "items": {
+                "type": "string"
+            }
         }
-
     }
 }
 

diff --git a/awscli/customizations/emr/createcluster.py b/awscli/customizations/emr/createcluster.py
@@ -302,6 +302,22 @@ def _build_ec2_attributes(self, cluster, parsed_attrs):
             src_params=parsed_attrs, src_key='InstanceProfile',
             dest_params=cluster, dest_key='JobFlowRole')
 
+        emrutils.apply_params(
+            src_params=parsed_attrs, src_key='EmrManagedMasterSecurityGroup',
+            dest_params=instances, dest_key='EmrManagedMasterSecurityGroup')
+
+        emrutils.apply_params(
+            src_params=parsed_attrs, src_key='EmrManagedSlaveSecurityGroup',
+            dest_params=instances, dest_key='EmrManagedSlaveSecurityGroup')
+
+        emrutils.apply_params(
+            src_params=parsed_attrs, src_key='AdditionalMasterSecurityGroups',
+            dest_params=instances, dest_key='AdditionalMasterSecurityGroups')
+
+        emrutils.apply_params(
+            src_params=parsed_attrs, src_key='AdditionalSlaveSecurityGroups',
+            dest_params=instances, dest_key='AdditionalSlaveSecurityGroups')
+
         emrutils.apply(params=cluster, key='Instances', value=instances)
 
         return cluster

diff --git a/awscli/customizations/emr/describecluster.py b/awscli/customizations/emr/describecluster.py
@@ -77,7 +77,7 @@ def _call(self, operation_object, parameters, parsed_globals):
             endpoint_url=parsed_globals.endpoint_url,
             verify=parsed_globals.verify_ssl)
         http_response, response_data = operation_object.call(endpoint,
-                                                                 **parameters)
+                                                             **parameters)
         return response_data
 
     def _get_key_of_result(self, keys):

diff --git a/awscli/customizations/emr/helptext.py b/awscli/customizations/emr/helptext.py
@@ -99,9 +99,12 @@
 
 EC2_ATTRIBUTES = (
     '<p>Specifies the following Amazon EC2 attributes: KeyName,'
-    ' AvailabilityZone, SubnetId, and InstanceProfile. AvailabilityZone and '
-    'Subnet cannot be specified together. To create the default '
-    'instance profile <code>' + EC2_ROLE_NAME + '</code>,'
+    ' AvailabilityZone, SubnetId, InstanceProfile,'
+    ' EmrManagedMasterSecurityGroup, EmrManagedSlaveSecurityGroup,'
+    ' AdditionalMasterSecurityGroups and AdditionalSlaveSecurityGroups.'
+    ' AvailabilityZone and Subnet cannot be specified together.'
+    ' To create the default instance profile <code>'
+    + EC2_ROLE_NAME + '</code>,'
     ' use <code>aws emr create-default-roles</code> command. </p>'
     'This command will also create the default EMR service role '
     '<code>' + EMR_ROLE_NAME + '</code>.'
@@ -112,7 +115,16 @@
     '<li>SubnetId- Assign the EMR cluster to this Amazon VPC Subnet. </li>'
     '<li>InstanceProfile - Provides access to other AWS services such as S3,'
     ' DynamoDB from EC2 instances that are launched by EMR.. </li>'
-    )
+    '<li>EmrManagedMasterSecurityGroup - The identifier of the Amazon EC2'
+    ' security group (managed by Amazon Elastic MapReduce)'
+    ' for the master node. </li>'
+    '<li>EmrManagedSlaveSecurityGroup - The identifier of the Amazon EC2'
+    ' security group (managed by Amazon Elastic MapReduce)'
+    ' for the slave nodes.</li>'
+    '<li>AdditionalMasterSecurityGroups - A list of additional Amazon EC2'
+    ' security group IDs for the master node</li>'
+    '<li>AdditionalSlaveSecurityGroups - A list of additional Amazon EC2'
+    ' security group IDs for the slave nodes.</li>')
 
 AUTO_TERMINATE = (
     '<p>Specifies whether the cluster should terminate after'
@@ -227,3 +239,19 @@
 LIST_CLUSTERS_CREATED_BEFORE = (
     '<p>The creation date and time end value filter for '
     'listing clusters. For example, 2014-07-15T00:01:30. </p>')
+
+EMR_MANAGED_MASTER_SECURITY_GROUP = (
+    '<p>The identifier of the Amazon EC2 security group (managed by Amazon '
+    'Elastic MapReduce) for the master node.</p>')
+
+EMR_MANAGED_SLAVE_SECURITY_GROUP = (
+    '<p>The identifier of the Amazon EC2 security group (managed by Amazon '
+    'Elastic MapReduce) for the slave nodes.</p>')
+
+ADDITIONAL_MASTER_SECURITY_GROUPS = (
+    '<p> A list of additional Amazon EC2 security group IDs for '
+    'the master node</p>')
+
+ADDITIONAL_SLAVE_SECURITY_GROUPS = (
+    '<p>A list of additional Amazon EC2 security group IDs for '
+    'the slave nodes.</p>')
diff --git a/awscli/examples/emr/create-cluster-examples.rst b/awscli/examples/emr/create-cluster-examples.rst
@@ -40,7 +40,39 @@
 
 - Create an Amazon EMR cluster in an AvailabilityZone. For example, us-east-1b::
 
-    aws emr create-cluster --ec2-attributes AvailabilityZone=us-east-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge 
+    aws emr create-cluster --ec2-attributes AvailabilityZone=us-east-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge
+
+- Create an Amazon EMR cluster specifying the Amazon EC2 security groups::
+
+	aws emr create-cluster --ami-version 3.3.1 --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=myRole,EmrManagedMasterSecurityGroup=sg-master1,EmrManagedSlaveSecurityGroup=sg-slave1,AdditionalMasterSecurityGroups=[sg-addMaster1,sg-addMaster2,sg-addMaster3,sg-addMaster4],AdditionalSlaveSecurityGroups=[sg-addSlave1,sg-addSlave2,sg-addSlave3,sg-addSlave4] --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge
+
+- Create an Amazon EMR cluster specifying only the EMR managed Amazon EC2 security groups::
+
+	aws emr create-cluster --ami-version 3.3.1 --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=myRole,EmrManagedMasterSecurityGroup=sg-master1,EmrManagedSlaveSecurityGroup=sg-slave1 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge
+
+- Create an Amazon EMR cluster specifying only the additional Amazon EC2 security groups::
+
+	aws emr create-cluster --ami-version 3.3.1 --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=myRole,AdditionalMasterSecurityGroups=[sg-addMaster1,sg-addMaster2,sg-addMaster3,sg-addMaster4],AdditionalSlaveSecurityGroups=[sg-addSlave1,sg-addSlave2,sg-addSlave3,sg-addSlave4] --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge
+
+- JSON equivalent (contents of ec2_attributes.json)::
+
+    [
+     {
+       "SubnetId": "subnet-xxxxx",
+       "KeyName": "myKey",
+       "InstanceProfile":"myRole",
+       "EmrManagedMasterSecurityGroup": "sg-master1",
+       "EmrManagedSlaveSecurityGroup": "sg-slave1",
+       "AdditionalMasterSecurityGroups": ["sg-addMaster1","sg-addMaster2","sg-addMaster3","sg-addMaster4"],
+       "AdditionalSlaveSecurityGroups": ["sg-addSlave1","sg-addSlave2","sg-addSlave3","sg-addSlave4"]
+     }
+   ]
+
+NOTE: JSON arguments must include options and values as their own items in the list.
+
+- Command (using ec2_attributes.json)::
+
+	aws emr create-cluster --ami-version 3.3.1 --service-role EMR_DefaultRole --ec2-attributes file://./ec2_attributes.json  --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge
 
 **7. Enable debugging and specify a Log URI**
 
@@ -211,4 +243,3 @@ NOTE: JSON arguments must include options and values as their own items in the l
 - Command::
 
     aws emr create-cluster --instance-type m3.xlarge --ami-version 3.2.1 --emrfs SSE=true,Consistent=true,RetryCount=5,RetryPeriod=30,Args=[fs.s3.serverSideEncryptionAlgorithm=AES256]
-
diff --git a/doc/source/conf.py b/doc/source/conf.py
@@ -52,7 +52,7 @@
 # The short X.Y version.
 version = '1.7'
 # The full version, including alpha/beta/rc tags.
-release = '1.7.2'
+release = '1.7.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/setup.py b/setup.py
@@ -6,7 +6,7 @@
 import awscli
 
 
-requires = ['botocore>=0.83.0,<0.84.0',
+requires = ['botocore>=0.84.0,<0.85.0',
             'bcdoc>=0.12.0,<0.13.0',
             'colorama==0.2.5',
             'docutils>=0.10',

diff --git a/tests/unit/customizations/emr/__init__.py b/tests/unit/customizations/emr/__init__.py
@@ -24,4 +24,5 @@ class EMRBaseAWSCommandParamsTest(BaseAWSCommandParamsTest):
     def setUp(self):
         super(EMRBaseAWSCommandParamsTest, self).setUp()
         # We actually will just disable preview mode completely.
-        self.driver.session.unregister('building-command-table.main', mark_as_preview)
+        self.driver.session.unregister('building-command-table.main',
+                                       mark_as_preview)
diff --git a/tests/unit/customizations/emr/input_ec2_attributes_with_security_groups.json b/tests/unit/customizations/emr/input_ec2_attributes_with_security_groups.json
@@ -0,0 +1,6 @@
+{
+    "EmrManagedMasterSecurityGroup": "sg-master1",
+    "EmrManagedSlaveSecurityGroup": "sg-slave1",
+    "AdditionalMasterSecurityGroups": ["sg-addMaster1","sg-addMaster2","sg-addMaster3","sg-addMaster4"],
+    "AdditionalSlaveSecurityGroups": ["sg-addSlave1","sg-addSlave2","sg-addSlave3","sg-addSlave4"]
+}
diff --git a/tests/unit/customizations/emr/test_add_instance_groups.py b/tests/unit/customizations/emr/test_add_instance_groups.py
@@ -61,7 +61,8 @@ def test_instance_groups_missing_instance_group_type_error(self):
         cmd = self.prefix + ' Name=Task,InstanceType=m1.small,' +\
             'InstanceCount=5'
         result = self.run_cmd(cmd, 255)
-        self.assert_error_message_has_field_name(result[1], 'InstanceGroupType')
+        self.assert_error_message_has_field_name(result[1],
+                                                 'InstanceGroupType')
 
     def test_instance_groups_missing_instance_type_error(self):
         cmd = self.prefix + ' Name=Task,InstanceGroupType=Task,' +\

diff --git a/tests/unit/customizations/emr/test_create_cluster.py b/tests/unit/customizations/emr/test_create_cluster.py
@@ -322,6 +322,16 @@
         'Tags': []
     }
 
+EMR_MANAGED_MASTER_SECURITY_GROUP = 'sg-master1'
+
+EMR_MANAGED_SLAVE_SECURITY_GROUP = 'sg-slave1'
+
+ADDITIONAL_MASTER_SECURITY_GROUPS = \
+    ['sg-addMaster1', 'sg-addMaster2', 'sg-addMaster3', 'sg-addMaster4']
+
+ADDITIONAL_SLAVE_SECURITY_GROUPS = \
+    ['sg-addSlave1', 'sg-addSlave2', 'sg-addSlave3', 'sg-addSlave4']
+
 
 class TestCreateCluster(BaseAWSCommandParamsTest):
     prefix = 'emr create-cluster '
@@ -1250,5 +1260,74 @@ def test_emr_fs_config(self):
         cmd = DEFAULT_CMD + '--emrfs file://' + data_path
         self.assert_params_for_cmd2(cmd, result)
 
+    def test_all_security_groups(self):
+        cmd = DEFAULT_CMD + (
+            '--ec2-attributes EmrManagedMasterSecurityGroup=sg-master1,'
+            'EmrManagedSlaveSecurityGroup=sg-slave1,AdditionalMasterSecu'
+            'rityGroups=[sg-addMaster1,sg-addMaster2,sg-addMaster3,'
+            'sg-addMaster4],AdditionalSlaveSecurityGroups=[sg-addSlave1,'
+            'sg-addSlave2,sg-addSlave3,sg-addSlave4]')
+
+        result = copy.deepcopy(DEFAULT_RESULT)
+        instances = result['Instances']
+        instances['EmrManagedMasterSecurityGroup'] = \
+            EMR_MANAGED_MASTER_SECURITY_GROUP
+        instances['EmrManagedSlaveSecurityGroup'] = \
+            EMR_MANAGED_SLAVE_SECURITY_GROUP
+        instances['AdditionalMasterSecurityGroups'] = \
+            ADDITIONAL_MASTER_SECURITY_GROUPS
+        instances['AdditionalSlaveSecurityGroups'] = \
+            ADDITIONAL_SLAVE_SECURITY_GROUPS
+
+        self.assert_params_for_cmd2(cmd, result)
+
+    def test_emr_managed_security_groups(self):
+        cmd = DEFAULT_CMD + (
+            '--ec2-attributes EmrManagedMasterSecurityGroup=sg-master1,'
+            'EmrManagedSlaveSecurityGroup=sg-slave1')
+
+        result = copy.deepcopy(DEFAULT_RESULT)
+        instances = result['Instances']
+        instances['EmrManagedMasterSecurityGroup'] = \
+            EMR_MANAGED_MASTER_SECURITY_GROUP
+        instances['EmrManagedSlaveSecurityGroup'] = \
+            EMR_MANAGED_SLAVE_SECURITY_GROUP
+
+        self.assert_params_for_cmd2(cmd, result)
+
+    def test_additional_security_groups(self):
+        cmd = DEFAULT_CMD + (
+            '--ec2-attributes AdditionalMasterSecurityGroups=[sg-addMaster1'
+            ',sg-addMaster2,sg-addMaster3,sg-addMaster4],AdditionalSlaveSecu'
+            'rityGroups=[sg-addSlave1,sg-addSlave2,sg-addSlave3,sg-addSlave4]')
+
+        result = copy.deepcopy(DEFAULT_RESULT)
+        instances = result['Instances']
+        instances['AdditionalMasterSecurityGroups'] = \
+            ADDITIONAL_MASTER_SECURITY_GROUPS
+        instances['AdditionalSlaveSecurityGroups'] = \
+            ADDITIONAL_SLAVE_SECURITY_GROUPS
+
+        self.assert_params_for_cmd2(cmd, result)
+
+    def test_security_groups_from_json_file(self):
+        data_path = os.path.join(
+            os.path.dirname(__file__),
+            'input_ec2_attributes_with_security_groups.json')
+        cmd = DEFAULT_CMD + '--ec2-attributes file://' + data_path
+
+        result = copy.deepcopy(DEFAULT_RESULT)
+        instances = result['Instances']
+        instances['EmrManagedMasterSecurityGroup'] = \
+            EMR_MANAGED_MASTER_SECURITY_GROUP
+        instances['EmrManagedSlaveSecurityGroup'] = \
+            EMR_MANAGED_SLAVE_SECURITY_GROUP
+        instances['AdditionalMasterSecurityGroups'] = \
+            ADDITIONAL_MASTER_SECURITY_GROUPS
+        instances['AdditionalSlaveSecurityGroups'] = \
+            ADDITIONAL_SLAVE_SECURITY_GROUPS
+
+        self.assert_params_for_cmd2(cmd, result)
+
 if __name__ == "__main__":
     unittest.main()