Add iam_lambda_role parameter under cluster section in the config file

CHANGELOG.md

@@ -15,6 +15,10 @@ CHANGELOG
   failures due to CloudFormation throttling.
 - Add support for io2 EBS volume type.
 - Install EFA kernel module also on ARM instances with `alinux2` and `ubuntu1804`
+- Add `iam_lambda_role` parameter under `cluster` section to enable the possibility to specify an existing IAM role to 
+  be used by AWS Lambda functions in CloudFormation. 
+  When using `sge`, `torque`, or `slurm` as the scheduler, 
+  `pcluster` will not create any IAM role if both `ec2_iam_role` and `iam_lambda_role` are provided.
 
 **CHANGES**
 

cli/src/pcluster/config/mappings.py

@@ -1007,6 +1007,10 @@
         "validators": [s3_bucket_validator],
         "update_policy": UpdatePolicy.READ_ONLY_RESOURCE_BUCKET,
     }),
+    ("iam_lambda_role", {
+        "cfn_param_mapping": "IAMLambdaRoleName",
+        "update_policy": UpdatePolicy.SUPPORTED,
+    }),
 ]
 
 

cli/src/pcluster/examples/config

@@ -109,6 +109,9 @@ key_name = mykey
 # Existing EC2 IAM policies to be associated with the EC2 instances
 # (defaults to NONE)
 #additional_iam_policies = NONE
+# Existing IAM role to be associated with Lambda functions
+# (defaults to NONE)
+#iam_lambda_role = NONE
 # Disable Hyperthreading on all instances
 # (defaults to False)
 #disable_hyperthreading = false

cli/tests/pcluster/config/defaults.py

@@ -145,6 +145,7 @@
     "architecture": "x86_64",
     "network_interfaces_count": ["1", "1"],
     "cluster_resource_bucket": None,
+    "iam_lambda_role": None,
 }
 
 DEFAULT_CLUSTER_HIT_DICT = {
@@ -194,6 +195,7 @@
     "architecture": "x86_64",
     "network_interfaces_count": ["1", "1"],
     "cluster_resource_bucket": None,  # cluster_resource_bucket no default, but this is here to make testing easier
+    "iam_lambda_role": None,
 }
 
 DEFAULT_CW_LOG_DICT = {"enable": True, "retention_days": 14}
@@ -226,8 +228,8 @@ class DefaultDict(Enum):
 # ------------------ Default CFN parameters ------------------ #
 
 # number of CFN parameters created by the PclusterConfig object.
-CFN_SIT_CONFIG_NUM_OF_PARAMS = 61
-CFN_HIT_CONFIG_NUM_OF_PARAMS = 52
+CFN_SIT_CONFIG_NUM_OF_PARAMS = 62
+CFN_HIT_CONFIG_NUM_OF_PARAMS = 53
 
 # CFN parameters created by the pcluster CLI
 CFN_CLI_RESERVED_PARAMS = ["ArtifactS3RootDirectory", "RemoveBucketOnDeletion"]
@@ -343,6 +345,7 @@ class DefaultDict(Enum):
     # architecture
     "Architecture": "x86_64",
     "NetworkInterfacesCount": "1,1",
+    "IAMLambdaRoleName": "NONE",
 }
 
 
@@ -412,6 +415,7 @@ class DefaultDict(Enum):
     # architecture
     "Architecture": "x86_64",
     "NetworkInterfacesCount": "1,1",
+    "IAMLambdaRoleName": "NONE",
 }
 
 

cloudformation/aws-parallelcluster.cfn.json

@@ -285,6 +285,11 @@
       "Type": "CommaDelimitedList",
       "Default": "NONE"
     },
+    "IAMLambdaRoleName": {
+      "Description": "Existing IAM role name for Lambda functions",
+      "Type": "String",
+      "Default": "NONE"
+    },
     "VPCSecurityGroupId": {
       "Description": "Existing VPC security group Id",
       "Type": "String",
@@ -660,6 +665,14 @@
         "NONE"
       ]
     },
+    "CreateIAMLambdaRole": {
+      "Fn::Equals": [
+        {
+          "Ref": "IAMLambdaRoleName"
+        },
+        "NONE"
+      ]
+    },
     "AddHITIamPolicies": {
       "Fn::And": [
         {
@@ -2350,6 +2363,9 @@
               "MasterServerSubstack",
               "Outputs.MasterPrivateIP"
             ]
+          },
+          "IAMLambdaRoleName": {
+            "Ref": "IAMLambdaRoleName"
           }
         },
         "TemplateURL": {
@@ -2510,7 +2526,8 @@
             "PolicyName": "LambdaPolicy"
           }
         ]
-      }
+      },
+      "Condition": "CreateIAMLambdaRole"
     },
     "CleanupResourcesS3BucketCustomResource": {
       "Type": "AWS::CloudFormation::CustomResource",
@@ -2587,9 +2604,17 @@
         "Handler": "cleanup_resources.handler",
         "MemorySize": 128,
         "Role": {
-          "Fn::GetAtt": [
-            "CleanupResourcesFunctionExecutionRole",
-            "Arn"
+          "Fn::If": [
+            "CreateIAMLambdaRole",
+            {
+              "Fn::GetAtt": [
+                "CleanupResourcesFunctionExecutionRole",
+                "Arn"
+              ]
+            },
+            {
+              "Fn::Sub": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${IAMLambdaRoleName}"
+            }
           ]
         },
         "Runtime": "python3.8",
@@ -3963,6 +3988,9 @@
               }
             ]
           },
+          "IAMLambdaRoleName": {
+            "Ref": "IAMLambdaRoleName"
+          },
           "ResourcesS3Bucket": {
             "Ref": "ResourcesS3Bucket"
           },

cloudformation/batch-substack.cfn.json

@@ -99,6 +99,10 @@
     "MasterPrivateIP": {
       "Description": "Private IP of the head node",
       "Type": "String"
+    },
+    "IAMLambdaRoleName": {
+      "Description": "Existing IAM role name for Lambda functions",
+      "Type": "String"
     }
   },
   "Conditions": {
@@ -117,6 +121,14 @@
         },
         "arm64"
       ]
+    },
+    "CreateIAMLambdaRole": {
+      "Fn::Equals": [
+        {
+          "Ref": "IAMLambdaRoleName"
+        },
+        "NONE"
+      ]
     }
   },
   "Resources": {
@@ -821,9 +833,17 @@
         "Handler": "manage_docker_images.handler",
         "MemorySize": 128,
         "Role": {
-          "Fn::GetAtt": [
-            "ManageDockerImagesFunctionExecutionRole",
-            "Arn"
+          "Fn::If": [
+            "CreateIAMLambdaRole",
+            {
+              "Fn::GetAtt": [
+                "ManageDockerImagesFunctionExecutionRole",
+                "Arn"
+              ]
+            },
+            {
+              "Fn::Sub": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${IAMLambdaRoleName}"
+            }
           ]
         },
         "Runtime": "python3.6",
@@ -897,7 +917,8 @@
             "PolicyName": "LambdaPolicy"
           }
         ]
-      }
+      },
+      "Condition": "CreateIAMLambdaRole"
     },
     "DockerBuildWaitHandle": {
       "Type": "AWS::CloudFormation::WaitConditionHandle",
@@ -952,7 +973,8 @@
             "PolicyName": "LambdaPolicy"
           }
         ]
-      }
+      },
+      "Condition": "CreateIAMLambdaRole"
     },
     "SendBuildNotificationFunction": {
       "Type": "AWS::Lambda::Function",
@@ -971,9 +993,17 @@
         "Handler": "send_build_notification.handler",
         "MemorySize": 128,
         "Role": {
-          "Fn::GetAtt": [
-            "SendBuildNotificationFunctionExecutionRole",
-            "Arn"
+          "Fn::If": [
+            "CreateIAMLambdaRole",
+            {
+              "Fn::GetAtt": [
+                "SendBuildNotificationFunctionExecutionRole",
+                "Arn"
+              ]
+            },
+            {
+              "Fn::Sub": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${IAMLambdaRoleName}"
+            }
           ]
         },
         "Runtime": "python3.6",

cloudformation/compute-fleet-hit-substack.cfn.yaml

@@ -85,6 +85,8 @@ Parameters:
     Type: AWS::EC2::VPC::Id
   RootRole:
     Type: String
+  IAMLambdaRoleName:
+    Type: String
   ResourcesS3Bucket:
     Type: String
   ArtifactS3RootDirectory:
@@ -101,6 +103,9 @@ Conditions:
   UseAssociatePublicIpAddress: !Equals
     - !Ref 'AssociatePublicIpAddress'
     - true
+  CreateIAMLambdaRole: !Equals
+    - !Ref 'IAMLambdaRoleName'
+    - NONE
 Resources:
 {%- for queue, queue_config in queues.items() %}
   {%- for compute_resource in queue_config.compute_resource_settings.values() %}
@@ -528,7 +533,10 @@ Resources:
         S3Key: !Sub '${ArtifactS3RootDirectory}/custom_resources_code/artifacts.zip'
       Handler: cleanup_resources.handler
       MemorySize: 128
-      Role: !GetAtt 'CleanupRoute53FunctionExecutionRole.Arn'
+      Role: !If
+        - CreateIAMLambdaRole
+        - !GetAtt 'CleanupRoute53FunctionExecutionRole.Arn'
+        - !Sub 'arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${IAMLambdaRoleName}'
       Runtime: python3.8
       Timeout: 900
   CleanupRoute53CustomResource:
@@ -570,6 +578,7 @@ Resources:
                     - ClusterHostedZone: !Ref 'ClusterHostedZone'
             Version: '2012-10-17'
           PolicyName: LambdaPolicy
+    Condition: CreateIAMLambdaRole
   {%- endif %}
   UpdateWaiterFunction:
     Type: AWS::Lambda::Function
@@ -580,7 +589,10 @@ Resources:
         S3Key: !Sub '${ArtifactS3RootDirectory}/custom_resources_code/artifacts.zip'
       Handler: wait_for_update.handler
       MemorySize: 128
-      Role: !GetAtt 'UpdateWaiterFunctionExecutionRole.Arn'
+      Role: !If
+        - CreateIAMLambdaRole
+        - !GetAtt 'UpdateWaiterFunctionExecutionRole.Arn'
+        - !Sub 'arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${IAMLambdaRoleName}'
       Runtime: python3.8
       Timeout: 900
   UpdateWaiterFunctionExecutionRole:
@@ -613,6 +625,7 @@ Resources:
                 Resource: !Sub 'arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${DynamoDBTable}'
             Version: '2012-10-17'
           PolicyName: LambdaPolicy
+    Condition: CreateIAMLambdaRole
 Metadata:
   RootRole: !Ref 'RootRole'
   VPCId: !Ref 'VPCId'

tests/integration-tests/configs/common/common.yaml

@@ -219,13 +219,19 @@ efa:
         instances: ["p4d.24xlarge"]
         oss: ["alinux", "ubuntu1804", "centos7"]
         schedulers: ["sge"]
-iam_policies:
-  test_iam_policies.py::test_iam_policies:
+iam:
+  test_iam.py::test_iam_policies:
     dimensions:
       - regions: ["eu-north-1"]
         instances: {{ common.INSTANCES_DEFAULT_X86 }}
         oss: ["alinux2"]
         schedulers: ["slurm", "awsbatch"]
+  test_iam.py::test_iam_roles:
+    dimensions:
+      - regions: ["us-east-2"]
+        schedulers: ["awsbatch", "slurm", "sge"]
+        oss: ["alinux2"]
+        instances: {{ common.INSTANCES_DEFAULT_X86 }}
 intel_hpc:
   test_intel_hpc.py::test_intel_hpc:
     dimensions:

tests/integration-tests/configs/new_region.yaml

@@ -108,8 +108,8 @@ test-suites:
           instances: ["c5n.18xlarge"]
           oss: ["alinux2"]
           schedulers: ["slurm"]
-  iam_policies:
-    test_iam_policies.py::test_iam_policies:
+  iam:
+    test_iam.py::test_iam_policies:
       dimensions:
         - regions: {{ NEW_REGION }}
           instances: {{ common.INSTANCES_DEFAULT_X86 }}