ecs-cluster-def.yml

AWSTemplateFormatVersion: '2010-09-09'
Description: EC2 ECS cluster running containers in a public subnet. Only supports
             public facing load balancer, and public service discovery namespaces.
             Adapted from https://github.com/brentley/tensorflow-container/blob/master/cluster-cpu-gpu.yml
Parameters:
  DynamoDBTable:
    Description: name of the dynamoDB table used in the project
    Type: String
    Default: my_derby_project
  S3Bucket:
    Description: name of the s3 bucket used in the project
    Type: String
    Default: cp-derby-bucket
  KeyName:
    Description: Name of an existing EC2 KeyPair to enable SSH access to the instance
    Type: AWS::EC2::KeyPair::KeyName
    Default: 'derby-cluster-sshkey'
    ConstraintDescription: must be the name of an existing EC2 KeyPair.
  EnvironmentName:
    Type: String
    Default: test
    Description: "A friendly environment name that will be used for namespacing all cluster resources. Example: staging, qa, or production"
#  InstanceType:
#    Description: EC2 instance type
#    Type: String
#    Default: t2.micro
#    Description: Class of EC2 instance used to host containers. Choose t2 for testing, m5 for general purpose, c5 for CPU intensive services, and r5 for memory intensive services
#    AllowedValues: [ t2.micro, t2.small, t2.medium, t2.large, t2.xlarge, t2.2xlarge,
#     m5.large, m5.xlarge, m5.2large, m5.4xlarge, m5.12xlarge, m5.24large,
#     c5.large, c5.xlarge, c5.2xlarge, c5.4xlarge, c5.9xlarge, c5.18xlarge,
#     r5.large, r5.xlarge, r5.2xlarge, r5.4xlarge, r5.12xlarge, r5.24xlarge ]
#    ConstraintDescription: Please choose a valid instance type.
  GPUInstanceType:
    Description: EC2 GPU instance type
    Type: String
    Default: p2.xlarge
    Description: Class of EC2 instance used to host containers. Choose t2 for testing, m5 for general purpose, c5 for CPU intensive services, and r5 for memory intensive services
    AllowedValues: [ p2.xlarge, p2.8xlarge, p2.16xlarge, p3.2xlarge, p3.8xlarge, p3.16xlarge, p3dn.24xlarge ]
    ConstraintDescription: Please choose a valid instance type.
#  DesiredCapacity:
#    Type: Number
#    Default: '0'
#    Description: Number of EC2 instances to launch in your ECS cluster.
  DesiredGPUCapacity:
    Type: Number
    Default: '1'
    Description: Number of EC2 instances to launch in your ECS cluster.
#  MaxSize:
#    Type: Number
#    Default: '2'
#    Description: Maximum number of EC2 instances that can be launched in your ECS cluster.
  MaxGPUSize:
    Type: Number
    Default: '2'
    Description: Maximum number of EC2 instances that can be launched in your ECS cluster.
#  ECSAMI:
#    Description: AMI ID
#    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
#    Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
#    Description: The Amazon Machine Image ID used for the cluster, leave it as the default value to get the latest AMI
  ECSGPUAMI:
    Description: GPU AMI ID
    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
    Default: /aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id #amzn2-ami-ecs-gpu-hvm-2.0.20190321-x86_64-ebs/image_id
    Description: The Amazon Machine Image ID used for the gpu instances, leave it as the default value to get the latest AMI

Mappings:
  # Hard values for the subnet masks. These masks define
  # the range of internal IP addresses that can be assigned.
  # The VPC can have all IP's from 10.0.0.0 to 10.0.255.255
  # There are two subnets which cover the ranges:
  #
  # 10.0.0.0 - 10.0.0.255
  # 10.0.1.0 - 10.0.1.255
  #
  # If you need more IP addresses (perhaps you have so many
  # instances that you run out) then you can customize these
  # ranges to add more
  SubnetConfig:
    VPC:
      CIDR: '10.0.0.0/16'
    PublicOne:
      CIDR: '10.0.0.0/24'
Resources:
  # VPC in which containers will be networked.
  # It has two public subnets
  # We distribute the subnets across the first two available subnets
  # for the region, for high availability.
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      EnableDnsSupport: true
      EnableDnsHostnames: true
      CidrBlock: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']

  # Two public subnets, where containers can have public IP addresses
  PublicSubnetOne:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone:
         Fn::Select:
         - 0
         - Fn::GetAZs: {Ref: 'AWS::Region'}
      VpcId: !Ref 'VPC'
      CidrBlock: !FindInMap ['SubnetConfig', 'PublicOne', 'CIDR']
      MapPublicIpOnLaunch: true

  # Setup networking resources for the public subnets. Containers
  # in the public subnets have public IP addresses and the routing table
  # sends network traffic via the internet gateway.
  InternetGateway:
    Type: AWS::EC2::InternetGateway
  GatewayAttachement:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId: !Ref 'VPC'
      InternetGatewayId: !Ref 'InternetGateway'
  PublicRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref 'VPC'
  PublicRoute:
    Type: AWS::EC2::Route
    DependsOn: GatewayAttachement
    Properties:
      RouteTableId: !Ref 'PublicRouteTable'
      DestinationCidrBlock: '0.0.0.0/0'
      GatewayId: !Ref 'InternetGateway'
  PublicSubnetOneRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PublicSubnetOne
      RouteTableId: !Ref PublicRouteTable

  # ECS Resources
  ECSCluster:
    Type: AWS::ECS::Cluster
    Properties:
      ClusterName: ecs-cluster-derby

  # A security group for the EC2 hosts that will run the containers.
  # Rules will be added depending on what ingress is created.
  ContainerSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Access to the ECS hosts that run containers
      VpcId: !Ref 'VPC'
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 22
          ToPort: 22
          CidrIp: 0.0.0.0/0

  # Autoscaling group. This launches the actual EC2 instances that will register
  # themselves as members of the cluster, and run the docker containers.
#  ECSAutoScalingGroup:
#    Type: AWS::AutoScaling::AutoScalingGroup
#    Properties:
#      VPCZoneIdentifier:
#        - !Ref PublicSubnetOne
##        - !Ref PublicSubnetTwo
#      LaunchConfigurationName: !Ref 'ContainerInstances'
#      MinSize: '1'
#      MaxSize: !Ref 'MaxSize'
#      DesiredCapacity: !Ref 'DesiredCapacity'
#    CreationPolicy:
#      ResourceSignal:
#        Timeout: PT15M
#    UpdatePolicy:
#      AutoScalingReplacingUpdate:
#        WillReplace: 'true'
#  ContainerInstances:
#    Type: AWS::AutoScaling::LaunchConfiguration
#    Properties:
#      ImageId: !Ref 'ECSAMI'
#      SecurityGroups: [!Ref 'ContainerSecurityGroup']
#      InstanceType: !Ref 'InstanceType'
#      IamInstanceProfile: !Ref 'EC2InstanceProfile'
#      UserData:
#        Fn::Base64: !Sub |
#          #!/bin/bash -xe
#          echo ECS_CLUSTER=${ECSCluster} >> /etc/ecs/ecs.config
#          yum install -y aws-cfn-bootstrap
#          /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource ECSAutoScalingGroup --region ${AWS::Region}
#  EC2InstanceProfile:
#    Type: AWS::IAM::InstanceProfile
#    Properties:
#      Path: /
#      Roles: [!Ref 'EC2Role']

  # Autoscaling group for gpu instances. This launches the actual EC2 instances that will register
  # themselves as members of the cluster, and run the docker containers.
  ECSGPUAutoScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      VPCZoneIdentifier:
        - !Ref PublicSubnetOne
#        - !Ref PublicSubnetTwo
      LaunchConfigurationName: !Ref 'GPUContainerInstances'
      MinSize: '0'
      MaxSize: !Ref 'MaxGPUSize'
      DesiredCapacity: !Ref 'DesiredGPUCapacity'
    CreationPolicy:
      ResourceSignal:
        Timeout: PT15M
    UpdatePolicy:
      AutoScalingReplacingUpdate:
        WillReplace: 'true'
  GPUContainerInstances:
    Type: AWS::AutoScaling::LaunchConfiguration
    Properties:
      ImageId: !Ref 'ECSGPUAMI'
      SecurityGroups: [!Ref 'ContainerSecurityGroup']
      InstanceType: !Ref 'GPUInstanceType'
      IamInstanceProfile: !Ref 'GPUEC2InstanceProfile'
      KeyName: !Ref 'KeyName'
      UserData: !Base64
        'Fn::Join':
          - ''
          - - |
              #!/bin/bash -x
            - |
              # Install the files and packages from the metadata
            - sudo mkdir /etc/ecs/
            - sudo chmod 777 /etc/ecs/
            - |+

            - |
            - 'echo ECS_CLUSTER='
            - !Ref 'ECSCluster'
            - ' >> /etc/ecs/ecs.config '
            - |+

            - |
            - 'yum install -y aws-cfn-bootstrap'
            - |+

            - |
              # Signal the status from cfn-init
            - '/opt/aws/bin/cfn-signal -e $? '
            - '         --stack '
            - !Ref 'AWS::StackName'
            - '         --resource ECSGPUAutoScalingGroup '
            - '         --region '
            - !Ref 'AWS::Region'
            - |+
  GPUEC2InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles: [!Ref 'EC2Role']

  # A role used to allow AWS Autoscaling to inspect stats and adjust scaleable targets
  # on your AWS account
  AutoscalingRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service: [application-autoscaling.amazonaws.com]
          Action: ['sts:AssumeRole']
      Path: /
      Policies:
      - PolicyName: service-autoscaling
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
              - 'application-autoscaling:*'
              - 'cloudwatch:DescribeAlarms'
              - 'cloudwatch:PutMetricAlarm'
              - 'ecs:DescribeServices'
              - 'ecs:UpdateService'
            Resource: '*'

  # Role for the EC2 hosts. This allows the ECS agent on the EC2 hosts
  # to communciate with the ECS control plane, as well as download the docker
  # images from ECR to run on your host.
  EC2Role:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service: [ec2.amazonaws.com]
          Action: ['sts:AssumeRole']
      Path: /
      Policies:
      - PolicyName: ecs-service
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
              - 'ecs:CreateCluster'
              - 'ecs:DeregisterContainerInstance'
              - 'ecs:DiscoverPollEndpoint'
              - 'ecs:Poll'
              - 'ecs:RegisterContainerInstance'
              - 'ecs:StartTelemetrySession'
              - 'ecs:Submit*'
              - 'logs:CreateLogStream'
              - 'logs:PutLogEvents'
              - 'ecr:GetAuthorizationToken'
              - 'ecr:BatchGetImage'
              - 'ecr:GetDownloadUrlForLayer'
              - 'cloudformation:SignalResource'
            Resource: '*'

  # This is an IAM role which authorizes ECS to manage resources on your
  # account on your behalf, such as updating your load balancer with the
  # details of where your containers are, so that traffic can reach your
  # containers.
  ECSRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service: [ecs.amazonaws.com]
          Action: ['sts:AssumeRole']
      Path: /
      Policies:
      - PolicyName: ecs-service
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
              # Rules which allow ECS to attach network interfaces to instances
              # on your behalf in order for awsvpc networking mode to work right
#              - 'ec2:AttachNetworkInterface'
#              - 'ec2:CreateNetworkInterface'
#              - 'ec2:CreateNetworkInterfacePermission'
#              - 'ec2:DeleteNetworkInterface'
#              - 'ec2:DeleteNetworkInterfacePermission'
#              - 'ec2:Describe*'
#              - 'ec2:DetachNetworkInterface'

              # Rules which allow ECS to update load balancers on your behalf
              # with the information sabout how to send traffic to your containers
              - 'elasticloadbalancing:DeregisterInstancesFromLoadBalancer'
              - 'elasticloadbalancing:DeregisterTargets'
              - 'elasticloadbalancing:Describe*'
              - 'elasticloadbalancing:RegisterInstancesWithLoadBalancer'
              - 'elasticloadbalancing:RegisterTargets'
            Resource: '*'
  # This is a role which is used by the ECS tasks themselves.
  ECSTaskExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: derbyEcsTaskExecutionRole
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service: [ecs-tasks.amazonaws.com]
          Action: ['sts:AssumeRole']
      Path: /
      Policies:
        - PolicyName: AmazonECSTaskExecutionRolePolicy
          PolicyDocument:
            Statement:
            - Effect: Allow
              Action:
                # Allow the ECS Tasks to download images from ECR
                - 'ecr:GetAuthorizationToken'
                - 'ecr:BatchCheckLayerAvailability'
                - 'ecr:GetDownloadUrlForLayer'
                - 'ecr:BatchGetImage'

                # Allow the ECS tasks to upload logs to CloudWatch
                - 'logs:CreateLogStream'
                - 'logs:PutLogEvents'
              Resource: '*'
        - PolicyName: dynamoDBAccessDerbyRW
          PolicyDocument:
            Statement:
            - Effect: Allow
              Action:
                - "dynamodb:PutItem"
                - "dynamodb:GetItem"
              Resource: !Sub
                - arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${TableId}
                - { TableId: !Ref 'DynamoDBTable' }
        - PolicyName: s3AccessDerbyRW
          PolicyDocument:
            Statement:
            - Effect: Allow
              Action:
                - "s3:PutObject"
                - "s3:GetObject"
                - "s3:GetObjectVersionTagging"
                - "s3:GetObjectVersion"
              Resource: !Sub
                - arn:aws:s3:::${BucketId}/*
                - { BucketId: !Ref 'S3Bucket' }
        - PolicyName: sqsAccessDerbyRW
          PolicyDocument:
            Statement:
            - Effect: Allow
              Action:
                - "sqs:DeleteMessage"
                - "sqs:ListQueues"
                - "sqs:GetQueueUrl"
                - "sqs:ReceiveMessage"
                - "sqs:GetQueueAttributes"
              Resource: "*"

# These are the values output by the CloudFormation template. Be careful
# about changing any of them, because of them are exported with specific
# names so that the other task related CF templates can use them.
Outputs:
  ClusterName:
    Description: The name of the ECS cluster
    Value: !Ref 'ECSCluster'
    Export:
      Name: !Sub ${EnvironmentName}:ClusterName
  AutoscalingRole:
    Description: The ARN of the role used for autoscaling
    Value: !GetAtt 'AutoscalingRole.Arn'
    Export:
      Name: !Sub ${EnvironmentName}:AutoscalingRole
  ECSRole:
    Description: The ARN of the ECS role
    Value: !GetAtt 'ECSRole.Arn'
    Export:
      Name: !Sub ${EnvironmentName}:ECSRole
  ECSTaskExecutionRole:
    Description: The ARN of the ECS role
    Value: !GetAtt 'ECSTaskExecutionRole.Arn'
    Export:
      Name: !Sub ${EnvironmentName}:ECSTaskExecutionRole
  VpcId:
    Description: The ID of the VPC that this stack is deployed in
    Value: !Ref 'VPC'
    Export:
      Name: !Sub ${EnvironmentName}:VpcId
  PublicSubnetOne:
    Description: Public subnet one
    Value: !Ref 'PublicSubnetOne'
    Export:
      Name: !Sub ${EnvironmentName}:PublicSubnetOne
  ContainerSecurityGroup:
    Description: A security group used to allow containers to receive traffic
    Value: !Ref 'ContainerSecurityGroup'
    Export:
      Name: !Sub ${EnvironmentName}:ContainerSecurityGroup