secure.yaml

Description:  This template deploys a VPC, with a public and private subnet. It deploys an internet gateway,
  with a default route on the public subnet with a NAT gateway, and default routes for them in the private subnet.
Mappings:
  RegionMap:
    ap-northeast-1:
      Master: ami-0349224eed420a3b5
      Agent: ami-0172f5479691feb3d
      Bastion: ami-002f145bfc0915873
    # TODO(DET-4258) Uncomment these when we fully support all P3 regions.
    # ap-northeast-2:
    #   Master: ami-0cd876b15dcf9249d
    #   Agent: ami-03816d7956e5b82d0
    #   Bastion: ami-0e95717ef0e92866b
    # ap-southeast-1:
    #   Master: ami-0e841276b0f027389
    #   Agent: ami-0fefe35c35540fa56
    #   Bastion: ami-0173e2856f6e1048d
    # ap-southeast-2:
    #   Master: ami-0257304d38f0c6e7d
    #   Agent: ami-0965b798500f9b145
    #   Bastion: ami-03646f515d078ec29
    eu-central-1:
      Master: ami-0db63f4dee2d55b7a
      Agent: ami-05d528707fd8c13c0
      Bastion: ami-05ef33ae54898e90c
    eu-west-1:
      Master: ami-08f3064d8481f3782
      Agent: ami-03fa6712e4367843a
      Bastion: ami-06868ad5a3642e4d7
    # eu-west-2:
    #   Master: ami-0b48089553c9d7962
    #   Agent: ami-05e6fe8ae7e579f1f
    #   Bastion: ami-0584b3a74f9c23f6a
    us-east-1:
      Master: ami-0f40c8f97004632f9
      Agent: ami-0c3b569c92293b06f
      Bastion: ami-013da1cc4ae87618c
    us-east-2:
      Master: ami-05692172625678b4e
      Agent: ami-006c6a9ab023ac6a1
      Bastion: ami-0fa9de1596e76fb6c
    us-west-2:
      Master: ami-09456e8683eb4d259
      Agent: ami-016299ceb9e8a109a
      Bastion: ami-0c1ab2d66f996cd4b

Parameters:
  VpcCIDR:
    Description: Please enter the IP range (CIDR notation) for this VPC
    Type: String
    Default: 10.192.0.0/16

  PublicSubnetCIDR:
    Description: Please enter the IP range (CIDR notation) for the public subnet in the first Availability Zone
    Type: String
    Default: 10.192.10.0/24

  PrivateSubnet1CIDR:
    Description: Please enter the IP range (CIDR notation) for the private subnet in the first Availability Zone
    Type: String
    Default: 10.192.20.0/24

  PrivateSubnet2CIDR:
    Description: Please enter the IP range (CIDR notation) for the private subnet in the second Availability Zone
    Type: String
    Default: 10.192.21.0/24

  Keypair:
    Description: Keypair for resources
    Type: AWS::EC2::KeyPair::KeyName

  MasterInstanceType:
    Type: String
    Description: Instance Type of Master
    Default: m5.large

  AgentInstanceType:
    Type: String
    Description: Instance Type of Agent
    Default: p2.8xlarge

  BastionInstanceType:
    Type: String
    Description: Instance Type of Bastion
    Default: m5.large

  InboundCIDRRange:
    Type: String
    Description: Ip range for Inbound
    Default: 0.0.0.0/0

  Version:
    Type: String
    Description: Determined version or commit for master image
    Default: 0.13.5.dev0

  DBPassword:
    Type: String
    Description: Password for database
    NoEcho: true

  MaxIdleAgentPeriod:
    Type: String
    Description: How long before idle agents are shutdown
    Default: 10m

  MaxAgentStartingPeriod:
    Type: String
    Description: How long for agent starting before retrying
    Default: 20m

  MaxDynamicAgents:
    Type: Number
    Description: Maximum number of agents to launch simultaneously
    Default: 5

Resources:
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: !Ref VpcCIDR
      EnableDnsSupport: true
      EnableDnsHostnames: true
      Tags:
        - Key: Name
          Value: !Ref AWS::StackName

  InternetGateway:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
        - Key: Name
          Value: !Ref AWS::StackName

  InternetGatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      InternetGatewayId: !Ref InternetGateway
      VpcId: !Ref VPC

  PublicSubnet:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      AvailabilityZone: !Select [ 0, !GetAZs '' ]
      CidrBlock: !Ref PublicSubnetCIDR
      MapPublicIpOnLaunch: true
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName} Public Subnet (AZ1)

  PrivateSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      AvailabilityZone: !Select [ 0, !GetAZs '' ]
      CidrBlock: !Ref PrivateSubnet1CIDR
      MapPublicIpOnLaunch: true
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName} Public Subnet (AZ1)

  PrivateSubnet2:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      AvailabilityZone: !Select [ 1, !GetAZs  '' ]
      CidrBlock: !Ref PrivateSubnet2CIDR
      MapPublicIpOnLaunch: true
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName} Public Subnet (AZ2)

  NatGatewayEIP:
    Type: AWS::EC2::EIP
    DependsOn: InternetGatewayAttachment
    Properties:
      Domain: vpc

  NatGateway:
    Type: AWS::EC2::NatGateway
    Properties:
      AllocationId: !GetAtt NatGatewayEIP.AllocationId
      SubnetId: !Ref PublicSubnet

  PublicRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName} Public Routes

  DefaultPublicRoute:
    Type: AWS::EC2::Route
    DependsOn: InternetGatewayAttachment
    Properties:
      RouteTableId: !Ref PublicRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId: !Ref InternetGateway

  PublicSubnetRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref PublicRouteTable
      SubnetId: !Ref PublicSubnet

  PrivateRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName} Private Routes (AZ1)

  DefaultPrivateRoute:
    Type: AWS::EC2::Route
    Properties:
      RouteTableId: !Ref PrivateRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      NatGatewayId: !Ref NatGateway

  PrivateSubnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref PrivateRouteTable
      SubnetId: !Ref PrivateSubnet1

  PrivateSubnet2RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      RouteTableId: !Ref PrivateRouteTable
      SubnetId: !Ref PrivateSubnet2

  S3Endpoint:
    Type: AWS::EC2::VPCEndpoint
    Properties:
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal: "*"
            Action: "*"
            Resource: "*"
      RouteTableIds:
        - !Ref PrivateRouteTable
        - !Ref PublicRouteTable
      ServiceName: !Sub com.amazonaws.${AWS::Region}.s3
      VpcId: !Ref VPC

  BastionSecurityGroupSSH:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !GetAtt BastionSecurityGroup.GroupId
      IpProtocol: tcp
      FromPort: 22
      ToPort: 22
      CidrIp: !Ref InboundCIDRRange

  BastionSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security Group For Bastion
      VpcId: !Ref VPC
      Tags:
        - Key: user
          Value: !Ref AWS::StackName

  MasterSecurityGroupEgress:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId: !GetAtt MasterSecurityGroup.GroupId
      DestinationSecurityGroupId: !GetAtt AgentSecurityGroup.GroupId
      FromPort: 0
      ToPort: 65535
      IpProtocol: tcp

  MasterSecurityGroupInternet:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId: !GetAtt MasterSecurityGroup.GroupId
      CidrIp: 0.0.0.0/0
      FromPort: 0
      ToPort: 65535
      IpProtocol: tcp

  MasterSecurityGroupIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !GetAtt MasterSecurityGroup.GroupId
      FromPort: 8080
      ToPort: 8080
      IpProtocol: tcp
      SourceSecurityGroupId: !GetAtt AgentSecurityGroup.GroupId

  MasterSecurityGroupIngressUI:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !GetAtt MasterSecurityGroup.GroupId
      FromPort: 8080
      ToPort: 8080
      IpProtocol: tcp
      SourceSecurityGroupId: !GetAtt BastionSecurityGroup.GroupId

  MasterSSHIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !GetAtt MasterSecurityGroup.GroupId
      IpProtocol: tcp
      FromPort: 22
      ToPort: 22
      SourceSecurityGroupId: !GetAtt BastionSecurityGroup.GroupId

  AgentSecurityGroupEgress:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId: !GetAtt AgentSecurityGroup.GroupId
      CidrIp: 0.0.0.0/0
      FromPort: 0
      ToPort: 65535
      IpProtocol: tcp

  AgentSecurityGroupIngressMaster:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !GetAtt AgentSecurityGroup.GroupId
      FromPort: 0
      ToPort: 65535
      IpProtocol: tcp
      SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId

  AgentSecurityGroupIngressAgent:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !GetAtt AgentSecurityGroup.GroupId
      FromPort: 0
      ToPort: 65535
      IpProtocol: tcp
      SourceSecurityGroupId: !GetAtt AgentSecurityGroup.GroupId

  AgentSSHIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !GetAtt AgentSecurityGroup.GroupId
      IpProtocol: tcp
      FromPort: 22
      ToPort: 22
      SourceSecurityGroupId: !GetAtt BastionSecurityGroup.GroupId

  DatabaseEgress:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId: !GetAtt DatabaseSecurityGroup.GroupId
      CidrIp: 0.0.0.0/0
      FromPort: 0
      ToPort: 65535
      IpProtocol: -1

  DatabaseIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !GetAtt DatabaseSecurityGroup.GroupId
      IpProtocol: tcp
      FromPort: 5432
      ToPort: 5432
      SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId

  MasterSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security Group For Master
      VpcId: !Ref VPC
      Tags:
        - Key: user
          Value: !Ref AWS::StackName

  AgentSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security Group For Agent
      VpcId: !Ref VPC
      Tags:
        - Key: user
          Value: !Ref AWS::StackName

  DatabaseSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security Group For Database
      VpcId: !Ref VPC
      Tags:
        - Key: user
          Value: !Ref AWS::StackName

  DatabaseSubnetGroup:
    Type: AWS::RDS::DBSubnetGroup
    Properties:
      DBSubnetGroupDescription: determined-db-subnet-group
      SubnetIds:
        - !Ref PrivateSubnet1
        - !Ref PrivateSubnet2
      Tags:
        - Key: user
          Value: !Ref AWS::StackName

  Database:
    Type: AWS::RDS::DBCluster
    DeletionPolicy: Delete
    Properties:
      Engine: aurora-postgresql
      EngineMode: serverless
      DatabaseName: determined
      DBClusterParameterGroupName: 'default.aurora-postgresql10'
      DBSubnetGroupName: !Ref DatabaseSubnetGroup
      MasterUsername: postgres
      MasterUserPassword: !Ref DBPassword
      Tags:
        - Key: user
          Value: !Ref AWS::StackName
      VpcSecurityGroupIds:
        - !GetAtt DatabaseSecurityGroup.GroupId
      ScalingConfiguration:
        AutoPause: false
        MinCapacity: 8
        MaxCapacity: 32

  LogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub /determined/det-${AWS::StackName}

  LogPolicy:
    Type: AWS::IAM::Policy
    Properties:
      Roles:
        - !Ref AgentRole
        - !Ref MasterRole
      PolicyName: agent-log-policy
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - logs:CreateLogStream
              - logs:PutLogEvents
              - logs:DescribeLogStreams
            Resource:
              - !Sub arn:aws:logs:*:*:log-group:${LogGroup},
              - !Sub arn:aws:logs:*:*:log-group:${LogGroup}:log-stream:*

  MetricPolicy:
    Type: AWS::IAM::Policy
    Properties:
      Roles:
        - !Ref AgentRole
        - !Ref MasterRole
      PolicyName: agent-metric-policy
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Action:
              - cloudwatch:PutMetricData
            Effect: Allow
            Resource: "*"

  BastionEC2Instance:
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !FindInMap
        - RegionMap
        - !Ref AWS::Region
        - Bastion
      InstanceType: !Ref BastionInstanceType
      KeyName: !Ref Keypair
      SubnetId: !Ref PublicSubnet
      SecurityGroupIds:
        - !Ref BastionSecurityGroup
      Tags:
        - Key: user
          Value: !Ref AWS::StackName

  AgentRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: agent-s3-policy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action: "s3:*"
                Resource: "*"
        - PolicyName: determined-ec2
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - ec2:DescribeInstances
                Resource: "*"

  AgentInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref AgentRole

  MasterRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: determined-agent-policy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - ec2:DescribeInstances
                  - ec2:TerminateInstances
                  - ec2:CreateTags
                  - ec2:RunInstances
                Resource: "*"
        - PolicyName: pass-role
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action: iam:PassRole
                Resource: !GetAtt AgentRole.Arn
      Tags:
        - Key: user
          Value: !Ref AWS::StackName

  MasterInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref MasterRole

  MasterInstance:
    Type: AWS::EC2::Instance
    Properties:
      SubnetId: !Ref PrivateSubnet1
      InstanceType: !Ref MasterInstanceType
      ImageId: !FindInMap
        - RegionMap
        - !Ref AWS::Region
        - Master
      KeyName: !Ref Keypair
      IamInstanceProfile: !Ref MasterInstanceProfile
      SecurityGroupIds:
        - !Ref MasterSecurityGroup
      BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
            VolumeSize: 200
            VolumeType: gp2
      UserData:
        Fn::Base64: !Sub
          - |
            Content-Type: multipart/mixed; boundary="//"
            MIME-Version: 1.0

            --//
            Content-Type: text/cloud-config; charset="us-ascii"
            MIME-Version: 1.0
            Content-Transfer-Encoding: 7bit
            Content-Disposition: attachment; filename="cloud-config.txt"

            #cloud-config
            cloud_final_modules:
            - [scripts-user, always]

            --//
            Content-Type: text/x-shellscript; charset="us-ascii"
            MIME-Version: 1.0
            Content-Transfer-Encoding: 7bit
            Content-Disposition: attachment; filename="userdata.txt"

            #!/bin/bash
            mkdir -p /usr/local/determined/etc
            cat << EOF > /usr/local/determined/etc/master.yaml
            checkpoint_storage:
              type: s3
              bucket: ${CheckpointBucket}
              save_experiment_best: 0
              save_trial_best: 1
              save_trial_latest: 1

            db:
              user: postgres
              password: "${DBPassword}"
              host: "${Database.Endpoint.Address}"
              port: 5432
              name: determined
              ssl_mode: verify-ca
              ssl_root_cert: /etc/determined/etc/db_ssl_root_cert.pem

            provisioner:
              iam_instance_profile_arn: ${AgentInstanceProfile.Arn}
              image_id: ${AgentAmi}
              agent_docker_image: determinedai/determined-agent:${Version}
              instance_name: determined-agent-${AWS::StackName}
              instance_type: ${AgentInstanceType}
              log_group: ${LogGroup}
              log_stream: determined-agent
              master_url: http://local-ipv4:8080
              max_idle_agent_period: ${MaxIdleAgentPeriod}
              max_agent_starting_period: ${MaxAgentStartingPeriod}
              max_instances: ${MaxDynamicAgents}
              network_interface:
                public_ip: false
                security_group_id: ${AgentSecurityGroup.GroupId}
                subnet_id: ${PrivateSubnet1}
              provider: aws
              root_volume_size: 200
              ssh_key_name: ${Keypair}
              tag_key: det-${AWS::StackName}
              tag_value: det-agent-${AWS::StackName}
            EOF

            apt-get remove docker docker-engine docker.io containerd runc
            apt-get update
            apt-get install -y \
                apt-transport-https \
                ca-certificates \
                curl \
                gnupg-agent \
                software-properties-common
            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
            add-apt-repository \
               "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
               $(lsb_release -cs) \
               stable"
            apt-get update
            apt-get install -y docker-ce docker-ce-cli containerd.io

            curl -fsSL https://www.amazontrust.com/repository/AmazonRootCA1.pem > \
              /usr/local/determined/etc/AmazonRootCA1.pem

            docker network create determined

            docker stop $(docker ps -a -q)
            docker rm $(docker ps -a -q)

            docker run \
                --name determined-master \
                --network determined \
                --restart unless-stopped \
                --log-driver=awslogs \
                --log-opt awslogs-group=${LogGroup} \
                --log-opt awslogs-stream=determined-master \
                -p 8080:8080 \
                -v /usr/local/determined/etc/master.yaml:/etc/determined/master.yaml \
                -v /usr/local/determined/etc/AmazonRootCA1.pem:/etc/determined/etc/db_ssl_root_cert.pem \
                determinedai/determined-master:${Version}
            --//

          - AgentAmi: !FindInMap [RegionMap, !Ref "AWS::Region", Agent]
      Tags:
        - Key: user
          Value: !Ref AWS::StackName
        - Key: Name
          Value: !Sub det-master-${AWS::StackName}

  CheckpointBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: !Sub det-${AWS::StackName}-${AWS::Region}-${AWS::AccountId}

Outputs:
  VPC:
    Description: A reference to the created VPC
    Value: !Ref VPC

  PublicSubnetId:
    Description: A list of the public subnets
    Value: !Ref PublicSubnet

  PrivateSubnetId:
    Description: A list of the private subnets
    Value: !Ref PrivateSubnet1

  BastionId:
    Description: Id of Bastion
    Value: !Ref BastionEC2Instance

  MasterId:
    Description: Id of Master Agent
    Value: !Ref MasterInstance

  MasterSecurityGroupId:
    Description: Id of Master Security Group
    Value: !GetAtt MasterSecurityGroup.GroupId

  AgentSecurityGroupId:
    Description: Id of Agent Security Group
    Value: !GetAtt AgentSecurityGroup.GroupId

  AgentInstanceProfile:
    Description: Instance Profile for Agent
    Value: !GetAtt AgentInstanceProfile.Arn

  CheckpointBucket:
    Description: S3 Bucket for checkpoints
    Value: !Ref CheckpointBucket

  Version:
    Description: Determined version or commit
    Value: !Ref Version

  AgentTagName:
    Description: Tag Name for the agents
    Value: !Sub determined-agent-${AWS::StackName}

  LogGroup:
    Description: The Log Group for Determined Logs
    Value: !Ref LogGroup

  Region:
    Description: The AWS Region the stack is deployed in
    Value: !Ref AWS::Region