ecs_cluster_prefect_agent.yml

# https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
# Larger list of available memory options:
#         options: ['512', '1024', '2048', '4096', '5120', '6144', '7168', '8192', '9216', '10240',
#  '11264', '12288', '13312', '14336', '15360', '16384', '17408', '18432', '19456', '20480', ..., '30720']
AWSTemplateFormatVersion: 2010-09-09

Description: >
  Creates a new AWS VPC and a new ECS Cluster. It then deploys an ECS task definition and ECS service 
  running the Prefect agent in a subnet created within that VPC.
  The service can then be used as execution layer for Prefect flow runs

Parameters:
  cpu:
    Type: String
    Description: Allow Dynamic CPU configuration
    Default: 512
    AllowedValues: [256, 512, 1024, 2048, 4096]
  memory:
    Type: String
    Description: Allow Increasing Memory - from 8192 on requires 4096 CPU and increases in 1024 increments
    Default: 1024
    AllowedValues: [512, 1024, 2048, 4096, 5120, 6144, 7168, 8192 , 9216, 10240]
  cluster:
    Type: String
    Description: Cluster name
    Default: prefect2
  awsaccountid:
    Type: String
    Description: AWS Account ID
  region:
    Type: String
    Description: AWS region name
    Default: us-east-1
  project:
    Type: String
    Description: Project name
    Default: dataflowops
  image:
    Type: String
    Description: Docker image for the service
    Default: prefecthq/prefect:2-python3.9

Resources:
  PrefectFargateCluster:
    Type: AWS::ECS::Cluster
    Properties:
      ClusterName: !Ref cluster

  PrefectLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Ref project
      RetentionInDays: 7

  PrefectVPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: 10.0.0.0/16
      EnableDnsSupport: true
      EnableDnsHostnames: true
  InternetGateway:
    Type: AWS::EC2::InternetGateway
  InternetGatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId: !Ref PrefectVPC
      InternetGatewayId: !Ref InternetGateway
  PublicRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref PrefectVPC
  RouteToGateway:
    Type: AWS::EC2::Route
    DependsOn: InternetGatewayAttachment
    Properties:
      RouteTableId: !Ref PublicRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId: !Ref InternetGateway
  PrefectECSServiceSubnet:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref PrefectVPC
      CidrBlock: 10.0.0.0/16
      AvailabilityZone:
        Fn::Select:
          - 0
          - Fn::GetAZs: { Ref: 'AWS::Region' }
      MapPublicIpOnLaunch: true
  SubnetRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PrefectECSServiceSubnet
      RouteTableId: !Ref PublicRouteTable

  ExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub "${project}_ecs_execution_role"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: ecs-tasks.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: SecretsFromParameterStoreCloudWatch
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - ssm:GetParameters
                  - logs:CreateLogStream
                  - logs:CreateLogGroup
                  - logs:PutLogEvents
                Resource: "*"
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy

  TaskRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub "${project}_ecs_task_role"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: ecs-tasks.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: PrefectS3Storage
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - s3:*
                  - ecs:RegisterTaskDefinition
                  - ecs:DeregisterTaskDefinition
                  - ecs:DescribeTasks
                  - ecs:RunTask
                  - logs:GetLogEvents
                  - ec2:DescribeSubnets
                  - ec2:DescribeVpcs
                Resource: "*"

  PrefectTaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family: !Ref project
      Cpu: !Ref cpu
      Memory: !Ref memory
      NetworkMode: awsvpc
      ExecutionRoleArn: !Ref ExecutionRole
      TaskRoleArn: !Ref TaskRole
      ContainerDefinitions:
        - Name: !Ref project
          Image: !Ref image
          EntryPoint:
            - "bash"
            - "-c"
          StopTimeout: 120
          Environment:
            - Name: PREFECT_LOGGING_LEVEL
              Value: INFO
            - Name: AWS_RETRY_MODE
              Value: adaptive
            - Name: AWS_MAX_ATTEMPTS
              Value: 10
          Command:
            - !Sub "prefect agent start -q ${project}"
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-region: !Ref AWS::Region
              awslogs-group: !Ref PrefectLogGroup
              awslogs-stream-prefix: !Ref project
          Secrets:
            - Name: PREFECT_API_URL
              ValueFrom: !Sub "arn:aws:ssm:${region}:${awsaccountid}:parameter/PREFECT_API_URL"
            - Name: PREFECT_API_KEY
              ValueFrom: !Sub "arn:aws:ssm:${region}:${awsaccountid}:parameter/PREFECT_API_KEY"
      RequiresCompatibilities:
        - FARGATE

  PrefectECSService:
    Type: AWS::ECS::Service
    DependsOn:
      - SubnetRouteTableAssociation
      - RouteToGateway
      - PrefectFargateCluster
    Properties:
      ServiceName: !Ref project
      Cluster: !Ref PrefectFargateCluster
      TaskDefinition: !Ref PrefectTaskDefinition
      DesiredCount: 1
      LaunchType: FARGATE
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          Subnets:
            - !Ref PrefectECSServiceSubnet