turbine-master.template

AWSTemplateFormatVersion: 2010-09-09
Description: >-
  This template creates a Quick Start VPC stack and a Turbine Airflow cluster
  stack. The Quick Start VPC stack creates a Multi-AZ VPC infrastructure with
  two private subnets and managed NAT gateways in the two public subnets. The
  Turbine Airflow cluster stack creates the Airflow supporting resources
  including an RDS instance to host the Airflow metadata database, an SQS queue
  to be used as broker backend, S3 buckets for logs and deployment packages, and
  then creates the Airflow scheduler, webserver and workers nested stacks. The
  nested Airflow services stacks create the Airflow instances in highly
  available auto scaling groups spanning two subnets, plus for the workers stack
  an EFS shared network directory and a custom cloudwatch load metric function
  to guide the auto scaling alarm triggers. **WARNING** This template creates
  AWS resources. You will be billed for the AWS resources used if you create a
  stack from this template. QS(0027)
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: VPC network configuration
        Parameters:
          - VPCCIDR
          - PublicSubnet1CIDR
          - PublicSubnet2CIDR
          - PrivateSubnet1ACIDR
          - PrivateSubnet2ACIDR
          - AllowHTTPAccessCIDR
      - Label:
          default: Turbine Cluster configuration
        Parameters:
          - SchedulerInstanceType
          - WebserverInstanceType
          - WorkerInstanceType
          - MinGroupSize
          - MaxGroupSize
          - ShrinkThreshold
          - GrowthThreshold
      - Label:
          default: Apache Airflow configuration
        Parameters:
          - LoadExampleDags
          - LoadDefaultCons
          - WebServerPort
      - Label:
          default: AWS Quick Start configuration
        Parameters:
          - QSS3BucketName
          - QSS3KeyPrefix
    ParameterLabels:
      VPCCIDR:
        default: VPC CIDR
      PublicSubnet1CIDR:
        default: Public subnet 1 CIDR
      PublicSubnet2CIDR:
        default: Public subnet 2 CIDR
      PrivateSubnet1ACIDR:
        default: Private subnet 1 CIDR
      PrivateSubnet2ACIDR:
        default: Private subnet 2 CIDR
      AllowHTTPAccessCIDR:
        default: Allowed HTTP access CIDR
      SchedulerInstanceType:
        default: Scheduler instance type
      WebserverInstanceType:
        default: Web server instance type
      WorkerInstanceType:
        default: Workers instance type
      MinGroupSize:
        default: Minimum group size
      MaxGroupSize:
        default: Maximum group size
      ShrinkThreshold:
        default: Shrink threshold
      GrowthThreshold:
        default: Growth threshold
      LoadExampleDags:
        default: Load example DAGs
      LoadDefaultCons:
        default: Load default connections
      WebServerPort:
        default: Web server port
      QSS3BucketName:
        default: Quick Start S3 bucket name
      QSS3KeyPrefix:
        default: Quick Start S3 key prefix
Parameters:

  VPCCIDR:
    Description: >-
      CIDR block for the VPC.
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28.
    AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$
    Default: 10.0.0.0/16
    Type: String
  PublicSubnet1CIDR:
    Description: >-
      CIDR block for the public subnet 1 located in Availability Zone 1.
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28.
    AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$
    Default: 10.0.0.0/24
    Type: String
  PublicSubnet2CIDR:
    Description: >-
      CIDR block for the public subnet 1 located in Availability Zone 1.
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28.
    AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$
    Default: 10.0.1.0/24
    Type: String
  PrivateSubnet1ACIDR:
    Description: >-
      CIDR block for private subnet 1 located in Availability Zone 1.
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28.
    AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$
    Default: 10.0.10.0/24
    Type: String
  PrivateSubnet2ACIDR:
    Description: >-
      CIDR block for private subnet 2 located in Availability Zone 2.
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28.
    AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$
    Default: 10.0.11.0/24
    Type: String
  AllowHTTPAccessCIDR:
    Description: >-
      The IPv4 CIDR block to allow HTTP access in the webserver. Using the same
      CIDR for the VPC is a secure default. Using 0.0.0.0/0 allows access from
      everywhere, which is convenient but less secure.
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28.
    AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$
    Default: 10.0.0.0/16
    Type: String

  SchedulerInstanceType:
    Description: >-
      EC2 instance type to use for the scheduler.
    Default: t3.micro
    Type: String
  WebserverInstanceType:
    Description: >-
      EC2 instance type to use for the webserver.
    Default: t3.micro
    Type: String
  WorkerInstanceType:
    Description: >-
      EC2 instance type to use for the workers.
    Default: t3.medium
    Type: String
  MinGroupSize:
    Description: >-
      The minimum number of active worker instances.
    Default: 0
    Type: Number
  MaxGroupSize:
    Description: >-
      The maximum number of active worker instances.
    Default: 10
    Type: Number
  ShrinkThreshold:
    Description: >-
      The threshold for the average queue size from which going equal or below
      will trigger the AutoScaling group to Scale In, deallocating one worker
      instance.
    Default: 0.5
    Type: Number
  GrowthThreshold:
    Description: >-
      The threshold for the average queue size from which going equal or above
      will trigger the AutoScaling group to Scale Out, allocating one worker
      instance.
    Default: 0.9
    Type: Number

  LoadExampleDags:
    Description: >-
      Load the example DAGs distributed with Airflow. Useful if deploying a
      stack for demonstrating a few topologies, operators and scheduling
      strategies.
    AllowedValues:
      - 'False'
      - 'True'
    Default: 'False'
    Type: String
  LoadDefaultCons:
    Description: >-
      Load the default connections initialized by Airflow. Most consider these
      unnecessary, which is why the default is to not load them.
    AllowedValues:
      - 'False'
      - 'True'
    Default: 'False'
    Type: String
  WebServerPort:
    Description: >-
      The port Airflow webserver will be listening.
    ConstraintDescription: >-
      Ports below 1024 can be opened only with root privileges and the airflow
      process does not run as such.
    MinValue: 1024
    MaxValue: 65535
    Default: 8080
    Type: Number

  QSS3BucketName:
    Description: >-
      S3 bucket name for the Quick Start assets. You can specify your own bucket
      providing assets and submodules, if you want to override the Quick Start
      behavior for your specific implementation.
    ConstraintDescription: >-
      Quick Start bucket name can include numbers, lowercase letters, uppercase
      letters, and hyphens (-). It cannot start or end with a hyphen (-).
    AllowedPattern: '^[0-9a-zA-Z]+([0-9a-zA-Z-]*[0-9a-zA-Z])*$'
    Default: turbine-quickstart
    Type: String
  QSS3KeyPrefix:
    Description: >-
      S3 key prefix for the Quick Start assets. You can scpeficy your own
      "directory" providing the stack templates, if you want to override the
      Quick Start behavior for your specific implementation.
    ConstraintDescription: >-
      Quick Start key prefix can include numbers, lowercase letters, uppercase
      letters, hyphens (-), and forward slash (/).
    AllowedPattern: '^[0-9a-zA-Z-/]*$'
    Default: quickstart-turbine-airflow/
    Type: String

Resources:

  VPCStack:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: !Join
        - ''
        - - !Sub https://${QSS3BucketName}.s3.amazonaws.com/
          - !Ref QSS3KeyPrefix
          - submodules/quickstart-aws-vpc/templates/aws-vpc.template
      Parameters:
        VPCCIDR: !Ref VPCCIDR
        PublicSubnet1CIDR: !Ref PublicSubnet1CIDR
        PublicSubnet2CIDR: !Ref PublicSubnet2CIDR
        CreatePrivateSubnets: 'true'
        PrivateSubnet1ACIDR: !Ref PrivateSubnet1ACIDR
        PrivateSubnet2ACIDR: !Ref PrivateSubnet2ACIDR
        NumberOfAZs: '2'
        AvailabilityZones: !Join
          - ','
          - - Fn::Select:
              - 0
              - !GetAZs
            - Fn::Select:
              - 1
              - !GetAZs

  TurbineCluster:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: !Join
        - ''
        - - !Sub https://${QSS3BucketName}.s3.amazonaws.com/
          - !Ref QSS3KeyPrefix
          - templates/turbine-cluster.template
      Parameters:
        VPCID: !GetAtt VPCStack.Outputs.VPCID
        PublicSubnet1ID: !GetAtt VPCStack.Outputs.PublicSubnet1ID
        PublicSubnet2ID: !GetAtt VPCStack.Outputs.PublicSubnet2ID
        PrivateSubnet1AID: !GetAtt VPCStack.Outputs.PrivateSubnet1AID
        PrivateSubnet2AID: !GetAtt VPCStack.Outputs.PrivateSubnet2AID
        AllowHTTPAccessCIDR: !Ref AllowHTTPAccessCIDR
        WebServerPort: !Ref WebServerPort
        SchedulerInstanceType: !Ref SchedulerInstanceType
        WebserverInstanceType: !Ref WebserverInstanceType
        WorkerInstanceType: !Ref WorkerInstanceType
        MinGroupSize: !Ref MinGroupSize
        MaxGroupSize: !Ref MaxGroupSize
        GrowthThreshold: !Ref GrowthThreshold
        ShrinkThreshold: !Ref ShrinkThreshold
        LoadExampleDags: !Ref LoadExampleDags
        LoadDefaultCons: !Ref LoadDefaultCons
        QSS3BucketName: !Ref QSS3BucketName
        QSS3KeyPrefix: !Ref QSS3KeyPrefix

Outputs:
  DeploymentsBucket:
    Value: !GetAtt TurbineCluster.Outputs.DeploymentsBucket
  CodeDeployApplication:
    Value: !GetAtt TurbineCluster.Outputs.CodeDeployApplication
  CodeDeployDeploymentGroup:
    Value: !GetAtt TurbineCluster.Outputs.CodeDeployDeploymentGroup