kubernetes-cluster.template

# Copyright 2017 by the contributors
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

---
AWSTemplateFormatVersion: '2010-09-09'
Description: 'QS(5042) Kubernetes AWS CloudFormation Template: Create a Kubernetes
  cluster in an existing VPC. This template is for users who want to add
  a Kubernetes cluster to existing AWS infrastructure. The master node is
  an auto-recovering Amazon EC2 instance. 1-20 additional EC2 instances in an
  AutoScalingGroup join the Kubernetes cluster as nodes. An ELB provides
  configurable external access to the Kubernetes API. If you choose a private
  subnet, make sure it has a bastion host for SSH access to your cluster. If you
  choose a public subnet, you can connect directly to the master node. The stack
  is suitable for development and small single-team clusters. **WARNING** This
  template creates four Amazon EC2 instances with default settings. You will
  be billed for the AWS resources used if you create a stack from this template.
  **SUPPORT** Please visit http://jump.heptio.com/aws-qs-help for support.
  **NEXT STEPS** Please visit http://jump.heptio.com/aws-qs-next.'

# The Metadata tells AWS how to display the parameters during stack creation
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
    - Label:
        default: Amazon EC2 Configuration
      Parameters:
      - VPCID
      - AvailabilityZone
      - InstanceType
      - DiskSizeGb
      - ClusterSubnetId
      - LoadBalancerSubnetId
    - Label:
        default: Access Configuration
      Parameters:
      - SSHLocation
      - ApiLbLocation
      - KeyName
    - Label:
        default: Kubernetes Configuration
      Parameters:
      - K8sNodeCapacity
      - NetworkingProvider
    - Label:
        default: Advanced
      Parameters:
      - QSS3BucketName
      - QSS3KeyPrefix
      - ClusterAssociation

    ParameterLabels:
      KeyName:
        default: SSH Key
      VPCID:
        default: VPC
      AvailabilityZone:
        default: Availability Zone
      ClusterSubnetId:
        default: Subnet
      SSHLocation:
        default: SSH Ingress Location
      ApiLbLocation:
        default: API Ingress Location
      InstanceType:
        default: Instance Type
      DiskSizeGb:
        default: Disk Size (GiB)
      K8sNodeCapacity:
        default: Node Capacity
      QSS3BucketName:
        default: S3 Bucket
      QSS3KeyPrefix:
        default: S3 Key Prefix
      ClusterAssociation:
        default: Cluster Association
      NetworkingProvider:
        default: Networking Provider
      LoadBalancerSubnetId:
        default: Load Balancer Subnet


# The Parameters allow the user to pass custom settings to the stack before creation
Parameters:
  # Required. Calls for the name of an existing EC2 KeyPair, to enable SSH access to the instances
  # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html
  KeyName:
    Description: Existing EC2 KeyPair for SSH access.
    Type: AWS::EC2::KeyPair::KeyName
    ConstraintDescription: must be the name of an existing EC2 KeyPair.

  VPCID:
    Description: Existing VPC to use for this cluster.
    Type: AWS::EC2::VPC::Id

  ClusterSubnetId:
    Description: Existing subnet to use for this cluster. Must belong to the Availability Zone above.
    Type: AWS::EC2::Subnet::Id

  LoadBalancerSubnetId:
    Description: Existing subnet to use for load balancing HTTPS access to the Kubernetes API server.
      Must be a public subnet. Must belong to the Availability Zone above.
    Type: AWS::EC2::Subnet::Id

  ClusterAssociation:
    Description: Enter a string, unique within your AWS account, to associate resources in
     this Kubernetes cluster with each other. This adds a tag, with the key KubernetesCluster
     and the value of this parameter, to resources created as part of this stack. Leave blank
     to use this Quick Start Stack name.
    Type: String

  # Default is t2.medium. EC2 instance type for the cluster
  # https://aws.amazon.com/ec2/instance-types/
  InstanceType:
    Description: EC2 instance type for the cluster.
    Type: String
    Default: t2.medium
    AllowedValues:
    - t2.nano
    - t2.micro
    - t2.small
    - t2.medium
    - t2.large
    - m3.medium
    - m3.large
    - m3.xlarge
    - m3.2xlarge
    - m4.large
    - m4.xlarge
    - m4.2xlarge
    - m4.4xlarge
    - m4.10xlarge
    - c3.large
    - c3.xlarge
    - c3.2xlarge
    - c3.4xlarge
    - c3.8xlarge
    - c4.large
    - c4.xlarge
    - c4.2xlarge
    - c4.4xlarge
    - c4.8xlarge
    - g2.2xlarge
    - g2.8xlarge
    - r3.large
    - r3.xlarge
    - r3.2xlarge
    - r3.4xlarge
    - r3.8xlarge
    - i2.xlarge
    - i2.2xlarge
    - i2.4xlarge
    - i2.8xlarge
    - d2.xlarge
    - d2.2xlarge
    - d2.4xlarge
    - d2.8xlarge
    ConstraintDescription: must be a valid EC2 instance type.

  # Specifies the size of the root disk for all EC2 instances, including master
  # and nodes.
  DiskSizeGb:
    Description: 'Size of the root disk for the EC2 instances, in GiB.  Default: 40'
    Default: 40
    Type: Number
    MinValue: 8
    MaxValue: 1024

  # Required. This is an availability zone from your region
  # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
  AvailabilityZone:
    Description: The Availability Zone for this cluster. Heptio recommends
      that you run one cluster per AZ and use tooling to coordinate across AZs.
    Type: AWS::EC2::AvailabilityZone::Name
    ConstraintDescription: must be the name of an AWS Availability Zone

  # Specifies the IP range from which you will have SSH access over port 22
  # Used in the allow22 SecurityGroup
  SSHLocation:
    Description: CIDR block (IP address range) to allow SSH access to the
      instances. Use 0.0.0.0/0 to allow access from all locations.
    Type: String
    MinLength: '9'
    MaxLength: '18'
    AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
    ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.

  # Specifies the IP range from which you will have HTTPS access to the Kubernetes API server load balancer
  # Used in the ApiLoadBalancerSecGroup SecurityGroup
  ApiLbLocation:
    Description: CIDR block (IP address range) to allow HTTPS access to
      the Kubernetes API. Use 0.0.0.0/0 to allow access from all locations.
    Type: String
    MinLength: '9'
    MaxLength: '18'
    AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
    ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.

  # Default 2. Choose 1-20 initial nodes to run cluster workloads (in addition to the master node instance)
  # You can scale up your cluster later and add more nodes
  K8sNodeCapacity:
    Default: '2'
    Description: Initial number of Kubernetes nodes (1-20).
    Type: Number
    MinValue: '1'
    MaxValue: '20'
    ConstraintDescription: must be between 1 and 20 EC2 instances.

  # S3 Bucket configuration: allows users to use their own downstream snapshots
  # of the quickstart-aws-vpc and quickstart-linux-bastion templates
  QSS3BucketName:
    AllowedPattern: "^[0-9a-zA-Z]+([0-9a-zA-Z-]*[0-9a-zA-Z])*$"
    ConstraintDescription: Quick Start bucket name can include numbers, lowercase
      letters, uppercase letters, and hyphens (-). It cannot start or end with a hyphen
      (-).

    Default: quickstart-reference
    Description: Only change this if you have set up assets, like your own networking
      configuration, in an S3 bucket. This and the S3 Key Prefix parameter let you access
      scripts from the scripts/ and templates/ directories of your own fork of the Heptio
      Quick Start assets, uploaded to S3 and stored at
      ${bucketname}.s3.amazonaws.com/${prefix}/scripts/somefile.txt.S3. The bucket name
      can include numbers, lowercase letters, uppercase letters, and hyphens (-).
      It cannot start or end with a hyphen (-).
    Type: String

  QSS3KeyPrefix:
    AllowedPattern: "^[0-9a-zA-Z-]+(/[0-9a-zA-Z-]+)*$"
    ConstraintDescription: Quick Start key prefix can include numbers, lowercase letters,
      uppercase letters, hyphens (-), and forward slash (/). It cannot start or end
      with forward slash (/) because they are automatically appended.
    Default: heptio/latest
    Description: Only change this if you have set up assets in an S3 bucket, as explained
      in the S3 Bucket parameter. The S3 key prefix can include numbers, lowercase letters,
      uppercase letters, hyphens (-), and forward slashes (/). It cannot start or end with
      forward slashes (/) because they are automatically appended.
    Type: String

  NetworkingProvider:
    AllowedValues:
    - calico
    - weave
    ConstraintDescription: 'Currently supported values are "calico" and "weave"'
    Default: calico
    Description: Choose the networking provider to use for communication between
      pods in the Kubernetes cluster. Supported configurations are calico
      (http://docs.projectcalico.org/v2.0/getting-started/kubernetes/installation/hosted/kubeadm/)
      and weave (https://github.com/weaveworks/weave/blob/master/site/kube-addon.md).
    Type: String

# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/mappings-section-structure.html
Mappings:

  # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
  RegionMap:
    us-west-1:
      '64': ami-28300048
    ap-south-1:
      '64': ami-28773647
    eu-west-2:
      '64': ami-57594b33
    eu-west-1:
      '64': ami-8331fafa
    ap-northeast-2:
      '64': ami-11c3197f
    ap-northeast-1:
      '64': ami-d4eb3fb2
    sa-east-1:
      '64': ami-0184f86d
    ca-central-1:
      '64': ami-f363da97
    ap-southeast-1:
      '64': ami-4c5d2f2f
    ap-southeast-2:
      '64': ami-fd84649f
    eu-central-1:
      '64': ami-2a972445
    us-east-1:
      '64': ami-312ddf4b
    us-east-2:
      '64': ami-ce270aab
    us-west-2:
      '64': ami-9143bae9

# Helper Conditions which help find the right values for resources
Conditions:
  AssociationProvidedCondition:
    Fn::Not:
    - Fn::Equals:
      - !Ref ClusterAssociation
      - ''
  LoadBalancerSubnetProvidedCondition:
    Fn::Not:
    - Fn::Equals:
      - !Ref LoadBalancerSubnetId
      - ''

# Resources are the AWS services we want to actually create as part of the Stack
Resources:
  # Install a CloudWatch logging group for system logs for each instance
  KubernetesLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Ref AWS::StackName
      RetentionInDays: 14

  # This is an EC2 instance that will serve as our master node
  K8sMasterInstance:
    Type: AWS::EC2::Instance
    DependsOn: ApiLoadBalancer
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          master-setup: master-setup
        master-setup:
          files:
            # Script that will allow for development kubernetes binaries to replace the pre-packaged AMI binaries.
            "/tmp/kubernetes-override-binaries.sh":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/kubernetes-override-binaries.sh.in"
              mode: '000755'
              context:
                BaseBinaryUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/bin/"

            # Configuration file for the cloudwatch agent. The file is a Mustache template, and we're creating it with
            # the below context (mainly to substitute in the AWS Stack Name for the logging group.)
            "/tmp/kubernetes-awslogs.conf":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/kubernetes-awslogs.conf"
              context:
                StackName: !Ref AWS::StackName

            # Installation script for the Cloudwatch agent
            "/usr/local/aws/awslogs-agent-setup.py":
              source: https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py
              mode: '000755'

            # systemd init script for the Cloudwatch logs agent
            "/etc/systemd/system/awslogs.service":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/awslogs.service"

            # Setup script for initializing the Kubernetes master instance.  This is where most of the cluster
            # initialization happens.  See scripts/setup-k8s-master.sh in the Quick Start repo for details.
            "/tmp/setup-k8s-master.sh":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/setup-k8s-master.sh.in"
              mode: '000755'
              context:
                LoadBalancerDns: !GetAtt ApiLoadBalancer.DNSName
                LoadBalancerName: !Ref ApiLoadBalancer
                ClusterToken: !GetAtt KubeadmToken.Token
                NetworkingProviderUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/${NetworkingProvider}.yaml"
                DashboardUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/dashboard.yaml"
                StorageClassUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/default.storageclass.yaml"
                Region: !Ref AWS::Region

          commands:
            # Override the AMI binaries with any kubelet/kubeadm/kubectl binaries in the S3 bucket
            "00-kubernetes-override-binaries":
              command: "/tmp/kubernetes-override-binaries.sh"
            # Install the Cloudwatch agent with configuration for the current region and log group name
            "01-cloudwatch-agent-setup":
              command: !Sub "python /usr/local/aws/awslogs-agent-setup.py -n -r ${AWS::Region} -c /tmp/kubernetes-awslogs.conf"
            # Enable the Cloudwatch service and launch it
            "02-cloudwatch-service-config":
              command: "systemctl enable awslogs.service && systemctl start awslogs.service"

            # Run the master setup
            "03-master-setup":
              command: "/tmp/setup-k8s-master.sh"
    Properties:
      # Where the EC2 instance gets deployed geographically
      AvailabilityZone: !Ref AvailabilityZone
      # Refers to the MasterInstanceProfile resource, which applies the IAM role for the master instance
      # The IAM role allows us to create further AWS resources (like an EBS drive) from the cluster
      # This is needed for the Kubernetes-AWS cloud-provider integration
      IamInstanceProfile: !Ref MasterInstanceProfile
      # Type of instance; the default is t2.medium
      InstanceType: !Ref InstanceType
      # Adds our SSH key to the instance
      KeyName: !Ref KeyName
      NetworkInterfaces:
      - DeleteOnTermination: true
        DeviceIndex: 0
        SubnetId: !Ref ClusterSubnetId
        # Joins the ClusterSecGroup Security Group for cluster communication and SSH access
        # The ClusterSecGroupCrossTalk rules allow all instances in the same stack to communicate internally
        # The ClusterSecGroupAllow22 rules allow external communication on port 22 from a chosen CIDR range
        # The ClusterSecGroupAllow6443FromLB rules allow HTTPS access to the load balancer on port 6443
        GroupSet:
        - !Ref ClusterSecGroup
      # Designates a name for this EC2 instance that will appear in the instances list (k8s-master)
      # Tags it with KubernetesCluster=<stackname> or chosen value (needed for cloud-provider's IAM roles)
      Tags:
      - Key: Name
        Value: k8s-master
      - Key: KubernetesCluster
        Value:
          Fn::If:
          - AssociationProvidedCondition
          - !Ref ClusterAssociation
          - !Ref AWS::StackName
        # Also tag it with kubernetes.io/cluster/clustername=owned, which is the newer convention for cluster resources
      - Key:
          Fn::Sub:
          - "kubernetes.io/cluster/${ClusterID}"
          - ClusterID:
              Fn::If:
              - AssociationProvidedCondition
              - !Ref ClusterAssociation
              - !Ref AWS::StackName
        Value: 'owned'
      # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-instance.html#cfn-ec2-instance-imageid
      ImageId:
        Fn::FindInMap:
        - RegionMap
        - !Ref AWS::Region
        - '64'
      BlockDeviceMappings:
      - DeviceName: '/dev/sda1'
        Ebs:
          VolumeSize: !Ref DiskSizeGb
          VolumeType: gp2
      # The userdata script is launched on startup, but contains only the commands that call out to cfn-init, which runs
      # the commands in the metadata above, and cfn-signal, which signals when the initialization is complete.
      UserData:
        Fn::Base64:
          Fn::Sub: |
            #!/bin/bash
            set -o xtrace

            /usr/local/bin/cfn-init \
              --verbose \
              --stack '${AWS::StackName}' \
              --region '${AWS::Region}' \
              --resource K8sMasterInstance \
              --configsets master-setup

            /usr/local/bin/cfn-signal \
              --exit-code $? \
              --stack '${AWS::StackName}' \
              --region '${AWS::Region}' \
              --resource K8sMasterInstance
    CreationPolicy:
      ResourceSignal:
        Timeout: PT10M

  # IAM role for Lambda function for generating kubeadm token
  LambdaExecutionRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
        - Effect: "Allow"
          Principal:
            Service: ["lambda.amazonaws.com"]
          Action: "sts:AssumeRole"
      Path: "/"
      Policies:
      - PolicyName: "lambda_policy"
        PolicyDocument:
          Version: "2012-10-17"
          Statement:
          - Effect: "Allow"
            Action:
            - "logs:CreateLogGroup"
            - "logs:CreateLogStream"
            - "logs:PutLogEvents"
            Resource: "arn:aws:logs:*:*:*"

  # Lambda Function for generating the kubeadm token
  GenKubeadmToken:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        ZipFile: |
          import random
          import string
          import cfnresponse
          def id_generator(size, chars=string.ascii_lowercase + string.digits):
            return ''.join(random.choice(chars) for _ in range(size))
          def handler(event, context):
            if event['RequestType'] == 'Delete':
              cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
            if event['RequestType'] == 'Create':
              token = ("%s.%s" % (id_generator(6), id_generator(16)))
              responseData = {}
              responseData['Token'] = token
              cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
              return token
      Handler: "index.handler"
      Runtime: "python2.7"
      Timeout: "5"
      Role: !GetAtt LambdaExecutionRole.Arn

  # A Custom resource that uses the lambda function to generate our cluster token
  KubeadmToken:
    Type: "Custom::GenerateToken"
    Version: "1.0"
    Properties:
      ServiceToken: !GetAtt GenKubeadmToken.Arn

  # This is a CloudWatch alarm https://aws.amazon.com/cloudwatch/
  # If the master node is unresponsive for 5 minutes, AWS will attempt to recover it
  # It will preserve the original IP, which is important for Kubernetes networking
  # Based on http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/quickref-cloudwatch.html#cloudwatch-sample-recover-instance
  RecoveryTestAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmDescription: Trigger a recovery when instance status check fails for 5
        consecutive minutes.
      Namespace: AWS/EC2
      MetricName: StatusCheckFailed_System
      Statistic: Minimum
      # 60-second periods (1 minute)
      Period: '60'
      # 5-minute check-ins
      EvaluationPeriods: '5'
      ComparisonOperator: GreaterThanThreshold
      Threshold: '0'
      # This is the call that actually tries to recover the instance
      AlarmActions:
        - !Sub "arn:aws:automate:${AWS::Region}:ec2:recover"
      # Applies this alarm to our K8sMasterInstance
      Dimensions:
      - Name: InstanceId
        Value: !Ref K8sMasterInstance

  # This is the Auto Scaling Group that contains EC2 instances that are Kubernetes nodes
  # http://docs.aws.amazon.com/autoscaling/latest/userguide/AutoScalingGroup.html
  K8sNodeGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    DependsOn: K8sMasterInstance
    CreationPolicy:
      ResourceSignal:
        # Ensure at least <K8sNodeCapacity> nodes have signaled success before
        # this resource is considered created.
        Count: !Ref K8sNodeCapacity
        Timeout: PT10M
    Properties:
      # Where the EC2 instance gets deployed geographically
      AvailabilityZones:
      - !Ref AvailabilityZone
      # Refers to the K8sNodeCapacity parameter, which specifies the number of nodes (1-20)
      DesiredCapacity: !Ref K8sNodeCapacity
      # Refers to the LaunchConfig, which has specific config details for the EC2 instances
      LaunchConfigurationName: !Ref LaunchConfig
      # More cluster sizing
      MinSize: '1'
      MaxSize: '20'
      # VPC Zone Identifier is the subnets to put the hosts in
      VPCZoneIdentifier:
        - !Ref ClusterSubnetId
      # Designates names for these EC2 instances that will appear in the instances list (k8s-node)
      # Tags each node with KubernetesCluster=<stackname> or chosen value (needed for cloud-provider's IAM roles)
      Tags:
      - Key: Name
        Value: k8s-node
        PropagateAtLaunch: 'true'
      - Key: KubernetesCluster
        Value:
          Fn::If:
          - AssociationProvidedCondition
          - !Ref ClusterAssociation
          - !Ref AWS::StackName
        PropagateAtLaunch: 'true'
        # Also tag it with kubernetes.io/cluster/clustername=owned, which is the newer convention for cluster resources
      - Key:
          Fn::Sub:
          - "kubernetes.io/cluster/${ClusterID}"
          - ClusterID:
              Fn::If:
              - AssociationProvidedCondition
              - !Ref ClusterAssociation
              - !Ref AWS::StackName
        Value: 'owned'
        PropagateAtLaunch: 'true'
    # Tells the group how many instances to update at a time, if an update is applied
    UpdatePolicy:
      AutoScalingRollingUpdate:
        MinInstancesInService: '1'
        MaxBatchSize: '1'

  # This tells AWS what kinds of servers we want in our Auto Scaling Group
  LaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          node-setup: node-setup
        node-setup:
          # (See comments in the master instance Metadata for details.)
          files:
            "/tmp/kubernetes-override-binaries.sh":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/kubernetes-override-binaries.sh.in"
              mode: '000755'
              context:
                BaseBinaryUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/bin/"
            "/tmp/kubernetes-awslogs.conf":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/kubernetes-awslogs.conf"
              context:
                StackName: !Ref AWS::StackName
            "/usr/local/aws/awslogs-agent-setup.py":
              source: https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py
              mode: '000755'
            "/etc/systemd/system/awslogs.service":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/awslogs.service"
          commands:
            "00-kubernetes-override-binaries":
              command: "/tmp/kubernetes-override-binaries.sh"
            "01-cloudwatch-agent-setup":
              command: !Sub "python /usr/local/aws/awslogs-agent-setup.py -n -r ${AWS::Region} -c /tmp/kubernetes-awslogs.conf"
            "02-cloudwatch-service-config":
              command: "systemctl enable awslogs.service && systemctl start awslogs.service"
            # This joins the existing cluster with kubeadm.  Kubeadm is preinstalled in the AMI this template uses, so
            # all that's left is to join the cluster with the correct token.
            "03-k8s-setup-node":
              command: !Sub "kubeadm reset && kubeadm join --node-name=\"$(hostname -f)\" --token=${KubeadmToken.Token} ${K8sMasterInstance.PrivateIp}:6443"
    Properties:
      # Refers to the NodeInstanceProfile resource, which applies the IAM role for the nodes
      # The IAM role allows us to create further AWS resources (like an EBS drive) from the cluster
      # This is needed for the Kubernetes-AWS cloud-provider integration
      IamInstanceProfile: !Ref NodeInstanceProfile
      # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-instance.html#cfn-ec2-instance-imageid
      ImageId:
        Fn::FindInMap:
        - RegionMap
        - !Ref AWS::Region
        - '64'
      BlockDeviceMappings:
      - DeviceName: '/dev/sda1'
        Ebs:
          VolumeSize: !Ref DiskSizeGb
          VolumeType: gp2
      # Type of instance; the default is t2.medium
      InstanceType: !Ref InstanceType
      # Adds our SSH key to the instance
      KeyName: !Ref KeyName
      # Join the cluster security group so that we can customize the access
      # control (See the ClusterSecGroup resource for details)
      SecurityGroups:
      - !Ref ClusterSecGroup
      # The userdata script is launched on startup, but contains only the commands that call out to cfn-init, which runs
      # the commands in the metadata above, and cfn-signal, which signals when the initialization is complete.
      UserData:
        Fn::Base64:
          Fn::Sub: |
            #!/bin/bash
            set -o xtrace

            /usr/local/bin/cfn-init \
              --verbose \
              --stack '${AWS::StackName}' \
              --region '${AWS::Region}' \
              --resource LaunchConfig \
              --configsets node-setup

            /usr/local/bin/cfn-signal \
              --exit-code $? \
              --stack '${AWS::StackName}' \
              --region '${AWS::Region}' \
              --resource K8sNodeGroup

  # Define the (one) security group for all machines in the cluster.  Keeping
  # just one security group helps with k8s's cloud-provider=aws integration so
  # that it knows what security group to manage.
  ClusterSecGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for all machines in the cluster
      VpcId: !Ref VPCID
      # Security Groups must be tagged with KubernetesCluster=<cluster> so that
      # they can coexist in the same VPC
      Tags:
      - Key: KubernetesCluster
        Value:
          Fn::If:
          - AssociationProvidedCondition
          - !Ref ClusterAssociation
          - !Ref AWS::StackName
      - Key:
          Fn::Sub:
          - "kubernetes.io/cluster/${ClusterID}"
          - ClusterID:
              Fn::If:
              - AssociationProvidedCondition
              - !Ref ClusterAssociation
              - !Ref AWS::StackName
        Value: 'owned'
      - Key: Name
        Value: k8s-cluster-security-group

  # Permissions we add to the main security group:
  # - Ensure cluster machines can talk to one another
  ClusterSecGroupCrossTalk:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref ClusterSecGroup
      SourceSecurityGroupId: !Ref ClusterSecGroup
      IpProtocol: '-1'
      FromPort: '0'
      ToPort: '65535'

  # - Open up port 22 for SSH into each machine
  # The allowed locations are chosen by the user in the SSHLocation parameter
  ClusterSecGroupAllow22:
    Metadata:
      Comment: Open up port 22 for SSH into each machine
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref ClusterSecGroup
      IpProtocol: tcp
      FromPort: '22'
      ToPort: '22'
      CidrIp: !Ref SSHLocation

  # Allow the apiserver load balancer to talk to the cluster on port 6443
  ClusterSecGroupAllow6443FromLB:
    Metadata:
      Comment: Open up port 6443 for load balancing the API server
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref ClusterSecGroup
      IpProtocol: tcp
      FromPort: '6443'
      ToPort: '6443'
      SourceSecurityGroupId: !Ref ApiLoadBalancerSecGroup

  # IAM role for nodes http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html
  NodeRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: "/"
      # IAM policy for nodes that allows specific AWS resource listing and creation
      # http://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html
      Policies:
      - PolicyName: node
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - ec2:Describe*
            - ecr:GetAuthorizationToken
            - ecr:BatchCheckLayerAvailability
            - ecr:GetDownloadUrlForLayer
            - ecr:GetRepositoryPolicy
            - ecr:DescribeRepositories
            - ecr:ListImages
            - ecr:BatchGetImage
            Resource: "*"

      - PolicyName: cwlogs
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - logs:DescribeLogStreams
            Resource: !Sub ["${LogGroupArn}:*", LogGroupArn: !GetAtt KubernetesLogGroup.Arn]

  # Resource that creates the node IAM role
  NodeInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: "/"
      Roles:
      - !Ref NodeRole

  # IAM role for the master node http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html
  MasterRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: "/"
      # IAM policy for the master node that allows specific AWS resource listing and creation
      # More permissive than the node role (it allows load balancer creation)
      # http://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html
      Policies:
      - PolicyName: master
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - ec2:*
            - elasticloadbalancing:*
            - ecr:GetAuthorizationToken
            - ecr:BatchCheckLayerAvailability
            - ecr:GetDownloadUrlForLayer
            - ecr:GetRepositoryPolicy
            - ecr:DescribeRepositories
            - ecr:ListImages
            - ecr:BatchGetImage
            - autoscaling:DescribeAutoScalingGroups
            - autoscaling:UpdateAutoScalingGroup
            Resource: "*"

      - PolicyName: cwlogs
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - logs:DescribeLogStreams
            Resource: !Sub ["${LogGroupArn}:*", LogGroupArn: !GetAtt KubernetesLogGroup.Arn]

  # Resource that creates the master node IAM role
  MasterInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: "/"
      Roles:
      - !Ref MasterRole

  # Create a placeholder load balancer for the API server. Backend instances will be added by the master itself on the
  # first boot in the startup script above.
  ApiLoadBalancer:
    Type: AWS::ElasticLoadBalancing::LoadBalancer
    Properties:
      Scheme: internet-facing
      Listeners:
      - Protocol: TCP
        InstancePort: 6443
        InstanceProtocol: TCP
        LoadBalancerPort: 443
      ConnectionSettings:
        IdleTimeout: 3600
      Subnets:
      - Fn::If:
        - LoadBalancerSubnetProvidedCondition
        - !Ref LoadBalancerSubnetId
        - !Ref ClusterSubnetId
      SecurityGroups:
      - !Ref ApiLoadBalancerSecGroup
      Tags:
      - Key: KubernetesCluster
        Value:
          Fn::If:
          - AssociationProvidedCondition
          - !Ref ClusterAssociation
          - !Ref AWS::StackName
      - Key:
          Fn::Sub:
          - "kubernetes.io/cluster/${ClusterID}"
          - ClusterID:
              Fn::If:
              - AssociationProvidedCondition
              - !Ref ClusterAssociation
              - !Ref AWS::StackName
        Value: 'owned'
      - Key: 'kubernetes.io/service-name'
        Value: 'kube-system/apiserver-public'

  # Security group to allow public access to port 443
  ApiLoadBalancerSecGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for API server load balancer
      VpcId: !Ref VPCID
      SecurityGroupIngress:
        CidrIp: !Ref ApiLbLocation
        FromPort: 443
        ToPort: 443
        IpProtocol: tcp
      Tags:
      - Key: Name
        Value: apiserver-lb-security-group

# Outputs are what AWS will show you after stack creation
# Generally they let you easily access some information about the stack
# like what IP address is assigned to your master node
# Read Descriptions below for more detail
Outputs:
  MasterInstanceId:
    Description: InstanceId of the master EC2 instance.
    Value: !Ref K8sMasterInstance

  MasterPrivateIp:
    Description: Private IP address of the master.
    Value: !GetAtt K8sMasterInstance.PrivateIp

  NodeGroupInstanceId:
    Description: InstanceId of the newly-created NodeGroup.
    Value: !Ref K8sNodeGroup

  JoinNodes:
    Description: Command to join more nodes to this cluster.
    Value: !Sub "kubeadm join --token=${KubeadmToken.Token} ${K8sMasterInstance.PrivateIp}"

  NextSteps:
    Description: Verify your cluster and deploy a test application. Instructions -
      http://jump.heptio.com/aws-qs-next
    Value: http://jump.heptio.com/aws-qs-next