Ubuntu-spot.yaml

---
AWSTemplateFormatVersion: '2010-09-09'
Description: 'Ubuntu EKS - SPOT'

Parameters:

  KeyName:
    Description: The EC2 Key Pair to allow SSH access to the instances
    Type: AWS::EC2::KeyPair::KeyName

  NodeImageId:
    Type: AWS::EC2::Image::Id
    Description: AMI id for the node instances.

  NodeInstanceType:
    Description: EC2 instance type for the node instances
    Type: String
    Default: t3.medium
    ConstraintDescription: Must be a valid EC2 instance type
    AllowedValues:
      - a1.medium
      - a1.large
      - a1.xlarge
      - a1.2xlarge
      - a1.4xlarge
      - c1.medium
      - c1.xlarge
      - c3.large
      - c3.xlarge
      - c3.2xlarge
      - c3.4xlarge
      - c3.8xlarge
      - c4.large
      - c4.xlarge
      - c4.2xlarge
      - c4.4xlarge
      - c4.8xlarge
      - c5.large
      - c5.xlarge
      - c5.2xlarge
      - c5.4xlarge
      - c5.9xlarge
      - c5.12xlarge
      - c5.18xlarge
      - c5.24xlarge
      - c5.metal
      - c5d.large
      - c5d.xlarge
      - c5d.2xlarge
      - c5d.4xlarge
      - c5d.9xlarge
      - c5d.12xlarge
      - c5d.18xlarge
      - c5d.24xlarge
      - c5d.metal
      - c5n.large
      - c5n.xlarge
      - c5n.2xlarge
      - c5n.4xlarge
      - c5n.9xlarge
      - c5n.18xlarge
      - c6g.medium
      - c6g.large
      - c6g.xlarge
      - c6g.2xlarge
      - c6g.4xlarge
      - c6g.8xlarge
      - c6g.12xlarge
      - c6g.16xlarge
      - c6g.metal
      - c6gd.medium
      - c6gd.large
      - c6gd.xlarge
      - c6gd.2xlarge
      - c6gd.4xlarge
      - c6gd.8xlarge
      - c6gd.12xlarge
      - c6gd.16xlarge
      - c6gd.metal
      - cc2.8xlarge
      - cr1.8xlarge
      - d2.xlarge
      - d2.2xlarge
      - d2.4xlarge
      - d2.8xlarge
      - f1.2xlarge
      - f1.4xlarge
      - f1.16xlarge
      - g2.2xlarge
      - g2.8xlarge
      - g3s.xlarge
      - g3.4xlarge
      - g3.8xlarge
      - g3.16xlarge
      - h1.2xlarge
      - h1.4xlarge
      - h1.8xlarge
      - h1.16xlarge
      - hs1.8xlarge
      - i2.xlarge
      - i2.2xlarge
      - i2.4xlarge
      - i2.8xlarge
      - i3.large
      - i3.xlarge
      - i3.2xlarge
      - i3.4xlarge
      - i3.8xlarge
      - i3.16xlarge
      - i3.metal
      - i3en.large
      - i3en.xlarge
      - i3en.2xlarge
      - i3en.3xlarge
      - i3en.6xlarge
      - i3en.12xlarge
      - i3en.24xlarge
      - inf1.xlarge
      - inf1.2xlarge
      - inf1.6xlarge
      - inf1.24xlarge
      - m1.small
      - m1.medium
      - m1.large
      - m1.xlarge
      - m2.xlarge
      - m2.2xlarge
      - m2.4xlarge
      - m3.medium
      - m3.large
      - m3.xlarge
      - m3.2xlarge
      - m4.large
      - m4.xlarge
      - m4.2xlarge
      - m4.4xlarge
      - m4.10xlarge
      - m4.16xlarge
      - m5.large
      - m5.xlarge
      - m5.2xlarge
      - m5.4xlarge
      - m5.8xlarge
      - m5.12xlarge
      - m5.16xlarge
      - m5.24xlarge
      - m5.metal
      - m5a.large
      - m5a.xlarge
      - m5a.2xlarge
      - m5a.4xlarge
      - m5a.8xlarge
      - m5a.12xlarge
      - m5a.16xlarge
      - m5a.24xlarge
      - m5ad.large
      - m5ad.xlarge
      - m5ad.2xlarge
      - m5ad.4xlarge
      - m5ad.12xlarge
      - m5ad.24xlarge
      - m5d.large
      - m5d.xlarge
      - m5d.2xlarge
      - m5d.4xlarge
      - m5d.8xlarge
      - m5d.12xlarge
      - m5d.16xlarge
      - m5d.24xlarge
      - m5d.metal
      - m5dn.large
      - m5dn.xlarge
      - m5dn.2xlarge
      - m5dn.4xlarge
      - m5dn.8xlarge
      - m5dn.12xlarge
      - m5dn.16xlarge
      - m5dn.24xlarge
      - m5n.large
      - m5n.xlarge
      - m5n.2xlarge
      - m5n.4xlarge
      - m5n.8xlarge
      - m5n.12xlarge
      - m5n.16xlarge
      - m5n.24xlarge
      - m6g.medium
      - m6g.large
      - m6g.xlarge
      - m6g.2xlarge
      - m6g.4xlarge
      - m6g.8xlarge
      - m6g.12xlarge
      - m6g.16xlarge
      - m6g.metal
      - m6gd.medium
      - m6gd.large
      - m6gd.xlarge
      - m6gd.2xlarge
      - m6gd.4xlarge
      - m6gd.8xlarge
      - m6gd.12xlarge
      - m6gd.16xlarge
      - m6gd.metal
      - p2.xlarge
      - p2.8xlarge
      - p2.16xlarge
      - p3.2xlarge
      - p3.8xlarge
      - p3.16xlarge
      - p3dn.24xlarge
      - g4dn.xlarge
      - g4dn.2xlarge
      - g4dn.4xlarge
      - g4dn.8xlarge
      - g4dn.12xlarge
      - g4dn.16xlarge
      - g4dn.metal
      - r3.large
      - r3.xlarge
      - r3.2xlarge
      - r3.4xlarge
      - r3.8xlarge
      - r4.large
      - r4.xlarge
      - r4.2xlarge
      - r4.4xlarge
      - r4.8xlarge
      - r4.16xlarge
      - r5.large
      - r5.xlarge
      - r5.2xlarge
      - r5.4xlarge
      - r5.8xlarge
      - r5.12xlarge
      - r5.16xlarge
      - r5.24xlarge
      - r5.metal
      - r5a.large
      - r5a.xlarge
      - r5a.2xlarge
      - r5a.4xlarge
      - r5a.8xlarge
      - r5a.12xlarge
      - r5a.16xlarge
      - r5a.24xlarge
      - r5ad.large
      - r5ad.xlarge
      - r5ad.2xlarge
      - r5ad.4xlarge
      - r5ad.12xlarge
      - r5ad.24xlarge
      - r5d.large
      - r5d.xlarge
      - r5d.2xlarge
      - r5d.4xlarge
      - r5d.8xlarge
      - r5d.12xlarge
      - r5d.16xlarge
      - r5d.24xlarge
      - r5d.metal
      - r5dn.large
      - r5dn.xlarge
      - r5dn.2xlarge
      - r5dn.4xlarge
      - r5dn.8xlarge
      - r5dn.12xlarge
      - r5dn.16xlarge
      - r5dn.24xlarge
      - r5n.large
      - r5n.xlarge
      - r5n.2xlarge
      - r5n.4xlarge
      - r5n.8xlarge
      - r5n.12xlarge
      - r5n.16xlarge
      - r5n.24xlarge
      - r6g.medium
      - r6g.large
      - r6g.xlarge
      - r6g.2xlarge
      - r6g.4xlarge
      - r6g.8xlarge
      - r6g.12xlarge
      - r6g.16xlarge
      - r6g.metal
      - r6gd.medium
      - r6gd.large
      - r6gd.xlarge
      - r6gd.2xlarge
      - r6gd.4xlarge
      - r6gd.8xlarge
      - r6gd.12xlarge
      - r6gd.16xlarge
      - r6gd.metal
      - t1.micro
      - t2.nano
      - t2.micro
      - t2.small
      - t2.medium
      - t2.large
      - t2.xlarge
      - t2.2xlarge
      - t3.nano
      - t3.micro
      - t3.small
      - t3.medium
      - t3.large
      - t3.xlarge
      - t3.2xlarge
      - t3a.nano
      - t3a.micro
      - t3a.small
      - t3a.medium
      - t3a.large
      - t3a.xlarge
      - t3a.2xlarge
      - u-6tb1.metal
      - u-9tb1.metal
      - u-12tb1.metal
      - x1.16xlarge
      - x1.32xlarge
      - x1e.xlarge
      - x1e.2xlarge
      - x1e.4xlarge
      - x1e.8xlarge
      - x1e.16xlarge
      - x1e.32xlarge
      - z1d.large
      - z1d.xlarge
      - z1d.2xlarge
      - z1d.3xlarge
      - z1d.6xlarge
      - z1d.12xlarge
      - z1d.metal

  NodeAutoScalingGroupMaxSize:
    Type: Number
    Description: Maximum size of Node Group ASG.
    Default: 3

  NodeVolumeSize:
    Type: Number
    Default: 20
    Description: Node volume size (in GB)
    
  NodeDesiredSize: 
    Type: Number
    Description: Desired Capacity of the Node Group ASG
    Default: 2

  ClusterName:
    Description: The cluster name provided when the cluster was created.  If it is incorrect, nodes will not be able to join the cluster.
    Type: String
    
  BootstrapArguments:
    Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami
    Type: String
    Default: ""
    
  NodeGroupName:
    Description: Unique identifier for the Node Group.
    Type: String

  ClusterControlPlaneSecurityGroup:
    Description: The security group of the cluster control plane.
    Type: AWS::EC2::SecurityGroup::Id

  WorkerNodeSecurityGroup:
    Description: The additional security group of the workernodes.
    Type: AWS::EC2::SecurityGroup::Id
    
  VpcId:
    Description: The VPC of the worker instances
    Type: AWS::EC2::VPC::Id

  Subnets:
    Description: The subnets where workers can be created.
    Type: List<AWS::EC2::Subnet::Id>
    
  SpotPrice:
    Description: The spot price for this Auto Scaling group
    Type: String

  ExtraPackages: 
    Description: Extra Ubuntu Package in WorkerNodeGroup. Check package name from apt search on Ubuntu 20.04
    Type: String
    Default: "glusterfs-client"

Mappings:
  PartitionMap:
    aws:
      EC2ServicePrincipal: "ec2.amazonaws.com"
    aws-us-gov:
      EC2ServicePrincipal: "ec2.amazonaws.com"
    aws-cn:
      EC2ServicePrincipal: "ec2.amazonaws.com.cn"
    aws-iso:
      EC2ServicePrincipal: "ec2.c2s.ic.gov"
    aws-iso-b:
      EC2ServicePrincipal: "ec2.sc2s.sgov.gov"
    
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      -
        Label:
          default: "EKS Cluster"
        Parameters:
          - ClusterName
          - ClusterControlPlaneSecurityGroup
      -
        Label:
          default: "Worker Node Configuration"
        Parameters:
          - NodeGroupName
          - NodeAutoScalingGroupMaxSize
          - NodeDesiredSize
          - NodeInstanceType
          - SpotPrice
          - NodeImageId
          - NodeVolumeSize
          - KeyName
          - WorkerNodeSecurityGroup
          - BootstrapArguments
          - ExtraPackages
      -
        Label:
          default: "Worker Network Configuration"
        Parameters:
          - VpcId
          - Subnets

Resources:
  
  NodeInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: "/"
      Roles:
      - !Ref NodeInstanceRole

  NodeInstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - !FindInMap [PartitionMap, !Ref "AWS::Partition", EC2ServicePrincipal]
          Action:
          - sts:AssumeRole
      Path: "/"
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
        - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy
        - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore

  NodeSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for all nodes in the cluster
      VpcId:
        !Ref VpcId
      Tags:
      - Key: !Sub "kubernetes.io/cluster/${ClusterName}"
        Value: 'owned'

  NodeSecurityGroupIngress:
    Type: AWS::EC2::SecurityGroupIngress
    DependsOn: NodeSecurityGroup
    Properties:
      Description: Allow node to communicate with each other
      GroupId: !Ref NodeSecurityGroup
      SourceSecurityGroupId: !Ref NodeSecurityGroup
      IpProtocol: '-1'
      FromPort: 0
      ToPort: 65535

  NodeSecurityGroupFromControlPlaneIngress:
    Type: AWS::EC2::SecurityGroupIngress
    DependsOn: NodeSecurityGroup
    Properties:
      Description: Allow worker Kubelets and pods to receive communication from the cluster control plane
      GroupId: !Ref NodeSecurityGroup
      SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup
      IpProtocol: tcp
      FromPort: 1025
      ToPort: 65535

  ControlPlaneEgressToNodeSecurityGroup:
    Type: AWS::EC2::SecurityGroupEgress
    DependsOn: NodeSecurityGroup
    Properties:
      Description: Allow the cluster control plane to communicate with worker Kubelet and pods
      GroupId: !Ref ClusterControlPlaneSecurityGroup
      DestinationSecurityGroupId: !Ref NodeSecurityGroup
      IpProtocol: tcp
      FromPort: 1025
      ToPort: 65535

  ClusterControlPlaneSecurityGroupIngress:
    Type: AWS::EC2::SecurityGroupIngress
    DependsOn: NodeSecurityGroup
    Properties:
      Description: Allow pods to communicate with the cluster API Server
      GroupId: !Ref ClusterControlPlaneSecurityGroup
      SourceSecurityGroupId: !Ref NodeSecurityGroup
      IpProtocol: tcp
      ToPort: 443
      FromPort: 443

  NodeSecurityGroupFromControlPlaneOn443Ingress:
    Type: AWS::EC2::SecurityGroupIngress
    DependsOn: NodeSecurityGroup
    Properties:
      Description: Allow pods running extension API servers on port 443 to receive communication from cluster control plane
      GroupId: !Ref NodeSecurityGroup
      SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup
      IpProtocol: tcp
      FromPort: 443
      ToPort: 443

  ControlPlaneEgressToNodeSecurityGroupOn443:
    Type: AWS::EC2::SecurityGroupEgress
    DependsOn: NodeSecurityGroup
    Properties:
      Description: Allow the cluster control plane to communicate with pods running extension API servers on port 443
      GroupId: !Ref ClusterControlPlaneSecurityGroup
      DestinationSecurityGroupId: !Ref NodeSecurityGroup
      IpProtocol: tcp
      FromPort: 443
      ToPort: 443

  NodeGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      DesiredCapacity: !Ref NodeDesiredSize
      LaunchConfigurationName: !Ref NodeLaunchConfig
      MinSize: 0
      MaxSize: !Ref NodeAutoScalingGroupMaxSize
      VPCZoneIdentifier:
        !Ref Subnets
      Tags:
      - Key: Name
        Value: !Sub "${ClusterName}-${NodeGroupName}-Node"
        PropagateAtLaunch: 'true'
      - Key: !Sub 'kubernetes.io/cluster/${ClusterName}'
        Value: 'owned'
        PropagateAtLaunch: 'true'
    UpdatePolicy:
      AutoScalingRollingUpdate:
        MinInstancesInService: '0'   
        MaxBatchSize: '1'
        PauseTime: PT3M

  NodeLaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Properties:
      AssociatePublicIpAddress: 'false'
      BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
            DeleteOnTermination: true
            VolumeSize: !Ref NodeVolumeSize
            VolumeType: gp2
      IamInstanceProfile: !Ref NodeInstanceProfile
      ImageId: !Ref NodeImageId
      InstanceType: !Ref NodeInstanceType
      KeyName: !Ref KeyName
      SpotPrice: !Ref SpotPrice
      SecurityGroups:
      - !Ref NodeSecurityGroup
      - !Ref WorkerNodeSecurityGroup   
      UserData:
        Fn::Base64: !Sub |     
           #cloud-config
           # Remove the built-in customization if you want/need to replace it completely
           
           write_files:
           - path: /etc/sysctl.conf
             permissions: "0644"
             content: |
               #fs.file-max = 200000
               net.core.somaxconn = 65535
               # max read buffer
               net.core.rmem_max = 67108864
               # max write buffer
               net.core.wmem_max = 67108864
               # default read buffer
               net.core.rmem_default = 212992
               # default write buffer
               net.core.wmem_default = 212992
               # max processor input queue
               net.core.netdev_max_backlog = 4096
               # turn on path MTU discovery
               net.ipv4.tcp_mtu_probing = 1
               # turn on TCP Fast Open on both client and server side
               net.ipv4.tcp_fastopen = 3
               # DCTCP for high speed internal network
               net.ipv4.tcp_congestion_control=dctcp
               # TCP Keepalive
               net.ipv4.tcp_keepalive_time = 600
               net.ipv4.tcp_keepalive_intvl = 60
               net.ipv4.tcp_keepalive_probes = 3
               # Turn on and log spoofed, source routed, and redirect packets
               #net.ipv4.conf.all.log_martians = 1
               #net.ipv4.conf.default.log_martians = 1
               # Maximum SYN Queue Size
               #net.ipv4.tcp_max_syn_backlog = 32768
               # Reset TCP Connection when SYN queue is full
               #net.ipv4.tcp_abort_on_overflow=1
               # Reduce TCP TIME_WAIT
               net.ipv4.tcp_fin_timeout = 10

           - path: /etc/apt/apt.conf.d/20auto-upgrades
             permissions: "0644"
             content: |
               APT::Periodic::Update-Package-Lists "0";
               APT::Periodic::Unattended-Upgrade "0";

           - path: /etc/security/limits.conf
             permissions: "0644"
             content: |
               # /etc/security/limits.conf
               #
               #This file sets the resource limits for the users logged in via PAM.
               #It does not affect resource limits of the system services.
               #
               #Also note that configuration files in /etc/security/limits.d directory,
               #which are read in alphabetical order, override the settings in this
               #file in case the domain is the same or more specific.
               #That means for example that setting a limit for wildcard domain here
               #can be overriden with a wildcard setting in a config file in the
               #subdirectory, but a user specific setting here can be overriden only
               #with a user specific setting in the subdirectory.
               #
               #Each line describes a limit for a user in the form:
               #
               #<domain>        <type>  <item>  <value>
               #
               #Where:
               #<domain> can be:
               #        - a user name
               #        - a group name, with @group syntax
               #        - the wildcard *, for default entry
               #        - the wildcard %, can be also used with %group syntax,
               #                 for maxlogin limit
               #
               #<type> can have the two values:
               #        - "soft" for enforcing the soft limits
               #        - "hard" for enforcing hard limits
               #
               #<item> can be one of the following:
               #        - core - limits the core file size (KB)
               #        - data - max data size (KB)
               #        - fsize - maximum filesize (KB)
               #        - memlock - max locked-in-memory address space (KB)
               #        - nofile - max number of open file descriptors
               #        - rss - max resident set size (KB)
               #        - stack - max stack size (KB)
               #        - cpu - max CPU time (MIN)
               #        - nproc - max number of processes
               #        - as - address space limit (KB)
               #        - maxlogins - max number of logins for this user
               #        - maxsyslogins - max number of logins on the system
               #        - priority - the priority to run user process with
               #        - locks - max number of file locks the user can hold
               #        - sigpending - max number of pending signals
               #        - msgqueue - max memory used by POSIX message queues (bytes)
               #        - nice - max nice priority allowed to raise to values: [-20, 19]
               #        - rtprio - max realtime priority
               #
               #<domain>      <type>  <item>         <value>
               #
               
               #*               soft    core            0
               #*               hard    rss             10000
               #@student        hard    nproc           20
               #@faculty        soft    nproc           20
               #@faculty        hard    nproc           50
               #ftp             hard    nproc           0
               #@student        -       maxlogins       4
               
               #
               
               root                     soft    nofile          500000
               root                     hard    nofile          500000
               *                        soft    nofile          500000
               *                        hard    nofile          500000
               # End of file
                  
               
           - path: /bin/startup.sh
             permissions: "0755"
             content: |
               #!/bin/bash
               export DEBIAN_FRONTEND=noninteractive
               ulimit -n 500000
               /sbin/sysctl -p /etc/sysctl.conf
               /usr/bin/apt-get update
               /usr/bin/apt-get install -y ${ExtraPackages}
               set -o xtrace
               /etc/eks/bootstrap.sh ${ClusterName} ${BootstrapArguments}
               /opt/aws/bin/cfn-signal --exit-code $? \
                     --stack  ${AWS::StackName} \
                     --resource NodeGroup  \
                     --region ${AWS::Region}
           
           
           runcmd:
           - [ mkdir, -p, /www, /gpfs_carwritable, /gpfs_http_data, /gpfs_writable ]
           - [ /bin/bash, /bin/startup.sh ]

        
        
Outputs:
  NodeInstanceRole:
    Description: The node instance role
    Value: !GetAtt NodeInstanceRole.Arn

  NodeInstanceRole:
    Description: The node instance role
    Value: !GetAtt NodeInstanceRole.Arn

  NodeSecurityGroup:
    Description: The security group for the node group
    Value: !Ref NodeSecurityGroup