TemplateClusterCreate-NoVPC.json

{
  "AWSTemplateFormatVersion": "2010-09-09",

  "Description": "AWS CloudFormation H2O Template: Create an Amazon Cluster of EC2 instance running the Linux H2O-AMI and H2O cloud. The AMI is chosen based on the region in which the stack is run. ",

  "Parameters": {
    "KeyName": {
      "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance",
      "Type": "AWS::EC2::KeyPair::KeyName",
      "ConstraintDescription": "must be the name of an existing EC2 KeyPair."
    },

    "InstanceType": {
      "Description": "EC2 instance type",
      "Type": "String",
      "Default": "m4.xlarge",
      "AllowedValues": [ "t2.small", "t2.medium", "t2.large", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "i2.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "hi1.4xlarge", "hs1.8xlarge", "cr1.8xlarge", "cc2.8xlarge", "cg1.4xlarge", "x1.16xlarge", "x1.32xlarge" ],
      "ConstraintDescription": "must be a valid EC2 instance type."
    },

    "vmCount": {
      "Description": "Number of EC2 instances to deploy",
      "Type": "Number",
      "Default": "1"
    },


    "SSHLocation": {
      "Description": "The IP address range that can be used to SSH to the EC2 instances",
      "Type": "String",
      "MinLength": "9",
      "MaxLength": "18",      
      "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
    }
  },

  "Mappings": {

    "SubnetConfig": {
      "VPC": { "CIDR": "10.0.0.0/16" },
      "Public1": { "CIDR": "10.0.0.0/24" },
      "Public2": { "CIDR": "10.0.1.0/24" }
    },

    "AWSInstanceType2Arch": {
      "t1.micro": { "Arch": "HVM64" },
      "t2.nano": { "Arch": "HVM64" },
      "t2.micro": { "Arch": "HVM64" },
      "t2.small": { "Arch": "HVM64" },
      "t2.medium": { "Arch": "HVM64" },
      "t2.large": { "Arch": "HVM64" },
      "m1.small": { "Arch": "HVM64" },
      "m1.medium": { "Arch": "HVM64" },
      "m1.large": { "Arch": "HVM64" },
      "m1.xlarge": { "Arch": "HVM64" },
      "m2.xlarge": { "Arch": "HVM64" },
      "m2.2xlarge": { "Arch": "HVM64" },
      "m2.4xlarge": { "Arch": "HVM64" },
      "m3.medium": { "Arch": "HVM64" },
      "m3.large": { "Arch": "HVM64" },
      "m3.xlarge": { "Arch": "HVM64" },
      "m3.2xlarge": { "Arch": "HVM64" },
      "m4.large": { "Arch": "HVM64" },
      "m4.xlarge": { "Arch": "HVM64" },
      "m4.2xlarge": { "Arch": "HVM64" },
      "m4.4xlarge": { "Arch": "HVM64" },
      "m4.10xlarge": { "Arch": "HVM64" },
      "c1.medium": { "Arch": "HVM64" },
      "c1.xlarge": { "Arch": "HVM64" },
      "c3.large": { "Arch": "HVM64" },
      "c3.xlarge": { "Arch": "HVM64" },
      "c3.2xlarge": { "Arch": "HVM64" },
      "c3.4xlarge": { "Arch": "HVM64" },
      "c3.8xlarge": { "Arch": "HVM64" },
      "c4.large": { "Arch": "HVM64" },
      "c4.xlarge": { "Arch": "HVM64" },
      "c4.2xlarge": { "Arch": "HVM64" },
      "c4.4xlarge": { "Arch": "HVM64" },
      "c4.8xlarge": { "Arch": "HVM64" },
      "g2.2xlarge": { "Arch": "HVMG2" },
      "g2.8xlarge": { "Arch": "HVMG2" },
      "r3.large": { "Arch": "HVM64" },
      "r3.xlarge": { "Arch": "HVM64" },
      "r3.2xlarge": { "Arch": "HVM64" },
      "r3.4xlarge": { "Arch": "HVM64" },
      "r3.8xlarge": { "Arch": "HVM64" },
      "r4.large": { "Arch": "HVM64" },
      "r4.xlarge": { "Arch": "HVM64" },
      "r4.2xlarge": { "Arch": "HVM64" },
      "r4.4xlarge": { "Arch": "HVM64" },
      "r4.8xlarge": { "Arch": "HVM64" },
      "r4.16xlarge": { "Arch": "HVM64" },
      "x1.16xlarge": { "Arch": "HVM64" },
      "x1.32xlarge": { "Arch": "HVM64" },
      "i2.xlarge": { "Arch": "HVM64" },
      "i2.2xlarge": { "Arch": "HVM64" },
      "i2.4xlarge": { "Arch": "HVM64" },
      "i2.8xlarge": { "Arch": "HVM64" },
      "x3.32xlarge": { "Arch": "HVM64" },
      "i3.xlarge": { "Arch": "HVM64" },
      "i3.2xlarge": { "Arch": "HVM64" },
      "i3.4xlarge": { "Arch": "HVM64" },
      "i3.8xlarge": { "Arch": "HVM64" },
      "i3.16xlarge": { "Arch": "HVM64" },
      "d2.xlarge": { "Arch": "HVM64" },
      "d2.2xlarge": { "Arch": "HVM64" },
      "d2.4xlarge": { "Arch": "HVM64" },
      "d2.8xlarge": { "Arch": "HVM64" },
      "hi1.4xlarge": { "Arch": "HVM64" },
      "hs1.8xlarge": { "Arch": "HVM64" },
      "cr1.8xlarge": { "Arch": "HVM64" },
      "cc2.8xlarge": { "Arch": "HVM64" }
    },

    "AWSRegionArch2AMI": {
      "us-east-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "us-west-2": {
        "HVM64": "ami-bdf4c7c6"
      },
      "us-west-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "eu-west-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "eu-west-2": {
        "HVM64": "ami-bdf4c7c6"
      },
      "eu-central-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "ap-northeast-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "ap-northeast-2": {
        "HVM64": "ami-bdf4c7c6"
      },
      "ap-southeast-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "ap-southeast-2": {
        "HVM64": "ami-bdf4c7c6"
      },
      "ap-south-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "us-east-2": {
        "HVM64": "ami-bdf4c7c6"
      },
      "ca-central-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "sa-east-1": {
        "HVM64": "ami-bdf4c7c6"
      },
      "cn-north-1": {
        "HVM64": "ami-bdf4c7c6"
      }
    }

  },

  "Resources": {

    "VPC": {
      "Type": "AWS::EC2::VPC",
      "Properties": {
        "CidrBlock": { "Fn::FindInMap": [ "SubnetConfig", "VPC", "CIDR" ] },
        "EnableDnsSupport": "true",
        "EnableDnsHostnames": "true",
        "Tags": [
          {
            "Key": "Application",
            "Value": { "Ref": "AWS::StackId" }
          },
          {
            "Key": "Network",
            "Value": "Public"
          }
        ]
      }
    },

    "PublicSubnet1": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "VpcId": { "Ref": "VPC" },
        "CidrBlock": { "Fn::FindInMap": [ "SubnetConfig", "Public1", "CIDR" ] },
        "AvailabilityZone": {
          "Fn::Select": [
            0,
            { "Fn::GetAZs": "" }
          ]
        },
        "Tags": [
          {
            "Key": "Application",
            "Value": { "Ref": "AWS::StackId" }
          },
          {
            "Key": "Network",
            "Value": "Public"
          }
        ]
      }
    },

    "PublicSubnet2": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "VpcId": { "Ref": "VPC" },
        "CidrBlock": { "Fn::FindInMap": [ "SubnetConfig", "Public2", "CIDR" ] },
        "AvailabilityZone": {
          "Fn::Select": [
            1,
            { "Fn::GetAZs": "" }
          ]
        },
        "Tags": [
          {
            "Key": "Application",
            "Value": { "Ref": "AWS::StackId" }
          },
          {
            "Key": "Network",
            "Value": "Public"
          }
        ]
      }
    },

    "InternetGateway": {
      "Type": "AWS::EC2::InternetGateway",
      "Properties": {
        "Tags": [
          {
            "Key": "Application",
            "Value": { "Ref": "AWS::StackId" }
          },
          {
            "Key": "Network",
            "Value": "Public"
          }
        ]
      }
    },

    "GatewayToInternet": {
      "Type": "AWS::EC2::VPCGatewayAttachment",
      "Properties": {
        "VpcId": { "Ref": "VPC" },
        "InternetGatewayId": { "Ref": "InternetGateway" }
      }
    },

    "PublicRouteTable": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "VpcId": { "Ref": "VPC" },
        "Tags": [
          {
            "Key": "Application",
            "Value": { "Ref": "AWS::StackId" }
          },
          {
            "Key": "Network",
            "Value": "Public"
          }
        ]
      }
    },

    "PublicRoute": {
      "Type": "AWS::EC2::Route",
      "DependsOn": "GatewayToInternet",
      "Properties": {
        "RouteTableId": { "Ref": "PublicRouteTable" },
        "DestinationCidrBlock": "0.0.0.0/0",
        "GatewayId": { "Ref": "InternetGateway" }
      }
    },

    "PublicSubnetRouteTableAssociation1": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "SubnetId": { "Ref": "PublicSubnet1" },
        "RouteTableId": { "Ref": "PublicRouteTable" }
      }
    },

    "PublicSubnetRouteTableAssociation2": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "SubnetId": { "Ref": "PublicSubnet2" },
        "RouteTableId": { "Ref": "PublicRouteTable" }
      }
    },

    "PublicNetworkAcl": {
      "Type": "AWS::EC2::NetworkAcl",
      "Properties": {
        "VpcId": { "Ref": "VPC" },
        "Tags": [
          {
            "Key": "Application",
            "Value": { "Ref": "AWS::StackId" }
          },
          {
            "Key": "Network",
            "Value": "Public"
          }
        ]
      }
    },

    "InboundHTTPPublicNetworkAclEntry": {
      "Type": "AWS::EC2::NetworkAclEntry",
      "Properties": {
        "NetworkAclId": { "Ref": "PublicNetworkAcl" },
        "RuleNumber": "100",
        "Protocol": "6",
        "RuleAction": "allow",
        "Egress": "false",
        "CidrBlock": "0.0.0.0/0",
        "PortRange": {
          "From": "80",
          "To": "80"
        }
      }
    },

    "InboundDynamicPortPublicNetworkAclEntry": {
      "Type": "AWS::EC2::NetworkAclEntry",
      "Properties": {
        "NetworkAclId": { "Ref": "PublicNetworkAcl" },
        "RuleNumber": "101",
        "Protocol": "6",
        "RuleAction": "allow",
        "Egress": "false",
        "CidrBlock": "0.0.0.0/0",
        "PortRange": {
          "From": "1024",
          "To": "65535"
        }
      }
    },

    "InboundSSHPublicNetworkAclEntry": {
      "Type": "AWS::EC2::NetworkAclEntry",
      "Properties": {
        "NetworkAclId": { "Ref": "PublicNetworkAcl" },
        "RuleNumber": "102",
        "Protocol": "6",
        "RuleAction": "allow",
        "Egress": "false",
        "CidrBlock": { "Ref": "SSHLocation" },
        "PortRange": {
          "From": "22",
          "To": "22"
        }
      }
    },

    "OutboundPublicNetworkAclEntry": {
      "Type": "AWS::EC2::NetworkAclEntry",
      "Properties": {
        "NetworkAclId": { "Ref": "PublicNetworkAcl" },
        "RuleNumber": "100",
        "Protocol": "6",
        "RuleAction": "allow",
        "Egress": "true",
        "CidrBlock": "0.0.0.0/0",
        "PortRange": {
          "From": "0",
          "To": "65535"
        }
      }
    },

    "PublicSubnetNetworkAclAssociation1": {
      "Type": "AWS::EC2::SubnetNetworkAclAssociation",
      "Properties": {
        "SubnetId": { "Ref": "PublicSubnet1" },
        "NetworkAclId": { "Ref": "PublicNetworkAcl" }
      }
    },

    "PublicSubnetNetworkAclAssociation2": {
      "Type": "AWS::EC2::SubnetNetworkAclAssociation",
      "Properties": {
        "SubnetId": { "Ref": "PublicSubnet2" },
        "NetworkAclId": { "Ref": "PublicNetworkAcl" }
      }
    },

    "PublicApplicationLoadBalancer": {
      "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
      "Properties": {
        "Subnets": [
          { "Ref": "PublicSubnet1" },
          { "Ref": "PublicSubnet2" }
        ],
        "SecurityGroups": [ { "Ref": "PublicLoadBalancerSecurityGroup" } ]
      }
    },

    "ALBListener": {
      "Type": "AWS::ElasticLoadBalancingV2::Listener",
      "Properties": {
        "DefaultActions": [
          {
            "Type": "forward",
            "TargetGroupArn": { "Ref": "ALBTargetGroup" }
          }
        ],
        "LoadBalancerArn": { "Ref": "PublicApplicationLoadBalancer" },
        "Port": "80",
        "Protocol": "HTTP"
      }
    },

    "ALBTargetGroup": {
      "Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
      "Properties": {
        "HealthCheckIntervalSeconds": 90,
        "HealthCheckTimeoutSeconds": 60,
        "HealthyThresholdCount": 3,
        "Port": 80,
        "Protocol": "HTTP",
        "UnhealthyThresholdCount": 5,
        "VpcId": { "Ref": "VPC" }
      }
    },

    "PublicLoadBalancerSecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Public ELB Security Group with HTTP access on port 80 from the internet",
        "VpcId": { "Ref": "VPC" },
        "SecurityGroupIngress": [
          {
            "IpProtocol": "tcp",
            "FromPort": "80",
            "ToPort": "80",
            "CidrIp": "0.0.0.0/0"
          }
        ],
        "SecurityGroupEgress": [
          {
            "IpProtocol": "tcp",
            "FromPort": "80",
            "ToPort": "80",
            "CidrIp": "0.0.0.0/0"
          }
        ]
      }
    },

    "H2OFleet": {
      "Type": "AWS::AutoScaling::AutoScalingGroup",
      "DependsOn": "PublicRoute",
      "Properties": {
        "VPCZoneIdentifier": [
          { "Ref": "PublicSubnet1" }
        ],
        "LaunchConfigurationName": { "Ref": "H2OLaunchConfig" },
        "MinSize": { "Ref": "vmCount" },
        "MaxSize": { "Ref": "vmCount" },
        "DesiredCapacity": { "Ref": "vmCount" },
        "Tags": [
          {
            "Key": "Network",
            "Value": "Public",
            "PropagateAtLaunch": "true"
          }
        ]
      }

    },

    "H2OLaunchConfig": {
      "Type": "AWS::AutoScaling::LaunchConfiguration",
      "Metadata": {
        "AWS::CloudFormation::Init": {
          "config": {

            "files": {

              "/opt/h2oai/create_flatfile.sh": {
                "content": {
                  "Fn::Join": [
                    "",
                    [
                      "#!/bin/bash -xe\n",
                      "#Create Flatfile\n",
                      "touch /opt/h2oai/flatfile.txt\n",
                      "flatfile=\"/opt/h2oai/flatfile.txt\" \n ",

                      "wget http://s3.amazonaws.com/ec2metadata/ec2-metadata \n",
                      "sudo chmod u+x ec2-metadata \n",

                      "INSTANCE_ID=$(./ec2-metadata | grep instance-id | awk 'NR==1{print $2}') \n",
                      "AG_NAME=$(aws autoscaling describe-auto-scaling-instances --instance-ids ${INSTANCE_ID}",
                      " --region ",
                      { "Ref": "AWS::Region" },
                      " --query AutoScalingInstances[].AutoScalingGroupName --output text) \n",

                      "for ID in $(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names ${AG_NAME}",
                      " --region ",
                      { "Ref": "AWS::Region" },
                      " --query AutoScalingGroups[].Instances[].InstanceId --output text); \n",


                      "do \n",
                      "IP=$(aws ec2 describe-instances --instance-ids $ID ",
                      "--region ",
                      { "Ref": "AWS::Region" },
                      " --query Reservations[].Instances[].PrivateIpAddress --output text) \n",
                      "echo \"${IP}:54321\" >> \"$flatfile\" \n ",
                      "done \n"

                    ]
                  ]
                }
              }

            }

          }
        }
      },
      "Properties": {
        "ImageId": {
          "Fn::FindInMap": [
            "AWSRegionArch2AMI",
            { "Ref": "AWS::Region" },
            {
              "Fn::FindInMap": [
                "AWSInstanceType2Arch",
                { "Ref": "InstanceType" },
                "Arch"
              ]
            }
          ]
        },
        "SecurityGroups": [ { "Ref": "H2OSecurityGroup" } ],
        "InstanceType": { "Ref": "InstanceType" },
        "KeyName": { "Ref": "KeyName" },
        "IamInstanceProfile": { "Ref": "TemplateInstanceProfile" },
        "AssociatePublicIpAddress": "true",
        "UserData": {
          "Fn::Base64": {
            "Fn::Join": [
              "",
              [
                "#cloud-boothook",
                "#!/bin/bash -xe\n",
                "apt-get -y install python-setuptools\n",
                "easy_install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz \n",
                "pip install --upgrade awscli \n",

                "# Install the sample application\n",
                "cfn-init -v ",
                "    --stack ",
                { "Ref": "AWS::StackId" },
                "    --resource H2OLaunchConfig ",
                "    --region ",
                { "Ref": "AWS::Region" },
                "\n",


                "# Create the FlatFile \n",
                "sh /opt/h2oai/create_flatfile.sh \n",
                "sh /opt/h2oai/install_h2o.sh \n",
                "touch /opt/h2oai/h2o.err \n",
                "touch /opt/h2oai/check.err \n",
                "ls /opt/h2oai -ls > /opt/h2oai/check.err \n",

                "# Start H2O Cloud\n",
                "# Use 90% of RAM for H2O. \n",
                "memTotalKb=`cat /proc/meminfo | grep MemTotal | sed 's/MemTotal:[ \t]*//' | sed 's/ kB//'` \n ",
                "memTotalMb=$(($memTotalKb / 1024)) \n ",
                "tmp=$(($memTotalMb * 90))  \n",
                "xmxMb=$(($tmp / 100))  \n",
                "sudo nohup java -Xmx${xmxMb}m -jar /opt/h2oai/h2o.jar -flatfile /opt/h2oai/flatfile.txt 1> /dev/null 2> /opt/h2oai/h2o.err & \n",

                "# Signal copletion\n",
                "cfn-signal -e $? ",
                "    --stack ",
                { "Ref": "AWS::StackId" },
                "    --resource H2OFleet ",
                "    --region ",
                { "Ref": "AWS::Region" },
                "\n"
              ]
            ]
          }
        }
      }
    },

    "H2OSecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Enable SSH access via port 22",
        "SecurityGroupIngress": [
          {
            "IpProtocol": "tcp",
            "FromPort": "22",
            "ToPort": "22",
            "CidrIp": { "Ref": "SSHLocation" }
          },
          {
            "IpProtocol": "tcp",
            "FromPort": "54322",
            "ToPort": "54322",
            "CidrIp": { "Ref": "SSHLocation" }
          },
          {
            "IpProtocol": "tcp",
            "FromPort": "54321",
            "ToPort": "54321",
            "CidrIp": { "Ref": "SSHLocation" }
          }
        ],

        "VpcId": { "Ref": "VPC" }
      }
    },

    "TemplateRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": {
                "Service": [ "ec2.amazonaws.com" ]
              },
              "Action": [ "sts:AssumeRole" ]
            }
          ]
        },
        "Path": "/",
        "Policies": [
          {
            "PolicyName": "root",
            "PolicyDocument": {
              "Version": "2012-10-17",
              "Statement": [
                {
                  "Effect": "Allow",
                  "Action": [ "autoscaling:Describe*", "ec2:DescribeInstances" ],
                  "Resource": "*"
                }
              ]
            }
          }
        ]
      }
    },
    "TemplateInstanceProfile": {
      "Type": "AWS::IAM::InstanceProfile",
      "Properties": {
        "Path": "/",
        "Roles": [
          {
            "Ref": "TemplateRole"
          }
        ]
      }
    }

  },


  "Outputs": {

  }
}