cloudformation_update.json

{
  "AWSTemplateFormatVersion" : "2010-09-09",

  "Description" : "Cloud Formation for Pivotal HD 3.0 and HAWQ cluster with Ambari",

  "Parameters" : {

    "KeyName": {
      "Default" : "phd",
      "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
      "Type": "String",
      "MinLength": "1",
      "MaxLength": "255",
      "AllowedPattern" : "[\\x20-\\x7E]*",
      "ConstraintDescription" : "can contain only ASCII characters."
    },

    "ManagementNodeInstanceType" : {
      "Description" : "Management Node",
      "Type" : "String",
      "Default" : "c3.4xlarge",
      "AllowedValues" : ["c3.4xlarge","m3.2xlarge"],
      "ConstraintDescription" : "must be a valid EC2 instance type."
    },

    "ClusterNodeInstanceType" : {
      "Description" : "Cluster Node",
      "Type" : "String",
      "Default" : "cc2.8xlarge",
      "AllowedValues" : ["cc2.8xlarge","cg1.4xlarge","i2.8xlarge","hs1.8xlarge","c3.4xlarge","m3.2xlarge"],
      "ConstraintDescription" : "must be a valid EC2 instance type."
    },
    "SSHLocation" : {
      "Description" : " The IP address range that can be used to SSH to the EC2 instances",
      "Type": "String",
      "MinLength": "9",
      "MaxLength": "18",
      "Default": "0.0.0.0/0",
      "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
    },


    "ClusterNodes" : {
      "Description" : "Maximum number of slave nodes",
      "Type": "Number",
      "Default": 10,
       "AllowedValues" : [2,6,10,18,34]
    },
    "AccessKey": {
      "Description": "Access Key ID",
      "Type": "String",
      "NoEcho": "True"
    },
    "SecretKey": {
      "Description": "Access Key ID",
      "Type": "String",
      "NoEcho": "True"
    }
  },

  "Mappings" : {
    "AWSInstanceType2Arch" : {
       "cc2.8xlarge" : { "Arch" : "64HVM" },
      "cg1.4xlarge" : { "Arch" : "64HVM" },
	  "i2.8xlarge" : { "Arch" : "64HVM" },
      "hs1.8xlarge" : { "Arch" : "64HVM" },
    "c3.4xlarge" : { "Arch" : "64HVM" },
      "m3.2xlarge": {"Arch": "64HVM"}


    },

    "AWSRegionArch2AMI" : {
      "us-east-1": {"64HVM": "ami-1293e87a"}
    }
  },
  "Resources": {
    "ManagementNode": {
      "Type": "AWS::EC2::Instance",
      "Metadata" : {

      },
      "Properties": {
        "ImageId" : { "Fn::FindInMap" : [ "AWSRegionArch2AMI", { "Ref" : "AWS::Region" },
                          { "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "ManagementNodeInstanceType" }, "Arch" ] } ] },
        "InstanceType"   : { "Ref" : "ManagementNodeInstanceType" },
        "SecurityGroups" : [ {"Ref" : "PHDSecurityGroup"} ],
        "BlockDeviceMappings": [
          {
            "DeviceName": "/dev/xvdb",
            "VirtualName": "ephemeral0"
          }
        ],


        "KeyName"        : { "Ref" : "KeyName" },
        "Tags" : [{
          "Key"   : "Name",
          "Value" : { "Fn::Join"  : [ "-" , [ { "Ref" : "AWS::StackName" }, "managementnode" ]]}
        }],
        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
          "#!/bin/bash -v\n",
          "stack_name=",
          {"Ref": "AWS::StackName"},
          "\n",
          "access_key=",
          {"Ref": "AccessKey"},
          "\n",
          "secret_key=",
          {"Ref": "SecretKey"},
          "\n",

          "\n",
          "yum clean all \n",
          "yum -y remove *openjdk \n",

          "wget --no-check-certificate --no-cookies --header 'Cookie: oraclelicense=accept-securebackup-cookie' http://download.oracle.com/otn-pub/java/jdk/7u67-b01/jdk-7u67-linux-x64.rpm \n",
      	  "wget --no-check-certificate --no-cookies --header 'Cookie: oraclelicense=accept-securebackup-cookie' http://download.oracle.com/otn-pub/java/jdk/7u67-b01/jdk-7u67-linux-x64.tar.gz \n",
		  "wget --no-check-certificate http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip \n",

		  "yum -y install jdk-7u67-linux-x64.rpm \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/setup.py \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/phd3.json \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/PHD3-RepoPrepare.py \n",
          "python setup.py install \n",
          "python PHD3-RepoPrepare.py get --key=$access_key --secret=$secret_key --stack=$stack_name\n",

          "yum install -y ambari-server\n",

		  "mv jdk-7u67-linux-x64.tar.gz /var/lib/ambari-server/resources/ \n",
		  "mv UnlimitedJCEPolicyJDK7.zip /var/lib/ambari-server/resources/ \n",

          "ambari-server setup -s\n",
          "yum install -y hawq-plugin >> phd.log\n",
          "ambari-server start >> phd.log \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/SetRepos.py \n",
          "python SetRepos.py \n",
          "yum install -y ambari-agent\n",
          "ambari-agent start\n",

          "MY_IP=`(curl http://169.254.169.254/latest/meta-data/local-hostname)`\n",
          "number_of_slaves=", { "Ref" : "ClusterNodes" } , "\n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/6-node-blueprint.json \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/10-node-blueprint.json \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/AgentRegistration.py \n",
          "python AgentRegistration.py monitor --agents=$number_of_slaves\n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/AmbariBlueprint.py \n",
          "python AmbariBlueprint.py hosts --key=$access_key --secret=$secret_key --stack=$stack_name\n",
          "python AmbariBlueprint.py install>> phd.log \n"
        ]
        ]
        }
        }
      }
    },
    "ClusterNodeLaunchConfiguration": {
      "Type": "AWS::AutoScaling::LaunchConfiguration",
      "Metadata" : {
      },
      "Properties": {
        "ImageId" : { "Fn::FindInMap" : [ "AWSRegionArch2AMI", { "Ref" : "AWS::Region" },
                          { "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "ClusterNodeInstanceType" }, "Arch" ] } ] },
        "InstanceType"   : { "Ref" : "ClusterNodeInstanceType" },
        "SecurityGroups" : [ {"Ref" : "PHDSecurityGroup"} ],
        "BlockDeviceMappings" : [
         {
           "DeviceName"  : "/dev/xvdb",
           "VirtualName" : "ephemeral0"
         },
         {
           "DeviceName"  : "/dev/xvdc",
           "VirtualName" : "ephemeral1"
         },
         {
           "DeviceName"  : "/dev/xvdd",
           "VirtualName" : "ephemeral2"
         },
         {
           "DeviceName"  : "/dev/xvde",
           "VirtualName" : "ephemeral3"
         }
       ],
        "KeyName"        : { "Ref" : "KeyName" },
        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
          "#!/bin/bash -v\n",
          "stack_name=",
          {"Ref": "AWS::StackName"},
          "\n",
          "access_key=",
          {"Ref": "AccessKey"},
          "\n",
          "secret_key=",
          {"Ref": "SecretKey"},
          "\n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/setup.py \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/phd3.json \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/ClusterNodePrepare.py \n",
          "wget https://raw.githubusercontent.com/dbbaskette/phd3-hawq-aws/master/PackageManager.py \n",
          "python setup.py install \n",
          "python ClusterNodePrepare.py prep --key=$access_key --secret=$secret_key --stack=$stack_name --disks=4 --ambari=",
          {
            "Fn::GetAtt": [
              "ManagementNode",
              "PrivateDnsName"
            ]
          },
          " \n"
        ]
        ]
        }
        }
      }
    },

    "ClusterNodesAutoScalingGroup": {
      "Type": "AWS::AutoScaling::AutoScalingGroup",
      "Properties": {
        "AvailabilityZones": { "Fn::GetAZs": "" },
        "LaunchConfigurationName": { "Ref": "ClusterNodeLaunchConfiguration" },
        "MinSize": { "Ref": "ClusterNodes" },
        "MaxSize": { "Ref": "ClusterNodes" },
        "Tags" : [{
                    "Key"   : "Name",
                    "Value" : { "Fn::Join"  : [ "-" , [ { "Ref" : "AWS::StackName" }, "clusternode" ]]},
                    "PropagateAtLaunch" : "true"
                  }]
      },
      "DependsOn" : "ManagementNode"
    },

    "PHDSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Enable HTTP access via port 80 and SSH access",
        "SecurityGroupIngress" : [
          {
            "IpProtocol": "-1",
            "FromPort": "1",
            "ToPort": "65536",
            "CidrIp": "0.0.0.0/0"
          }

        ]
      }
    }
  },
  "Outputs" : {
    "AmbariNode" : {
      "Value" : { "Fn::Join" : ["", ["http://", { "Fn::GetAtt" : [ "ManagementNode", "PublicIp" ]}, ":8080/"]] },
      "Description" : "PHD Ambari Host node"
    }
  }
}