cwl-elasticsearch.template

{
  "AWSTemplateFormatVersion" : "2010-09-09",

  "Description" : "A sample Elasticsearch/Kibana stack that hooks up with real-time data from CloudWatch Logs using a Subscription Filter. **NOTE** This template creates one or more Amazon EC2 instances, an Amazon Kinesis stream and an Elastic Load Balancer. You will be billed for the AWS resources used if you create a stack from this template. This template requires setting the \"Create IAM resources\" parameter to True.",

  "Parameters" : {

    "KeyName": {
      "Description" : "The name of an existing key pair to enable SSH access to the EC2 instances",
      "Type": "AWS::EC2::KeyPair::KeyName"
    },

    "InstanceType" : {
      "Description" : "EC2 instance type for the Elasticsearch nodes",
      "Type" : "String",
      "Default" : "t2.medium",
      "AllowedValues" : [ 
        "t2.micro", 
        "t2.small",
        "t2.medium",
        "t2.large",
        "m4.large",
        "m4.xlarge",
        "m4.2xlarge",
        "m4.4xlarge",
        "m4.10clarge",
        "m3.medium", 
        "m3.large", 
        "m3.xlarge", 
        "m3.2xlarge", 
        "c3.large", 
        "c3.xlarge", 
        "c3.2xlarge", 
        "c3.4xlarge", 
        "c3.8xlarge", 
        "c4.large", 
        "c4.xlarge", 
        "c4.2xlarge", 
        "c4.4xlarge", 
        "c4.8xlarge", 
        "r3.large", 
        "r3.xlarge", 
        "r3.2xlarge", 
        "r3.4xlarge", 
        "r3.8xlarge", 
        "i2.xlarge", 
        "i2.2xlarge", 
        "i2.4xlarge", 
        "i2.8xlarge", 
        "d2.xlarge", 
        "d2.2xlarge", 
        "d2.4xlarge", 
        "d2.8xlarge", 
        "hi1.4xlarge", 
        "hs1.8xlarge", 
        "cr1.8xlarge", 
        "cc2.8xlarge"
       ],
      "ConstraintDescription" : "Must be a valid EC2 instance type"
    },

    "AllowedIpSource" : {
      "Description" : "The IP address range that can be used to access the Kibana interface, the Elasticsearch API, and the EC2 instances via SSH",
      "Type": "String",
      "MinLength": "9",
      "MaxLength": "18",
      "Default": "0.0.0.0/0",
      "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
      "ConstraintDescription": "Must be a valid IP CIDR range of the form x.x.x.x/x"
    },

    "ClusterSize" : {
      "Description" : "The number of Elasticsearch nodes",
      "Type" : "Number",
      "Default" : "1"
    },

    "LogGroupName": {
      "Description": "The CloudWatch Logs log group to use as the source for the subscription that feeds the Elasticsearch cluster",
      "Type": "String",
      "Default": ""
    },

    "NginxUsername" : {
      "Description": "Username to access the Kibana interface and the Elasticsearch API",
      "Type": "String",
      "MinLength" : "1"
    },

    "NginxPassword" : {
      "Description": "Password to access the Kibana interface and the Elasticsearch API",
      "Type": "String",
      "NoEcho" : "true"
    },

    "RetentionDays" : {
      "Description": "Number of days to retain logs in the Elasticsearch cluster",
      "Type": "Number",
      "Default" : "14",
      "MinValue" : "1"
    },

    "MonitorStack" : {
      "Description": "Push logs from Elasticsearch, Nginx, KCL and CloudFormation to CloudWatch Logs",
      "Type": "String",
      "Default" : "true",
      "AllowedValues" : ["true", "false"]
    },

    "LogFormat" : {
      "Description": "Choose the format that best describes the type of logs in the selected log group",
      "Type": "String",
      "Default" : "Custom",
      "AllowedValues" : ["AmazonVPCFlowLogs", "AWSLambda", "AWSCloudTrail", "Custom"]
    },

    "SubscriptionFilterPattern" : {
      "Description": "Optional filter pattern for custom log formats - Use this only if LogFormat is set to Custom",
      "Type": "String"
    },

    "KinesisShards": {
      "Description": "Number of shards to create for the Kinesis subscription stream",
      "Type": "Number",
      "Default" : "1",
      "MinValue" : "1"
    },

    "ElasticsearchShards": {
      "Description": "The number of primary shards that an Elasticsearch index should have",
      "Type": "Number",
      "Default" : "5",
      "MinValue" : "0"
    },

    "ElasticsearchReplicas": {
      "Description": "The number of replica shards (copies) that each primary Elasticsearch shard should have",
      "Type": "Number",
      "Default" : "0",
      "MinValue" : "0"
    }
  },

  "Mappings" : {
    "Constants" : {
      "S3DownloadPath"   : { "Value": "aws-cloudwatch/downloads/cloudwatch-logs-subscription-consumer" },
      "S3DownloadFile"   : { "Value": "cloudwatch-logs-subscription-consumer-1.2.0" }
    },

    "LogFormats" : {
      "AmazonVPCFlowLogs" : { "Value": "[version, account_id, interface_id, srcaddr, dstaddr, srcport, dstport, protocol, packets, bytes, start, end, action, log_status]" },
      "AWSLambda": { "Value": "[timestamp=*Z, request_id=\"*-*\", event]" },
      "AWSCloudTrail": { "Value": "" },
      "Custom": { "Value": "" }
    },

    "AWSInstanceType2Arch" : {
      "t2.micro"    : { "Arch" : "HVM64"  },
      "t2.small"    : { "Arch" : "HVM64"  },
      "t2.medium"   : { "Arch" : "HVM64"  },
      "t2.large"    : { "Arch" : "HVM64"  },
      "m4.large"    : { "Arch" : "HVM64"  },
      "m4.xlarge"   : { "Arch" : "HVM64"  },
      "m4.2xlarge"  : { "Arch" : "HVM64"  },
      "m4.4xlarge"  : { "Arch" : "HVM64"  },
      "m4.10xlarge" : { "Arch" : "HVM64"  },
      "m3.medium"   : { "Arch" : "HVM64"  },
      "m3.large"    : { "Arch" : "HVM64"  },
      "m3.xlarge"   : { "Arch" : "HVM64"  },
      "m3.2xlarge"  : { "Arch" : "HVM64"  },
      "c3.large"    : { "Arch" : "HVM64"  },
      "c3.xlarge"   : { "Arch" : "HVM64"  },
      "c3.2xlarge"  : { "Arch" : "HVM64"  },
      "c3.4xlarge"  : { "Arch" : "HVM64"  },
      "c3.8xlarge"  : { "Arch" : "HVM64"  },
      "c4.large"    : { "Arch" : "HVM64"  },
      "c4.xlarge"   : { "Arch" : "HVM64"  },
      "c4.2xlarge"  : { "Arch" : "HVM64"  },
      "c4.4xlarge"  : { "Arch" : "HVM64"  },
      "c4.8xlarge"  : { "Arch" : "HVM64"  },
      "r3.large"    : { "Arch" : "HVM64"  },
      "r3.xlarge"   : { "Arch" : "HVM64"  },
      "r3.2xlarge"  : { "Arch" : "HVM64"  },
      "r3.4xlarge"  : { "Arch" : "HVM64"  },
      "r3.8xlarge"  : { "Arch" : "HVM64"  },
      "i2.xlarge"   : { "Arch" : "HVM64"  },
      "i2.2xlarge"  : { "Arch" : "HVM64"  },
      "i2.4xlarge"  : { "Arch" : "HVM64"  },
      "i2.8xlarge"  : { "Arch" : "HVM64"  },
      "d2.xlarge"   : { "Arch" : "HVM64"  },
      "d2.2xlarge"  : { "Arch" : "HVM64"  },
      "d2.4xlarge"  : { "Arch" : "HVM64"  },
      "d2.8xlarge"  : { "Arch" : "HVM64"  },
      "hi1.4xlarge" : { "Arch" : "HVM64"  },
      "hs1.8xlarge" : { "Arch" : "HVM64"  },
      "cr1.8xlarge" : { "Arch" : "HVM64"  },
      "cc2.8xlarge" : { "Arch" : "HVM64"  }
    },

    "AWSRegionArch2AMI" : {
      "us-east-1"        : {"PV64" : "ami-1ccae774", "HVM64" : "ami-1ecae776", "HVMG2" : "ami-8c6b40e4"},
      "us-west-2"        : {"PV64" : "ami-ff527ecf", "HVM64" : "ami-e7527ed7", "HVMG2" : "ami-abbe919b"},
      "us-west-1"        : {"PV64" : "ami-d514f291", "HVM64" : "ami-d114f295", "HVMG2" : "ami-f31ffeb7"},
      "eu-west-1"        : {"PV64" : "ami-bf0897c8", "HVM64" : "ami-a10897d6", "HVMG2" : "ami-d5bc24a2"},
      "eu-central-1"     : {"PV64" : "ami-ac221fb1", "HVM64" : "ami-a8221fb5", "HVMG2" : "ami-7cd2ef61"},
      "ap-northeast-1"   : {"PV64" : "ami-27f90e27", "HVM64" : "ami-cbf90ecb", "HVMG2" : "ami-6318e863"},
      "ap-southeast-1"   : {"PV64" : "ami-acd9e8fe", "HVM64" : "ami-68d8e93a", "HVMG2" : "ami-3807376a"},
      "ap-southeast-2"   : {"PV64" : "ami-ff9cecc5", "HVM64" : "ami-fd9cecc7", "HVMG2" : "ami-89790ab3"},
      "sa-east-1"        : {"PV64" : "ami-bb2890a6", "HVM64" : "ami-b52890a8", "HVMG2" : "NOT_SUPPORTED"},
      "cn-north-1"       : {"PV64" : "ami-fa39abc3", "HVM64" : "ami-f239abcb", "HVMG2" : "NOT_SUPPORTED"}
    }
  },
  
  "Conditions" : {
    "CreateCWLForStack" : {"Fn::Equals" : [{"Ref" : "MonitorStack"}, "true"]},
    "NoKeySpecified" : {"Fn::Equals" : [{"Ref" : "KeyName"}, ""]},
    "CustomLogFormat": {"Fn::Equals" : [{"Ref" : "LogFormat"}, "Custom"]}
  },

  "Resources" : {
      "KCLLeasesTable": {
         "Type": "AWS::DynamoDB::Table",
         "Properties": {
        "TableName": { "Ref": "AWS::StackName" },
          "ProvisionedThroughput" : {
          "ReadCapacityUnits" : "10",
          "WriteCapacityUnits" : "10"
        },
        "AttributeDefinitions" : [
          {
            "AttributeName" : "leaseKey",
            "AttributeType" : "S"   
          }
        ],
        "KeySchema" : [
          {
            "AttributeName" : "leaseKey",
            "KeyType" : "HASH"
          }
        ]
      }
    },

    "KinesisSubscriptionStream": {
      "Type": "AWS::Kinesis::Stream",
      "Properties" :  {
        "ShardCount": { "Ref": "KinesisShards" }
      }
    },

    "CloudWatchLogsKinesisRole" : {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument" : {
          "Version": "2012-10-17",
          "Statement": [{
            "Effect": "Allow",
            "Principal": { 
                "Service": { "Fn::Join": ["", ["logs.", { "Ref": "AWS::Region" } ,".amazonaws.com" ]]}
            },
            "Action": "sts:AssumeRole"
          }]
        }
      }  
    },

    "CloudWatchLogsKinesisPolicy": {
      "Type": "AWS::IAM::Policy",
      "Properties": {
        "PolicyName": "CloudWatchLogsKinesisPolicy",
        "Roles": [ { "Ref": "CloudWatchLogsKinesisRole" } ],
        "PolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Action": "kinesis:PutRecord",
              "Resource": { "Fn::Join" : ["", ["arn:aws:kinesis:", { "Ref": "AWS::Region" }, ":", { "Ref": "AWS::AccountId" }, ":stream/", { "Ref": "KinesisSubscriptionStream"} ]] }
            },
            {
              "Effect": "Allow",
              "Action": "iam:PassRole",
              "Resource": { "Fn::Join" : ["", ["arn:aws:iam::", { "Ref": "AWS::AccountId" }, ":role/", { "Ref": "CloudWatchLogsKinesisRole" } ]] }
            }
          ]
        }
      }
    },

    "ElasticsearchNodeRole" : {
      "Type" : "AWS::IAM::Role",
      "Properties" : {
        "AssumeRolePolicyDocument" : {
          "Version" : "2012-10-17",
          "Statement" : [ {
            "Effect" : "Allow",
            "Principal" : {
              "Service" : [ "ec2.amazonaws.com" ]
            },
            "Action" : [ "sts:AssumeRole" ]
          } ]
        },
        "Path" : "/"
      }
    },

    "ElasticsearchRolePolicies" : {
      "Type" : "AWS::IAM::Policy",
      "Properties" : {
        "PolicyName" : "elasticsearch-node-policy",
        "PolicyDocument": {
          "Version" : "2012-10-17",
          "Statement": [ {
              "Effect": "Allow",
              "Action": "ec2:*",
              "Resource": "*"
            },
            {
              "Effect": "Allow",
              "Action": "dynamodb:*",
              "Resource": { "Fn::Join": ["", ["arn:aws:dynamodb:",{ "Ref": "AWS::Region" },":", {"Ref": "AWS::AccountId"},":table/", { "Ref": "AWS::StackName" } ]]}
            },
            {
              "Effect": "Allow",
              "Action": [
                "kinesis:Get*",
                "kinesis:List*",
                "kinesis:Describe*"
              ],
              "Resource": { "Fn::Join": ["", ["arn:aws:kinesis:", { "Ref": "AWS::Region" }, ":",{"Ref": "AWS::AccountId"},":stream/", { "Ref": "KinesisSubscriptionStream" }]]}
            },
            {
              "Effect": "Allow",
              "Action": [
                "logs:*"
              ],
              "Resource": "*"
            },
            {
              "Effect": "Allow",
              "Action": "iam:PassRole",
              "Resource": { "Fn::Join" : ["", ["arn:aws:iam::", { "Ref": "AWS::AccountId" }, ":role/", { "Ref": "CloudWatchLogsKinesisRole" } ]] }
            }
          ]
        },
        "Roles": [ { "Ref": "ElasticsearchNodeRole" } ]
      }
    },

    "ElasticsearchInstanceProfile" : {
      "Type": "AWS::IAM::InstanceProfile",
      "Properties": {
        "Path": "/",
        "Roles": [ { "Ref": "ElasticsearchNodeRole" } ]
      }
    },
    
    "VPC" : {
      "Type" : "AWS::EC2::VPC",
      "Properties" : {
        "CidrBlock" : "10.0.0.0/16",
        "Tags" : [ {"Key" : "Application", "Value" : { "Ref" : "AWS::StackId"} } ]
      }
    },

    "Subnet" : {
      "Type" : "AWS::EC2::Subnet",
      "Properties" : {
        "VpcId" : { "Ref" : "VPC" },
        "CidrBlock" : "10.0.0.0/24",
        "Tags" : [ {"Key" : "Application", "Value" : { "Ref" : "AWS::StackId"} } ]
      }
    },

    "InternetGateway" : {
      "Type" : "AWS::EC2::InternetGateway",
      "Properties" : {
        "Tags" : [ {"Key" : "Application", "Value" : { "Ref" : "AWS::StackId"} } ]
      }
    },

    "AttachGateway" : {
       "Type" : "AWS::EC2::VPCGatewayAttachment",
       "Properties" : {
         "VpcId" : { "Ref" : "VPC" },
         "InternetGatewayId" : { "Ref" : "InternetGateway" }
       }
    },

    "RouteTable" : {
      "Type" : "AWS::EC2::RouteTable",
      "Properties" : {
        "VpcId" : {"Ref" : "VPC"},
        "Tags" : [ {"Key" : "Application", "Value" : { "Ref" : "AWS::StackId"} } ]
      }
    },

    "Route" : {
      "Type" : "AWS::EC2::Route",
      "DependsOn" : "AttachGateway",
      "Properties" : {
        "RouteTableId" : { "Ref" : "RouteTable" },
        "DestinationCidrBlock" : "0.0.0.0/0",
        "GatewayId" : { "Ref" : "InternetGateway" }
      }
    },

    "SubnetRouteTableAssociation" : {
      "Type" : "AWS::EC2::SubnetRouteTableAssociation",
      "Properties" : {
        "SubnetId" : { "Ref" : "Subnet" },
        "RouteTableId" : { "Ref" : "RouteTable" }
      }
    },

    "NetworkAcl" : {
      "Type" : "AWS::EC2::NetworkAcl",
      "Properties" : {
        "VpcId" : {"Ref" : "VPC"},
        "Tags" : [ {"Key" : "Application", "Value" : { "Ref" : "AWS::StackId"} } ]
      }
    },

    "InboundHTTPNetworkAclEntry" : {
      "Type" : "AWS::EC2::NetworkAclEntry",
      "Properties" : {
        "NetworkAclId" : {"Ref" : "NetworkAcl"},
        "RuleNumber" : "100",
        "Protocol" : "6",
        "RuleAction" : "allow",
        "Egress" : "false",
        "CidrBlock" : "0.0.0.0/0",
        "PortRange" : {"From" : "80", "To" : "80"}
      }
    },

    "InboundSSHNetworkAclEntry" : {
      "Type" : "AWS::EC2::NetworkAclEntry",
      "Properties" : {
        "NetworkAclId" : {"Ref" : "NetworkAcl"},
        "RuleNumber" : "101",
        "Protocol" : "6",
        "RuleAction" : "allow",
        "Egress" : "false",
        "CidrBlock" : "0.0.0.0/0",
        "PortRange" : {"From" : "22", "To" : "22"}
      }
    },

    "InboundResponsePortsNetworkAclEntry" : {
      "Type" : "AWS::EC2::NetworkAclEntry",
      "Properties" : {
        "NetworkAclId" : {"Ref" : "NetworkAcl"},
        "RuleNumber" : "102",
        "Protocol" : "6",
        "RuleAction" : "allow",
        "Egress" : "false",
        "CidrBlock" : "0.0.0.0/0",
        "PortRange" : {"From" : "1024", "To" : "65535"}
      }
    },

    "OutBoundNetworkAclEntry" : {
      "Type" : "AWS::EC2::NetworkAclEntry",
      "Properties" : {
        "NetworkAclId" : {"Ref" : "NetworkAcl"},
        "RuleNumber" : "100",
        "Protocol" : "-1",
        "RuleAction" : "allow",
        "Egress" : "true",
        "CidrBlock" : "0.0.0.0/0"
      }
    },

    "SubnetNetworkAclAssociation" : {
      "Type" : "AWS::EC2::SubnetNetworkAclAssociation",
      "Properties" : {
        "SubnetId" : { "Ref" : "Subnet" },
        "NetworkAclId" : { "Ref" : "NetworkAcl" }
      }
    },

    "ElasticsearchSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "VpcId" : { "Ref" : "VPC" },
        "GroupDescription" : "Enable Elasticsearch access",
        "SecurityGroupIngress" : [
          {
            "IpProtocol" : "tcp", 
            "FromPort" : "8080", 
            "ToPort" : "8080",
            "SourceSecurityGroupOwnerId" : {"Fn::GetAtt" : ["ElasticsearchELB", "SourceSecurityGroup.OwnerAlias"]},
            "SourceSecurityGroupId" : { "Ref" : "ElasticsearchELBSecurityGroup" } 
          },
          {
            "IpProtocol" : "tcp", 
            "FromPort" : "22", 
            "ToPort" : "22", 
            "CidrIp" : { "Ref" : "AllowedIpSource" }
          }
        ]
      }
    },

    "ElasticsearchSecurityGroupIngress" : {
      "Type" : "AWS::EC2::SecurityGroupIngress",
      "Properties" : {
        "GroupId" : { "Ref": "ElasticsearchSecurityGroup" },
        "IpProtocol" : "tcp", 
        "FromPort" : "9300", 
        "ToPort" : "9399",
        "SourceSecurityGroupId" : { "Ref" : "ElasticsearchSecurityGroup" }
      }
    }, 

    "ElasticsearchELBSecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "VpcId" : { "Ref" : "VPC" },
        "GroupDescription" : "Enable restricted ELB access",
        "SecurityGroupIngress" : [
          {
            "IpProtocol" : "tcp", 
            "FromPort" : "80", 
            "ToPort" : "80",
            "CidrIp" : { "Ref" : "AllowedIpSource" }
          }
        ]
      }
    },

    "ElasticsearchServerGroup" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "VPCZoneIdentifier" : [{ "Ref" : "Subnet" }],
        "LaunchConfigurationName" : { "Ref" : "ElasticsearchServer" },
        "LoadBalancerNames": [  { "Ref": "ElasticsearchELB" } ],
        "MinSize" : "1",
        "MaxSize" : "10",
        "DesiredCapacity" : { "Ref" : "ClusterSize" },
        "Tags" : [ { "Key" : "type", "Value" : "elasticsearch", "PropagateAtLaunch" : "true" } ]
      }
    },

    "ElasticsearchELB" : {
      "Type": "AWS::ElasticLoadBalancing::LoadBalancer",
      "Properties" : {
        "Subnets" : [{ "Ref" : "Subnet" }],
        "CrossZone" : "true",
        "SecurityGroups": [ { "Fn::GetAtt" : [ "ElasticsearchELBSecurityGroup", "GroupId" ] } ],
        "Listeners" : [ {
          "LoadBalancerPort" : "80",
          "InstancePort" : "8080",
          "Protocol" : "HTTP"
        } ],
        "HealthCheck" : {
          "Target" : "TCP:8080",
          "HealthyThreshold" : 3,
          "UnhealthyThreshold" : 2,
          "Interval": 10,
          "Timeout": 5
        }
      }
    },

    "ElasticsearchLogs": {
      "Type": "AWS::Logs::LogGroup",
      "Condition": "CreateCWLForStack"
    },

    "NginxAccessLogs": {
      "Type": "AWS::Logs::LogGroup",
      "Condition": "CreateCWLForStack"
    },

    "NginxErrorLogs": {
      "Type": "AWS::Logs::LogGroup",
      "Condition": "CreateCWLForStack"
    },

    "CloudFormationLogs": {
      "Type": "AWS::Logs::LogGroup",
      "Condition": "CreateCWLForStack"
    },

    "KCLLogs": {
      "Type": "AWS::Logs::LogGroup",
      "Condition": "CreateCWLForStack"
    },

    "SubscriptionFilter" : {
      "Type" : "AWS::Logs::SubscriptionFilter",
      "DependsOn" : [ "KinesisSubscriptionStream", "CloudWatchLogsKinesisRole", "CloudWatchLogsKinesisPolicy", "WaitCondition" ],
      "Properties" : {
        "RoleArn" : { "Fn::GetAtt" : [ "CloudWatchLogsKinesisRole", "Arn" ] },
        "LogGroupName" : { "Ref": "LogGroupName" },
        "FilterPattern" : {
          "Fn::If" : [
            "CustomLogFormat",
            { "Ref": "SubscriptionFilterPattern" },
            { "Fn::FindInMap" : [ "LogFormats", { "Ref": "LogFormat" }, "Value" ]}
          ]},
        "DestinationArn" : {
          "Fn::Join" : [ ":", [ "arn", "aws", "kinesis",
            { "Ref" : "AWS::Region" },
            { "Ref" : "AWS::AccountId" },
            { "Fn::Join" : [ "/", [ "stream", { "Ref": "KinesisSubscriptionStream"} ] ] } ]
          ]
        }
      }
    },

    "ElasticsearchServer": {
      "Type": "AWS::AutoScaling::LaunchConfiguration",
      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config" : {
            "packages": {
              "yum": {
                "httpd": [],
                "nginx": [],
                "awslogs": []
              },
              "rpm": {
                "elasticsearch-1.6.0": "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.6.0.noarch.rpm"
              }
            },
            "commands" : {
              "00_copyNginxConfig": {
                "command": { "Fn::Join" : ["", [
                  "cp ./", { "Fn::FindInMap" : [ "Constants", "S3DownloadFile", "Value" ]} ,"/configuration/nginx/nginx.conf /etc/nginx/nginx.conf"
                 ]]}
              },
              "01_copyElasticsearchTemplates": {
                "command": { "Fn::Join" : ["", [
                  "cp -R ./", { "Fn::FindInMap" : [ "Constants", "S3DownloadFile", "Value" ]} ,"/configuration/elasticsearch/* /etc/elasticsearch/"
                ]]}
              },
              "02_updateESConfigForNode": {
                "command": { "Fn::Join" : ["", [
                  "echo \"",
                  "node.name: $(curl -silent http://169.254.169.254/latest/dynamic/instance-identity/document 2>/dev/null ",
                  "| grep instanceId ",
                  "| awk -F\\\" '{print $4}')",
                  "\" >> /etc/elasticsearch/elasticsearch.yml && ",
                  "echo \"",
                  "node.rack: $(curl -silent http://169.254.169.254/latest/dynamic/instance-identity/document 2>/dev/null ",
                  "| grep availabilityZone ",
                  "| awk -F\\\" '{print $4}')",
                  "\" >> /etc/elasticsearch/elasticsearch.yml"
                ]]}
              },
              "03_installESPlugins": {
                "command": { "Fn::Join" : ["", [
                  "/usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.6.0",
                  "&&",
                  "/usr/share/elasticsearch/bin/plugin -install lmenezes/elasticsearch-kopf/1.5.5"
                ]]}
              },
              "04_nginxAuthConfig": {
                "command": { "Fn::Join" : ["", [ 
                  "htpasswd -c -b /etc/nginx/.htpasswd ", 
                  { "Ref": "NginxUsername" }, " ", 
                  { "Ref": "NginxPassword" }, "\n"
                 ]]}
              },
              "05_startKCL": {
                "command": { "Fn::Join" : ["", [
                  "{ nohup java -DkinesisInputStream=", { "Ref": "KinesisSubscriptionStream" },
                    " -DregionName=", { "Ref": "AWS::Region" },
                    " -DappName=", { "Ref": "KCLLeasesTable" },
                    " -Dlog4j.configuration=log4j-prod.properties",
                    " -DelasticsearchClusterName=elasticsearch",
                    " -cp ", { "Fn::FindInMap" : [ "Constants", "S3DownloadFile", "Value" ]} ,"/", { "Fn::FindInMap" : [ "Constants", "S3DownloadFile", "Value" ]} ,".jar",
                    " com.amazonaws.services.logs.connectors.samples.elasticsearch.ElasticsearchConnector",
                    " > /dev/null 2>&1 & } && disown -h %1"
                  ]]},
                "cwd": "~"
              },
              "06_startKibana4": {
                "command": "{ nohup ./kibana-4.1.1-linux-x64/bin/kibana > /dev/null 2>&1 & } && disown -h %1",
                "cwd": "~"
              },
              "07_getPipAndCurator": {
                "command": "python get-pip.py && /usr/local/bin/pip install elasticsearch-curator",
                "cwd": "/pipstuff"
              },
              "08_CWLStateDir" : {
                "command" : "mkdir -p /var/awslogs/state"
              }
            },

            "files": {
              "/etc/elasticsearch/elasticsearch.yml" : {
                "content" : { "Fn::Join" : ["", [
                  "cloud.aws.region:\n",
                  " ", { "Ref" : "AWS::Region" }, "\n",
                  "discovery:\n",
                  " type: ec2\n",
                  "\n",
                  "discovery.ec2.groups: ", { "Ref" : "ElasticsearchSecurityGroup" }, "\n",
                  "discovery.ec2.tag.type: elasticsearch\n",
                  "\n",
                  "cloud.node.auto_attributes: true\n"
                ]]}
              },
              "/etc/logrotate.d/elasticsearch": {
                   "content": { "Fn::Join" : ["", [
                  "/var/log/elasticsearch/elasticsearch_index_indexing_slowlog.log\n",
                  "/var/log/elasticsearch/elasticsearch_index_search_slowlog.log\n",
                  "/var/log/elasticsearch/elasticsearch.log\n",
                  "{\n",
                  "    daily\n",
                  "    rotate 5\n",
                  "    copytruncate\n",
                  "    dateext\n",
                  "    compress\n",
                  "}\n"
                ]]}
              },
              "/etc/logrotate.d/nginx": {
                   "content": { "Fn::Join" : ["", [
                  "/var/log/nginx/access.log\n",
                  "/var/log/nginx/error.log\n",
                  "{\n",
                  "    daily\n",
                  "    rotate 5\n",
                  "    copytruncate\n",
                  "    dateext\n",
                  "    compress\n",
                  "}\n"
                ]]}
              },
              "/etc/logrotate.d/cloudwatch-logs-subscription-consumer": {
                   "content": { "Fn::Join" : ["", [
                  "/var/log/cloudwatch-logs-subscription-consumer.log\n",
                  "{\n",
                  "    daily\n",
                  "    rotate 5\n",
                  "    copytruncate\n",
                  "    dateext\n",
                  "    compress\n",
                  "}\n"
                ]]}
              },                 
              "/etc/awslogs/awscli.conf": {
                   "content": { "Fn::Join" : ["", [
                  "[plugins]\n",
                  "cwlogs = cwlogs\n",
                  "[default]\n",
                  "region = ", { "Ref": "AWS::Region"} ,"\n"
                ]]}
              },
              "/pipstuff/get-pip.py": {
                "source": "https://bootstrap.pypa.io/get-pip.py"
              },
              "/etc/awslogs/awslogs.conf": {
                "Fn::If": [
                  "CreateCWLForStack",
                  {
                    "content": { "Fn::Join": [ "", [
                       "[general]\n",
                       "state_file= /var/awslogs/state/agent-state\n",

                      "[/var/log/cfn-init.log]\n",
                      "file = /var/log/cfn-init.log\n",
                      "log_group_name = ", { "Ref": "CloudFormationLogs" }, "\n",
                      "log_stream_name = {instance_id}\n",

                      "[/var/log/elasticsearch/elasticsearch.log]\n",
                      "file = /var/log/elasticsearch/elasticsearch.log\n",
                      "log_group_name = ", { "Ref": "ElasticsearchLogs" }, "\n",
                      "log_stream_name = {instance_id}\n",

                      "[/var/log/nginx/access.log]\n",
                      "file = /var/log/nginx/access.log\n",
                      "log_group_name = ", { "Ref": "NginxAccessLogs" }, "\n",
                      "log_stream_name = {instance_id}\n",

                      "[/var/log/nginx/error.log]\n",
                      "file = /var/log/nginx/error.log\n",
                      "log_group_name = ", { "Ref": "NginxErrorLogs" }, "\n",
                      "log_stream_name = {instance_id}\n",

                      "[/var/log/cloudwatch-logs-subscription-consumer.log]\n",
                      "file = /var/log/cloudwatch-logs-subscription-consumer.log\n",
                      "log_group_name = ", { "Ref": "KCLLogs" }, "\n",
                      "log_stream_name = {instance_id}\n"
                   ]]},
                   "mode": "000444",
                   "owner": "root",
                   "group": "root"
                   },
                 {
                   "content": "# Find original defaults in .bak file"
                 }
                ]
              } 
            },

            "services": {
              "sysvinit": {
                "nginx": {
                  "enabled": true,
                  "ensureRunning": true,
                  "files": [ "/etc/nginx/nginx.conf" ],
                  "sources": [ "usr/share/nginx/html" ]
                },
                "elasticsearch": {
                  "enabled": true,
                  "ensureRunning": true,
                  "files": [ "/etc/elasticsearch/elasticsearch.yml", "/etc/elasticsearch/logging.yml", "/etc/sysconfig/elasticsearch" ],
                  "sources": [ "/usr/share/elasticsearch/bin" ]
                },
                "awslogs": {
                  "Fn::If": [
                    "CreateCWLForStack",
                    {
                      "enabled" : "true", 
                      "ensureRunning" : "true",
                      "files" : [ "/etc/awslogs/awslogs.conf" ]
                    },
                    "AWS::NoValue"
                  ]
                }
              }
            }
          }
        }
      },

      "Properties": {
        "ImageId" : { "Fn::FindInMap" : [ "AWSRegionArch2AMI", { "Ref" : "AWS::Region" }, { "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "InstanceType" }, "Arch" ] } ] },
        "InstanceType"   : { "Ref" : "InstanceType" },
        "SecurityGroups" : [ {"Ref" : "ElasticsearchSecurityGroup"} ],
        "KeyName"        : { "Ref" : "KeyName" },
        "IamInstanceProfile" : { "Ref" : "ElasticsearchInstanceProfile" },
        "AssociatePublicIpAddress" : true,
        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
          "#!/bin/bash\n",
          "yum update -y aws-cfn-bootstrap\n",
          "yum update -y aws-cli\n",

          "# Helper function\n",
          "function error_exit\n",
          "{\n",
          "  /opt/aws/bin/cfn-signal -e 1 -r \"$1\" '", { "Ref" : "WaitHandle" }, "'\n",
          "  exit 1\n",
          "}\n",

          "cd ~\n",

          "# KCL and config files\n",
          "wget https://s3.amazonaws.com/", { "Fn::FindInMap" : [ "Constants", "S3DownloadPath", "Value" ]} ,"/", { "Fn::FindInMap" : [ "Constants", "S3DownloadFile", "Value" ]} ,"-cfn.zip || error_exit \"Failed to get cf package\" \n",
          "unzip ", { "Fn::FindInMap" : [ "Constants", "S3DownloadFile", "Value" ]} ,"-cfn.zip || error_exit \"Failed to unzip cf package\" \n",

          "# Get Kibana 4\n",
          "wget https://download.elastic.co/kibana/kibana/kibana-4.1.1-linux-x64.tar.gz || error_exit \"Failed to download Kibana 4\" \n",
          "tar -xf kibana-4.1.1-linux-x64.tar.gz || error_exit \"Kibana 4 untar failed\" \n",

          "# Config\n",
          "/opt/aws/bin/cfn-init -s ", { "Ref" : "AWS::StackId" }, " -r ElasticsearchServer ",
          "    --region ", { "Ref" : "AWS::Region" }, " -v || error_exit \"failed to run cfn-init\"\n",

          "# Allow 60% of total mem as heap size for cluster \n",
          "freemem=$(free -m | grep Mem | awk '{ printf \"%1.0f\\n\", 0.6*$2; }')",
          " && sed -i \"s/#ES_HEAP_SIZE=.*/ES_HEAP_SIZE=$(echo $freemem)m/g\" /etc/sysconfig/elasticsearch",
          " || error_exit \"Failed to change heap size for es \" \n",

          "# Get Kibana 3\n",
          "wget https://download.elasticsearch.org/kibana/kibana/kibana-3.1.2.zip || error_exit \"Failed to download Kibana 3\" \n",
          "unzip kibana-3.1.2.zip || error_exit \"Kibana 3 unzip failed\" \n",
          "mkdir -p /usr/share/nginx/html/kibana3 || error_exit \"Failed to create Kibana 3 directory\" \n",
          "mv kibana-3.1.2/* /usr/share/nginx/html/kibana3 || error_exit \"Failed to move Kibana 3 assets\" \n",
          "sed -i 's/elasticsearch:.*/elasticsearch: \"http:\\/\\/\" + window.location.hostname,/g' /usr/share/nginx/html/kibana3/config.js || error_exit \"Failed to update Kibana 3 config\"\n",

          "# Add static Kibana 3 dashboards\n",
          "cp -f ", { "Fn::FindInMap" : [ "Constants", "S3DownloadFile", "Value" ]} ,"/configuration/kibana/* /usr/share/nginx/html/kibana3/app/dashboards/ || error_exit \"Failed to copy the Kibana 3 static dashboards\" \n",

          "# Update Elasticsearch index templates\n",
          "sed -i '5i\\ \\ \\ \\ \\ \\ \"index.number_of_replicas\" : ", { "Ref": "ElasticsearchReplicas" } ,",' /etc/elasticsearch/templates/cwl_template.json || error_exit \"Failed update cwl_template.json\"\n",
          "sed -i '5i\\ \\ \\ \\ \\ \\ \"number_of_shards\" : ", { "Ref": "ElasticsearchShards" } ,",' /etc/elasticsearch/templates/cwl_template.json || error_exit \"Failed update cwl_template.json\"\n",
          "sed -i '5i\\ \\ \\ \\ \\ \\ \"index.number_of_replicas\" : ", { "Ref": "ElasticsearchReplicas" } ,",' /etc/elasticsearch/templates/kibana3_template.json || error_exit \"Failed update kibana3_template.json\"\n",
          "sed -i '5i\\ \\ \\ \\ \\ \\ \"index.number_of_replicas\" : ", { "Ref": "ElasticsearchReplicas" } ,",' /etc/elasticsearch/templates/kibana4_template.json || error_exit \"Failed update kibana4_template.json\"\n",

          "# Set hourly cron for curator\n",
          "echo \"0 * * * * /usr/local/bin/curator --host localhost --port 9200 --debug delete indices --prefix \\\"cwl-\\\" --time-unit days --older-than ",
          { "Ref" : "RetentionDays" },
          " --timestring \\\"\\%Y.\\%m.\\%d\\\"\" >> mycron || error_exit \"Failed to create cron file\" \n",

          "crontab mycron && rm -f mycron || error_exit \"Failed to set cron \" \n",

          "# Signal success\n",
          "/opt/aws/bin/cfn-signal -e $? '", { "Ref" : "WaitHandle" }, "'\n"
        ]]}}
      }
    },

    "WaitHandle" : {
      "Type" : "AWS::CloudFormation::WaitConditionHandle"
    },

    "WaitCondition" : {
      "Type" : "AWS::CloudFormation::WaitCondition",
      "DependsOn" : "ElasticsearchServer",
      "Properties" : {
        "Handle" : {"Ref" : "WaitHandle"},
        "Timeout" : "600"
      }
    }
  },

  "Outputs" : {
    "Kibana3" : {
      "Description" : "Kibana 3 (Includes Sample Dashboards)",
      "Value" :  { "Fn::Join" : [ "", [ "http://", { "Fn::GetAtt" : [ "ElasticsearchELB", "DNSName" ]}, "/kibana3/#"]]}
    },
    "Kibana4" : {
      "Description" : "Kibana 4",
      "Value" :  { "Fn::Join" : [ "", [ "http://", { "Fn::GetAtt" : [ "ElasticsearchELB", "DNSName" ]}, "/kibana4/#"]]}
    },
    "Elasticsearch" : {
      "Description" : "Elasticsearch API",
      "Value" :  { "Fn::Join" : [ "", [ "http://", { "Fn::GetAtt" : [ "ElasticsearchELB", "DNSName" ]}, "/?pretty"]]}
    },
    "Kopf" : {
      "Description" : "Kopf Admin Tool",
      "Value" :  { "Fn::Join" : [ "", [ "http://", { "Fn::GetAtt" : [ "ElasticsearchELB", "DNSName" ]}, "/_plugin/kopf"]]}
    }
  }
}