Switch to Application Load Balancers.

remind101 · Aug 11, 2016 · 978a6c9 · 978a6c9
1 parent b7ecf89
commit 978a6c9
Show file tree

Hide file tree

Showing 9 changed files with 287 additions and 228 deletions.
diff --git a/cmd/empire/factories.go b/cmd/empire/factories.go
@@ -149,6 +149,7 @@ func newCloudFormationScheduler(db *empire.DB, c *Context) (*cloudformation.Sche
 	}
 
 	t := &cloudformation.EmpireTemplate{
+		VpcId:                   c.String(FlagELBVpcId),
 		Cluster:                 c.String(FlagECSCluster),
 		InternalSecurityGroupID: c.String(FlagELBSGPrivate),
 		ExternalSecurityGroupID: c.String(FlagELBSGPublic),

diff --git a/cmd/empire/main.go b/cmd/empire/main.go
@@ -53,6 +53,7 @@ const (
 
 	FlagELBSGPrivate = "elb.sg.private"
 	FlagELBSGPublic  = "elb.sg.public"
+	FlagELBVpcId     = "elb.vpc.id"
 
 	FlagEC2SubnetsPrivate = "ec2.subnets.private"
 	FlagEC2SubnetsPublic  = "ec2.subnets.public"
@@ -278,6 +279,11 @@ var EmpireFlags = []cli.Flag{
 		Usage:  "The ELB security group to assign public load balancers",
 		EnvVar: "EMPIRE_ELB_SG_PUBLIC",
 	},
+	cli.StringFlag{
+		Name:   FlagELBVpcId,
+		Usage:  "The comma separated private subnet ids",
+		EnvVar: "EMPIRE_ELB_VPC_ID",
+	},
 	cli.StringSliceFlag{
 		Name:   FlagEC2SubnetsPrivate,
 		Value:  &cli.StringSlice{},

diff --git a/docs/cloudformation.json b/docs/cloudformation.json
@@ -507,16 +507,7 @@
             {
               "Effect": "Allow",
               "Action": [
-                "elasticloadbalancing:DeleteLoadBalancer",
-                "elasticloadbalancing:CreateLoadBalancer",
-                "elasticloadbalancing:DescribeLoadBalancers",
-                "elasticloadbalancing:DescribeTags",
-                "elasticloadbalancing:ConfigureHealthCheck",
-                "elasticloadbalancing:ModifyLoadBalancerAttributes",
-                "elasticloadbalancing:SetLoadBalancerListenerSSLCertificate",
-                "elasticloadbalancing:CreateLoadBalancerListeners",
-                "elasticloadbalancing:DeleteLoadBalancerListeners",
-                "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+                "elasticloadbalancing:*"
               ],
               "Resource": ["*"]
             },

diff --git a/pkg/troposphere/troposphere.go b/pkg/troposphere/troposphere.go
@@ -45,6 +45,7 @@ type Output struct {
 // Resource represents a CloudFormation Resource.
 type Resource struct {
 	Condition  interface{} `json:"Condition,omitempty"`
+	DependsOn  interface{} `json:"DependsOn,omitempty"`
 	Properties interface{} `json:"Properties,omitempty"`
 	Type       interface{} `json:"Type,omitempty"`
 	Version    interface{} `json:"Version,omitempty"`

diff --git a/scheduler/cloudformation/template.go b/scheduler/cloudformation/template.go
@@ -59,6 +59,10 @@ type EmpireTemplate struct {
 	// The ECS cluster to run the services in.
 	Cluster string
 
+	// The VPC to create ALB target groups within. Should be the same VPC
+	// that ECS services will run within.
+	VpcId string
+
 	// The hosted zone to add CNAME's to.
 	HostedZone *route53.HostedZone
 
@@ -93,6 +97,9 @@ func (t *EmpireTemplate) Validate() error {
 		return errors.New(fmt.Sprintf("%s is required", n))
 	}
 
+	if t.VpcId == "" {
+		return r("VpcId")
+	}
 	if t.Cluster == "" {
 		return r("Cluster")
 	}
@@ -321,6 +328,7 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
 
 	var portMappings []*PortMappingProperties
 
+	var serviceDependencies []string
 	loadBalancers := []map[string]interface{}{}
 	if p.Exposure != nil {
 		scheme := schemeInternal
@@ -333,73 +341,91 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
 			subnets = t.ExternalSubnetIDs
 		}
 
-		instancePort := fmt.Sprintf("%s%dInstancePort", key, ContainerPort)
-		tmpl.Resources[instancePort] = troposphere.Resource{
-			Type:    "Custom::InstancePort",
-			Version: "1.0",
-			Properties: map[string]interface{}{
-				"ServiceToken": t.CustomResourcesTopic,
-			},
-		}
-
-		listeners := []map[string]interface{}{
-			map[string]interface{}{
-				"LoadBalancerPort": 80,
-				"Protocol":         "http",
-				"InstancePort":     GetAtt(instancePort, "InstancePort"),
-				"InstanceProtocol": "http",
-			},
-		}
-
-		if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok {
-			var cert interface{}
-			if _, err := arn.Parse(e.Cert); err == nil {
-				cert = e.Cert
-			} else {
-				cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert)
-			}
-
-			listeners = append(listeners, map[string]interface{}{
-				"LoadBalancerPort": 443,
-				"Protocol":         "https",
-				"InstancePort":     GetAtt(instancePort, "InstancePort"),
-				"SSLCertificateId": cert,
-				"InstanceProtocol": "http",
-			})
-		}
-
 		portMappings = append(portMappings, &PortMappingProperties{
 			ContainerPort: ContainerPort,
-			HostPort:      GetAtt(instancePort, "InstancePort"),
+			HostPort:      0,
 		})
 		p.Env["PORT"] = fmt.Sprintf("%d", ContainerPort)
 
 		loadBalancer := fmt.Sprintf("%sLoadBalancer", key)
-		loadBalancers = append(loadBalancers, map[string]interface{}{
-			"ContainerName":    p.Type,
-			"ContainerPort":    ContainerPort,
-			"LoadBalancerName": Ref(loadBalancer),
-		})
 		tmpl.Resources[loadBalancer] = troposphere.Resource{
-			Type: "AWS::ElasticLoadBalancing::LoadBalancer",
+			Type: "AWS::ElasticLoadBalancingV2::LoadBalancer",
 			Properties: map[string]interface{}{
 				"Scheme":         scheme,
 				"SecurityGroups": []string{sg},
 				"Subnets":        subnets,
-				"Listeners":      listeners,
-				"CrossZone":      true,
 				"Tags": []map[string]string{
 					map[string]string{
 						"Key":   "empire.app.process",
 						"Value": p.Type,
 					},
 				},
-				"ConnectionDrainingPolicy": map[string]interface{}{
-					"Enabled": true,
-					"Timeout": defaultConnectionDrainingTimeout,
+			},
+		}
+
+		targetGroup := fmt.Sprintf("%sTargetGroup", key)
+		tmpl.Resources[targetGroup] = troposphere.Resource{
+			Type: "AWS::ElasticLoadBalancingV2::TargetGroup",
+			Properties: map[string]interface{}{
+				"Port":     65535, // Not used. ECS sets a port override when registering targets.
+				"Protocol": "HTTP",
+				"VpcId":    t.VpcId,
+			},
+		}
+
+		httpListener := fmt.Sprintf("%sPort%dListener", loadBalancer, 80)
+		tmpl.Resources[httpListener] = troposphere.Resource{
+			Type: "AWS::ElasticLoadBalancingV2::Listener",
+			Properties: map[string]interface{}{
+				"LoadBalancerArn": Ref(loadBalancer),
+				"Port":            80,
+				"Protocol":        "HTTP",
+				"DefaultActions": []interface{}{
+					map[string]interface{}{
+						"TargetGroupArn": Ref(targetGroup),
+						"Type":           "forward",
+					},
 				},
 			},
 		}
+		serviceDependencies = append(serviceDependencies, httpListener)
+
+		if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok {
+			var cert interface{}
+			if _, err := arn.Parse(e.Cert); err == nil {
+				cert = e.Cert
+			} else {
+				cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert)
+			}
+
+			httpsListener := fmt.Sprintf("%sPort%dListener", loadBalancer, 443)
+			tmpl.Resources[httpsListener] = troposphere.Resource{
+				Type: "AWS::ElasticLoadBalancingV2::Listener",
+				Properties: map[string]interface{}{
+					"Certificates": []interface{}{
+						map[string]interface{}{
+							"CertificateArn": cert,
+						},
+					},
+					"LoadBalancerArn": GetAtt(loadBalancer, "Arn"),
+					"Port":            443,
+					"Protocol":        "HTTPS",
+					"DefaultActions": []interface{}{
+						map[string]interface{}{
+							"TargetGroupArn": Ref(targetGroup),
+							"Type":           "forward",
+						},
+					},
+				},
+			}
+			serviceDependencies = append(serviceDependencies, httpsListener)
+		}
+
+		loadBalancers = append(loadBalancers, map[string]interface{}{
+			"ContainerName":  p.Type,
+			"ContainerPort":  ContainerPort,
+			"TargetGroupArn": Ref(targetGroup),
+		})
 
 		if p.Type == "web" {
 			tmpl.Resources["CNAME"] = troposphere.Resource{
@@ -421,7 +447,6 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
 	containerDefinition.DockerLabels[restartLabel] = Ref(restartParameter)
 	containerDefinition.PortMappings = portMappings
 
-	service := fmt.Sprintf("%sService", key)
 	serviceProperties := map[string]interface{}{
 		"Cluster":        t.Cluster,
 		"DesiredCount":   Ref(scaleParameter(p.Type)),
@@ -433,11 +458,18 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
 	if len(loadBalancers) > 0 {
 		serviceProperties["Role"] = t.ServiceRole
 	}
-	tmpl.Resources[service] = troposphere.Resource{
-		Type:       ecsServiceType,
-		Properties: serviceProperties,
+	service := troposphere.NamedResource{
+		Name: fmt.Sprintf("%sService", key),
+		Resource: troposphere.Resource{
+			Type:       ecsServiceType,
+			Properties: serviceProperties,
+		},
+	}
+	if len(serviceDependencies) > 0 {
+		service.Resource.DependsOn = serviceDependencies
 	}
-	return service
+	tmpl.AddResource(service)
+	return service.Name
 }
 
 // If the ServiceRole option is not an ARN, it will return a CloudFormation

diff --git a/scheduler/cloudformation/templates/basic.json b/scheduler/cloudformation/templates/basic.json
@@ -122,33 +122,8 @@
       },
       "Type": "AWS::Route53::RecordSet"
     },
-    "web8080InstancePort": {
-      "Properties": {
-        "ServiceToken": "sns topic arn"
-      },
-      "Type": "Custom::InstancePort",
-      "Version": "1.0"
-    },
     "webLoadBalancer": {
       "Properties": {
-        "ConnectionDrainingPolicy": {
-          "Enabled": true,
-          "Timeout": 30
-        },
-        "CrossZone": true,
-        "Listeners": [
-          {
-            "InstancePort": {
-              "Fn::GetAtt": [
-                "web8080InstancePort",
-                "InstancePort"
-              ]
-            },
-            "InstanceProtocol": "http",
-            "LoadBalancerPort": 80,
-            "Protocol": "http"
-          }
-        ],
         "Scheme": "internal",
         "SecurityGroups": [
           "sg-e7387381"
@@ -164,9 +139,30 @@
           }
         ]
       },
-      "Type": "AWS::ElasticLoadBalancing::LoadBalancer"
+      "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer"
+    },
+    "webLoadBalancerPort80Listener": {
+      "Properties": {
+        "DefaultActions": [
+          {
+            "TargetGroupArn": {
+              "Ref": "webTargetGroup"
+            },
+            "Type": "forward"
+          }
+        ],
+        "LoadBalancerArn": {
+          "Ref": "webLoadBalancer"
+        },
+        "Port": 80,
+        "Protocol": "HTTP"
+      },
+      "Type": "AWS::ElasticLoadBalancingV2::Listener"
     },
     "webService": {
+      "DependsOn": [
+        "webLoadBalancerPort80Listener"
+      ],
       "Properties": {
         "Cluster": "cluster",
         "DesiredCount": {
@@ -176,8 +172,8 @@
           {
             "ContainerName": "web",
             "ContainerPort": 8080,
-            "LoadBalancerName": {
-              "Ref": "webLoadBalancer"
+            "TargetGroupArn": {
+              "Ref": "webTargetGroup"
             }
           }
         ],
@@ -190,6 +186,14 @@
       },
       "Type": "Custom::ECSService"
     },
+    "webTargetGroup": {
+      "Properties": {
+        "Port": 65535,
+        "Protocol": "HTTP",
+        "VpcId": ""
+      },
+      "Type": "AWS::ElasticLoadBalancingV2::TargetGroup"
+    },
     "webTaskDefinition": {
       "Properties": {
         "ContainerDefinitions": [
@@ -229,12 +233,7 @@
             "PortMappings": [
               {
                 "ContainerPort": 8080,
-                "HostPort": {
-                  "Fn::GetAtt": [
-                    "web8080InstancePort",
-                    "InstancePort"
-                  ]
-                }
+                "HostPort": 0
               }
             ],
             "Ulimits": [