aws/ELBs: merge tnc with api_internal and cleanup

Remove the TNC ELB with and move all associated resources to the api_internal ELB. This also allows to cleanup the DNS A record and the security group for the TNC. It also changes the FQDN for the TNC, which is now the same as for the API, though it remains exclusive to the internal zone. Configure the ELB to listen on the TNC port directly.
openshift · Sep 21, 2018 · 239373f · 239373f
1 parent da9b472
commit 239373f
Show file tree

Hide file tree

Showing 11 changed files with 37 additions and 126 deletions.
diff --git a/installer/pkg/config-generator/ignition.go b/installer/pkg/config-generator/ignition.go
@@ -125,19 +125,13 @@ func (c *ConfigGenerator) embedUserBlock(ignCfg *ignconfigtypes.Config) {
 
 func (c *ConfigGenerator) getTNCURL(role string, query string) string {
 	var u string
-
-	// cloud platforms put this behind a load balancer which remaps ports;
-	// libvirt doesn't do that - use the tnc port directly
-	port := 80
-	if c.Platform == config.PlatformLibvirt {
-		port = 49500
-	}
+	port := 49500
 
 	if role == "master" || role == "worker" {
 		u = func() *url.URL {
 			return &url.URL{
 				Scheme:   "https",
-				Host:     fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port),
+				Host:     fmt.Sprintf("%s-api.%s:%d", c.Name, c.BaseDomain, port),
 				Path:     fmt.Sprintf("/config/%s", role),
 				RawQuery: query,
 			}

diff --git a/installer/pkg/config-generator/tls.go b/installer/pkg/config-generator/tls.go
@@ -240,7 +240,7 @@ func (c *ConfigGenerator) GenerateTLSConfig(clusterDir string) error {
 	}
 
 	// MachineConfigServer certs
-	mcsDomain := fmt.Sprintf("%s-tnc.%s", c.Name, c.BaseDomain)
+	mcsDomain := fmt.Sprintf("%s-api.%s", c.Name, c.BaseDomain)
 	cfg = &tls.CertCfg{
 		ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		DNSNames:     []string{mcsDomain},

diff --git a/modules/aws/master/main.tf b/modules/aws/master/main.tf
@@ -112,12 +112,6 @@ resource "aws_instance" "master" {
   ), var.extra_tags)}"
 }
 
-resource "aws_elb_attachment" "masters_tnc" {
-  count    = "${var.private_endpoints ? var.instance_count : 0}"
-  elb      = "${var.elb_tnc_id}"
-  instance = "${aws_instance.master.*.id[count.index]}"
-}
-
 resource "aws_elb_attachment" "masters_internal" {
   count    = "${var.private_endpoints ? var.instance_count : 0}"
   elb      = "${var.elb_api_internal_id}"

diff --git a/modules/aws/master/variables.tf b/modules/aws/master/variables.tf
@@ -56,10 +56,6 @@ variable "public_endpoints" {
   default     = true
 }
 
-variable "elb_tnc_id" {
-  type = "string"
-}
-
 variable "elb_api_internal_id" {
   type = "string"
 }

diff --git a/modules/aws/vpc/master-elb.tf b/modules/aws/vpc/master-elb.tf
@@ -1,36 +1,3 @@
-resource "aws_elb" "tnc" {
-  count           = "${var.private_master_endpoints ? 1 : 0}"
-  name            = "${var.cluster_name}-tnc"
-  subnets         = ["${local.master_subnet_ids}"]
-  internal        = true
-  security_groups = ["${aws_security_group.tnc.id}"]
-
-  idle_timeout                = 3600
-  connection_draining         = true
-  connection_draining_timeout = 300
-
-  listener {
-    instance_port     = 49500
-    instance_protocol = "tcp"
-    lb_port           = 80
-    lb_protocol       = "tcp"
-  }
-
-  health_check {
-    healthy_threshold   = 2
-    unhealthy_threshold = 2
-    timeout             = 3
-    target              = "TCP:49500"
-    interval            = 5
-  }
-
-  tags = "${merge(map(
-      "Name", "${var.cluster_name}-int",
-      "kubernetes.io/cluster/${var.cluster_name}", "owned",
-      "tectonicClusterID", "${var.cluster_id}"
-    ), var.extra_tags)}"
-}
-
 resource "aws_elb" "api_internal" {
   count           = "${var.private_master_endpoints ? 1 : 0}"
   name            = "${var.cluster_name}-int"
@@ -49,6 +16,13 @@ resource "aws_elb" "api_internal" {
     lb_protocol       = "tcp"
   }
 
+  listener {
+    instance_port     = 49500
+    instance_protocol = "tcp"
+    lb_port           = 49500
+    lb_protocol       = "tcp"
+  }
+
   health_check {
     healthy_threshold   = 2
     unhealthy_threshold = 2
@@ -57,6 +31,15 @@ resource "aws_elb" "api_internal" {
     interval            = 5
   }
 
+  # TODO: we only have on health_check per ELB but need to check the following too
+  # health_check {
+  #   healthy_threshold   = 2
+  #   unhealthy_threshold = 2
+  #   timeout             = 3
+  #   target              = "TCP:49500"
+  #   interval            = 5
+  # }
+
   tags = "${merge(map(
       "Name", "${var.cluster_name}-int",
       "kubernetes.io/cluster/${var.cluster_name}", "owned",

diff --git a/modules/aws/vpc/outputs.tf b/modules/aws/vpc/outputs.tf
@@ -42,23 +42,19 @@ output "aws_elb_console_id" {
   value = "${aws_elb.console.id}"
 }
 
-output "aws_elb_tnc_id" {
-  value = "${aws_elb.tnc.0.id}"
-}
-
 output "aws_lbs" {
-  value = ["${compact(concat(aws_elb.api_internal.*.id, list(aws_elb.console.id), aws_elb.api_external.*.id, aws_elb.tnc.*.id))}"]
+  value = ["${compact(concat(aws_elb.api_internal.*.id, list(aws_elb.console.id), aws_elb.api_external.*.id))}"]
 }
 
-output "aws_api_external_dns_name" {
+output "aws_elb_api_external_dns_name" {
   value = "${element(concat(aws_elb.api_external.*.dns_name, list("")), 0)}"
 }
 
 output "aws_elb_api_external_zone_id" {
   value = "${element(concat(aws_elb.api_external.*.zone_id, list("")), 0)}"
 }
 
-output "aws_api_internal_dns_name" {
+output "aws_elb_api_internal_dns_name" {
   value = "${element(concat(aws_elb.api_internal.*.dns_name, list("")), 0)}"
 }
 
@@ -73,11 +69,3 @@ output "aws_console_dns_name" {
 output "aws_elb_console_zone_id" {
   value = "${aws_elb.console.zone_id}"
 }
-
-output "aws_elb_tnc_dns_name" {
-  value = "${element(concat(aws_elb.tnc.*.dns_name, list("")), 0)}"
-}
-
-output "aws_elb_tnc_zone_id" {
-  value = "${element(concat(aws_elb.tnc.*.zone_id, list("")), 0)}"
-}
diff --git a/modules/aws/vpc/sg-elb.tf b/modules/aws/vpc/sg-elb.tf
@@ -1,43 +1,3 @@
-resource "aws_security_group" "tnc" {
-  vpc_id = "${data.aws_vpc.cluster_vpc.id}"
-
-  tags = "${merge(map(
-      "Name", "${var.cluster_name}_tnc_sg",
-      "kubernetes.io/cluster/${var.cluster_name}", "owned",
-      "tectonicClusterID", "${var.cluster_id}"
-    ), var.extra_tags)}"
-}
-
-resource "aws_security_group_rule" "tnc_egress" {
-  type              = "egress"
-  security_group_id = "${aws_security_group.tnc.id}"
-
-  from_port   = 0
-  to_port     = 0
-  protocol    = "-1"
-  cidr_blocks = ["0.0.0.0/0"]
-}
-
-resource "aws_security_group_rule" "tnc_ingress_http" {
-  type              = "ingress"
-  security_group_id = "${aws_security_group.tnc.id}"
-
-  protocol    = "tcp"
-  cidr_blocks = ["0.0.0.0/0"]
-  from_port   = 80
-  to_port     = 80
-}
-
-resource "aws_security_group_rule" "tnc_ingress_https" {
-  type              = "ingress"
-  security_group_id = "${aws_security_group.tnc.id}"
-
-  protocol    = "tcp"
-  cidr_blocks = ["0.0.0.0/0"]
-  from_port   = 443
-  to_port     = 443
-}
-
 resource "aws_security_group" "api" {
   vpc_id = "${data.aws_vpc.cluster_vpc.id}"
 
@@ -68,6 +28,16 @@ resource "aws_security_group_rule" "api_ingress_console" {
   to_port     = 6443
 }
 
+resource "aws_security_group_rule" "tnc_ingress" {
+  type              = "ingress"
+  security_group_id = "${aws_security_group.api.id}"
+
+  protocol    = "tcp"
+  cidr_blocks = ["0.0.0.0/0"]
+  from_port   = 49500
+  to_port     = 49500
+}
+
 resource "aws_security_group" "console" {
   vpc_id = "${data.aws_vpc.cluster_vpc.id}"
 

diff --git a/pkg/asset/ignition/node.go b/pkg/asset/ignition/node.go
@@ -67,7 +67,7 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro
 					Source: func() *url.URL {
 						return &url.URL{
 							Scheme:   "https",
-							Host:     fmt.Sprintf("%s-tnc.%s:49500", installConfig.Name, installConfig.BaseDomain),
+							Host:     fmt.Sprintf("%s-api.%s:49500", installConfig.Name, installConfig.BaseDomain),
 							Path:     fmt.Sprintf("/config/%s", role),
 							RawQuery: query,
 						}

diff --git a/pkg/asset/tls/helper.go b/pkg/asset/tls/helper.go
@@ -80,9 +80,9 @@ func genIPAddressesForOpenshiftAPIServerCertKey(cfg *types.InstallConfig) ([]net
 }
 
 func genDNSNamesForTNCCertKey(cfg *types.InstallConfig) ([]string, error) {
-	return []string{fmt.Sprintf("%s-tnc.%s", cfg.Name, cfg.BaseDomain)}, nil
+	return []string{fmt.Sprintf("%s-api.%s", cfg.Name, cfg.BaseDomain)}, nil
 }
 
 func genSubjectForTNCCertKey(cfg *types.InstallConfig) (pkix.Name, error) {
-	return pkix.Name{CommonName: fmt.Sprintf("%s-tnc.%s", cfg.Name, cfg.BaseDomain)}, nil
+	return pkix.Name{CommonName: fmt.Sprintf("%s-api.%s", cfg.Name, cfg.BaseDomain)}, nil
 }
diff --git a/steps/infra/aws/main.tf b/steps/infra/aws/main.tf
@@ -18,7 +18,6 @@ provider "aws" {
 module "masters" {
   source = "../../../modules/aws/master"
 
-  elb_tnc_id          = "${module.vpc.aws_elb_tnc_id}"
   elb_api_internal_id = "${module.vpc.aws_elb_api_internal_id}"
   elb_api_external_id = "${module.vpc.aws_elb_api_external_id}"
   elb_console_id      = "${module.vpc.aws_elb_console_id}"
@@ -51,9 +50,9 @@ module "iam" {
 module "dns" {
   source = "../../../modules/dns/route53"
 
-  api_external_elb_dns_name = "${module.vpc.aws_api_external_dns_name}"
+  api_external_elb_dns_name = "${module.vpc.aws_elb_api_external_dns_name}"
   api_external_elb_zone_id  = "${module.vpc.aws_elb_api_external_zone_id}"
-  api_internal_elb_dns_name = "${module.vpc.aws_api_internal_dns_name}"
+  api_internal_elb_dns_name = "${module.vpc.aws_elb_api_internal_dns_name}"
   api_internal_elb_zone_id  = "${module.vpc.aws_elb_api_internal_zone_id}"
   api_ip_addresses          = "${module.vpc.aws_lbs}"
   base_domain               = "${var.tectonic_base_domain}"
@@ -100,18 +99,6 @@ resource "aws_route53_record" "etcd_a_nodes" {
   records = ["${module.masters.ip_addresses[count.index]}"]
 }
 
-resource "aws_route53_record" "tectonic_tnc_a" {
-  zone_id = "${local.private_zone_id}"
-  name    = "${var.tectonic_cluster_name}-tnc.${var.tectonic_base_domain}"
-  type    = "A"
-
-  alias {
-    name                   = "${module.vpc.aws_elb_tnc_dns_name}"
-    zone_id                = "${module.vpc.aws_elb_tnc_zone_id}"
-    evaluate_target_health = true
-  }
-}
-
 resource "aws_route53_zone" "tectonic_int" {
   count         = "${local.private_endpoints ? "${var.tectonic_aws_external_private_zone == "" ? 1 : 0 }" : 0}"
   vpc_id        = "${module.vpc.vpc_id}"

diff --git a/steps/infra/libvirt/main.tf b/steps/infra/libvirt/main.tf
@@ -80,7 +80,6 @@ resource "libvirt_domain" "master" {
 locals {
   "hostnames" = [
     "${var.tectonic_cluster_name}-api",
-    "${var.tectonic_cluster_name}-tnc",
   ]
 }