From 239373f9bb191da038d83d3910b31f8b2c99ad86 Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Wed, 19 Sep 2018 21:38:02 +0200 Subject: [PATCH] aws/ELBs: merge tnc with api_internal and cleanup Remove the TNC ELB with and move all associated resources to the api_internal ELB. This also allows to cleanup the DNS A record and the security group for the TNC. It also changes the FQDN for the TNC, which is now the same as for the API, though it remains exclusive to the internal zone. Configure the ELB to listen on the TNC port directly. --- installer/pkg/config-generator/ignition.go | 10 +---- installer/pkg/config-generator/tls.go | 2 +- modules/aws/master/main.tf | 6 --- modules/aws/master/variables.tf | 4 -- modules/aws/vpc/master-elb.tf | 49 +++++++-------------- modules/aws/vpc/outputs.tf | 18 ++------ modules/aws/vpc/sg-elb.tf | 50 +++++----------------- pkg/asset/ignition/node.go | 2 +- pkg/asset/tls/helper.go | 4 +- steps/infra/aws/main.tf | 17 +------- steps/infra/libvirt/main.tf | 1 - 11 files changed, 37 insertions(+), 126 deletions(-) diff --git a/installer/pkg/config-generator/ignition.go b/installer/pkg/config-generator/ignition.go index 162a81c516e..695e44984ec 100644 --- a/installer/pkg/config-generator/ignition.go +++ b/installer/pkg/config-generator/ignition.go @@ -125,19 +125,13 @@ func (c *ConfigGenerator) embedUserBlock(ignCfg *ignconfigtypes.Config) { func (c *ConfigGenerator) getTNCURL(role string, query string) string { var u string - - // cloud platforms put this behind a load balancer which remaps ports; - // libvirt doesn't do that - use the tnc port directly - port := 80 - if c.Platform == config.PlatformLibvirt { - port = 49500 - } + port := 49500 if role == "master" || role == "worker" { u = func() *url.URL { return &url.URL{ Scheme: "https", - Host: fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port), + Host: fmt.Sprintf("%s-api.%s:%d", c.Name, c.BaseDomain, port), Path: fmt.Sprintf("/config/%s", role), RawQuery: query, } diff --git a/installer/pkg/config-generator/tls.go b/installer/pkg/config-generator/tls.go index 48130fd1b07..ee64315190c 100644 --- a/installer/pkg/config-generator/tls.go +++ b/installer/pkg/config-generator/tls.go @@ -240,7 +240,7 @@ func (c *ConfigGenerator) GenerateTLSConfig(clusterDir string) error { } // MachineConfigServer certs - mcsDomain := fmt.Sprintf("%s-tnc.%s", c.Name, c.BaseDomain) + mcsDomain := fmt.Sprintf("%s-api.%s", c.Name, c.BaseDomain) cfg = &tls.CertCfg{ ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, DNSNames: []string{mcsDomain}, diff --git a/modules/aws/master/main.tf b/modules/aws/master/main.tf index c0bc084e456..bbfd7284480 100644 --- a/modules/aws/master/main.tf +++ b/modules/aws/master/main.tf @@ -112,12 +112,6 @@ resource "aws_instance" "master" { ), var.extra_tags)}" } -resource "aws_elb_attachment" "masters_tnc" { - count = "${var.private_endpoints ? var.instance_count : 0}" - elb = "${var.elb_tnc_id}" - instance = "${aws_instance.master.*.id[count.index]}" -} - resource "aws_elb_attachment" "masters_internal" { count = "${var.private_endpoints ? var.instance_count : 0}" elb = "${var.elb_api_internal_id}" diff --git a/modules/aws/master/variables.tf b/modules/aws/master/variables.tf index 5a5348cd3eb..32e0715a99f 100644 --- a/modules/aws/master/variables.tf +++ b/modules/aws/master/variables.tf @@ -56,10 +56,6 @@ variable "public_endpoints" { default = true } -variable "elb_tnc_id" { - type = "string" -} - variable "elb_api_internal_id" { type = "string" } diff --git a/modules/aws/vpc/master-elb.tf b/modules/aws/vpc/master-elb.tf index 6e04a547646..300bdbd5df5 100644 --- a/modules/aws/vpc/master-elb.tf +++ b/modules/aws/vpc/master-elb.tf @@ -1,36 +1,3 @@ -resource "aws_elb" "tnc" { - count = "${var.private_master_endpoints ? 1 : 0}" - name = "${var.cluster_name}-tnc" - subnets = ["${local.master_subnet_ids}"] - internal = true - security_groups = ["${aws_security_group.tnc.id}"] - - idle_timeout = 3600 - connection_draining = true - connection_draining_timeout = 300 - - listener { - instance_port = 49500 - instance_protocol = "tcp" - lb_port = 80 - lb_protocol = "tcp" - } - - health_check { - healthy_threshold = 2 - unhealthy_threshold = 2 - timeout = 3 - target = "TCP:49500" - interval = 5 - } - - tags = "${merge(map( - "Name", "${var.cluster_name}-int", - "kubernetes.io/cluster/${var.cluster_name}", "owned", - "tectonicClusterID", "${var.cluster_id}" - ), var.extra_tags)}" -} - resource "aws_elb" "api_internal" { count = "${var.private_master_endpoints ? 1 : 0}" name = "${var.cluster_name}-int" @@ -49,6 +16,13 @@ resource "aws_elb" "api_internal" { lb_protocol = "tcp" } + listener { + instance_port = 49500 + instance_protocol = "tcp" + lb_port = 49500 + lb_protocol = "tcp" + } + health_check { healthy_threshold = 2 unhealthy_threshold = 2 @@ -57,6 +31,15 @@ resource "aws_elb" "api_internal" { interval = 5 } + # TODO: we only have on health_check per ELB but need to check the following too + # health_check { + # healthy_threshold = 2 + # unhealthy_threshold = 2 + # timeout = 3 + # target = "TCP:49500" + # interval = 5 + # } + tags = "${merge(map( "Name", "${var.cluster_name}-int", "kubernetes.io/cluster/${var.cluster_name}", "owned", diff --git a/modules/aws/vpc/outputs.tf b/modules/aws/vpc/outputs.tf index 11cc81e6884..cab4d40f9b6 100644 --- a/modules/aws/vpc/outputs.tf +++ b/modules/aws/vpc/outputs.tf @@ -42,15 +42,11 @@ output "aws_elb_console_id" { value = "${aws_elb.console.id}" } -output "aws_elb_tnc_id" { - value = "${aws_elb.tnc.0.id}" -} - output "aws_lbs" { - value = ["${compact(concat(aws_elb.api_internal.*.id, list(aws_elb.console.id), aws_elb.api_external.*.id, aws_elb.tnc.*.id))}"] + value = ["${compact(concat(aws_elb.api_internal.*.id, list(aws_elb.console.id), aws_elb.api_external.*.id))}"] } -output "aws_api_external_dns_name" { +output "aws_elb_api_external_dns_name" { value = "${element(concat(aws_elb.api_external.*.dns_name, list("")), 0)}" } @@ -58,7 +54,7 @@ output "aws_elb_api_external_zone_id" { value = "${element(concat(aws_elb.api_external.*.zone_id, list("")), 0)}" } -output "aws_api_internal_dns_name" { +output "aws_elb_api_internal_dns_name" { value = "${element(concat(aws_elb.api_internal.*.dns_name, list("")), 0)}" } @@ -73,11 +69,3 @@ output "aws_console_dns_name" { output "aws_elb_console_zone_id" { value = "${aws_elb.console.zone_id}" } - -output "aws_elb_tnc_dns_name" { - value = "${element(concat(aws_elb.tnc.*.dns_name, list("")), 0)}" -} - -output "aws_elb_tnc_zone_id" { - value = "${element(concat(aws_elb.tnc.*.zone_id, list("")), 0)}" -} diff --git a/modules/aws/vpc/sg-elb.tf b/modules/aws/vpc/sg-elb.tf index d40f9bdf3b5..f8548e8a93e 100644 --- a/modules/aws/vpc/sg-elb.tf +++ b/modules/aws/vpc/sg-elb.tf @@ -1,43 +1,3 @@ -resource "aws_security_group" "tnc" { - vpc_id = "${data.aws_vpc.cluster_vpc.id}" - - tags = "${merge(map( - "Name", "${var.cluster_name}_tnc_sg", - "kubernetes.io/cluster/${var.cluster_name}", "owned", - "tectonicClusterID", "${var.cluster_id}" - ), var.extra_tags)}" -} - -resource "aws_security_group_rule" "tnc_egress" { - type = "egress" - security_group_id = "${aws_security_group.tnc.id}" - - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "tnc_ingress_http" { - type = "ingress" - security_group_id = "${aws_security_group.tnc.id}" - - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 80 - to_port = 80 -} - -resource "aws_security_group_rule" "tnc_ingress_https" { - type = "ingress" - security_group_id = "${aws_security_group.tnc.id}" - - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - from_port = 443 - to_port = 443 -} - resource "aws_security_group" "api" { vpc_id = "${data.aws_vpc.cluster_vpc.id}" @@ -68,6 +28,16 @@ resource "aws_security_group_rule" "api_ingress_console" { to_port = 6443 } +resource "aws_security_group_rule" "tnc_ingress" { + type = "ingress" + security_group_id = "${aws_security_group.api.id}" + + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 49500 + to_port = 49500 +} + resource "aws_security_group" "console" { vpc_id = "${data.aws_vpc.cluster_vpc.id}" diff --git a/pkg/asset/ignition/node.go b/pkg/asset/ignition/node.go index b15511ecb9a..f7e097ce906 100644 --- a/pkg/asset/ignition/node.go +++ b/pkg/asset/ignition/node.go @@ -67,7 +67,7 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro Source: func() *url.URL { return &url.URL{ Scheme: "https", - Host: fmt.Sprintf("%s-tnc.%s:49500", installConfig.Name, installConfig.BaseDomain), + Host: fmt.Sprintf("%s-api.%s:49500", installConfig.Name, installConfig.BaseDomain), Path: fmt.Sprintf("/config/%s", role), RawQuery: query, } diff --git a/pkg/asset/tls/helper.go b/pkg/asset/tls/helper.go index b4647d16d54..283c05ea798 100644 --- a/pkg/asset/tls/helper.go +++ b/pkg/asset/tls/helper.go @@ -80,9 +80,9 @@ func genIPAddressesForOpenshiftAPIServerCertKey(cfg *types.InstallConfig) ([]net } func genDNSNamesForTNCCertKey(cfg *types.InstallConfig) ([]string, error) { - return []string{fmt.Sprintf("%s-tnc.%s", cfg.Name, cfg.BaseDomain)}, nil + return []string{fmt.Sprintf("%s-api.%s", cfg.Name, cfg.BaseDomain)}, nil } func genSubjectForTNCCertKey(cfg *types.InstallConfig) (pkix.Name, error) { - return pkix.Name{CommonName: fmt.Sprintf("%s-tnc.%s", cfg.Name, cfg.BaseDomain)}, nil + return pkix.Name{CommonName: fmt.Sprintf("%s-api.%s", cfg.Name, cfg.BaseDomain)}, nil } diff --git a/steps/infra/aws/main.tf b/steps/infra/aws/main.tf index 7d61bef40aa..7ab49631256 100644 --- a/steps/infra/aws/main.tf +++ b/steps/infra/aws/main.tf @@ -18,7 +18,6 @@ provider "aws" { module "masters" { source = "../../../modules/aws/master" - elb_tnc_id = "${module.vpc.aws_elb_tnc_id}" elb_api_internal_id = "${module.vpc.aws_elb_api_internal_id}" elb_api_external_id = "${module.vpc.aws_elb_api_external_id}" elb_console_id = "${module.vpc.aws_elb_console_id}" @@ -51,9 +50,9 @@ module "iam" { module "dns" { source = "../../../modules/dns/route53" - api_external_elb_dns_name = "${module.vpc.aws_api_external_dns_name}" + api_external_elb_dns_name = "${module.vpc.aws_elb_api_external_dns_name}" api_external_elb_zone_id = "${module.vpc.aws_elb_api_external_zone_id}" - api_internal_elb_dns_name = "${module.vpc.aws_api_internal_dns_name}" + api_internal_elb_dns_name = "${module.vpc.aws_elb_api_internal_dns_name}" api_internal_elb_zone_id = "${module.vpc.aws_elb_api_internal_zone_id}" api_ip_addresses = "${module.vpc.aws_lbs}" base_domain = "${var.tectonic_base_domain}" @@ -100,18 +99,6 @@ resource "aws_route53_record" "etcd_a_nodes" { records = ["${module.masters.ip_addresses[count.index]}"] } -resource "aws_route53_record" "tectonic_tnc_a" { - zone_id = "${local.private_zone_id}" - name = "${var.tectonic_cluster_name}-tnc.${var.tectonic_base_domain}" - type = "A" - - alias { - name = "${module.vpc.aws_elb_tnc_dns_name}" - zone_id = "${module.vpc.aws_elb_tnc_zone_id}" - evaluate_target_health = true - } -} - resource "aws_route53_zone" "tectonic_int" { count = "${local.private_endpoints ? "${var.tectonic_aws_external_private_zone == "" ? 1 : 0 }" : 0}" vpc_id = "${module.vpc.vpc_id}" diff --git a/steps/infra/libvirt/main.tf b/steps/infra/libvirt/main.tf index 6dbb7dd4758..026ad77df4e 100644 --- a/steps/infra/libvirt/main.tf +++ b/steps/infra/libvirt/main.tf @@ -80,7 +80,6 @@ resource "libvirt_domain" "master" { locals { "hostnames" = [ "${var.tectonic_cluster_name}-api", - "${var.tectonic_cluster_name}-tnc", ] }