openstack: Remove the Service VM

The experimental OpenStack backend used to create an extra server running DNS and load balancer services that the cluster needed. OpenStack does not always come with DNSaaS or LBaaS so we had to provide the functionality the OpenShift cluster depends on (e.g. the etcd SRV records, the api-int records & load balancing, etc.). This approach is undesirable for two reasons: first, it adds an extra node that the other IPI platforms do not need. Second, this node is a single point of failure. The Baremetal platform has faced the same issues and they have solved them with a few virtual IP addresses managed by keepalived in combination with coredns static pod running on every node using the mDNS protocol to update records as new nodes are added or removed and a similar static pod haproxy to load balance the control plane internally. The VIPs are defined here in the installer and they use the PlatformStatus field to be passed to the necessary machine-config-operator fields: openshift/api#374 The Bare Metal IPI Networking Infrastructure document is applicable here as well: https://github.com/openshift/installer/blob/master/docs/design/baremetal/networking-infrastructure.md There is also a great oportunity to share some of the configuration files and scripts here. This change needs several other pull requests: Keepalived plus the coredns & haproxy static pods in the MCO: openshift/machine-config-operator/pull/740 Passing the API and DNS VIPs through the installer: openshift#1998 Co-authored-by: Emilio Garcia <egarcia@redhat.com> Co-authored-by: John Trowbridge <trown@redhat.com> Co-authored-by: Martin Andre <m.andre@redhat.com> Co-authored-by: Tomas Sedovic <tsedovic@redhat.com> Massive thanks to the Bare Metal and oVirt people!
tomassedovic · Jul 26, 2019 · e3efd68 · e3efd68
1 parent e63c1f9
commit e3efd68
Show file tree

Hide file tree

Showing 23 changed files with 234 additions and 587 deletions.
diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template
@@ -19,6 +19,8 @@ fi
 MACHINE_CONFIG_OPERATOR_IMAGE=$(podman run --quiet --rm ${release} image machine-config-operator)
 MACHINE_CONFIG_OSCONTENT=$(podman run --quiet --rm ${release} image machine-os-content)
 MACHINE_CONFIG_ETCD_IMAGE=$(podman run --quiet --rm ${release} image etcd)
+# FIXME(shadower): without this, the etcd containers later on keep failing with our custom MCO. Investigate what's goin on.
+podman pull --quiet $MACHINE_CONFIG_ETCD_IMAGE
 MACHINE_CONFIG_KUBE_CLIENT_AGENT_IMAGE=$(podman run --quiet --rm ${release} image kube-client-agent)
 MACHINE_CONFIG_INFRA_IMAGE=$(podman run --quiet --rm ${release} image pod)
 
@@ -33,6 +35,12 @@ OPENSHIFT_HYPERSHIFT_IMAGE=$(podman run --quiet --rm ${release} image hypershift
 OPENSHIFT_HYPERKUBE_IMAGE=$(podman run --quiet --rm ${release} image hyperkube)
 
 CLUSTER_BOOTSTRAP_IMAGE=$(podman run --quiet --rm ${release} image cluster-bootstrap)
+KEEPALIVED_IMAGE=$(podman run --quiet --rm ${release} image keepalived-ipfailover)
+COREDNS_IMAGE=$(podman run --quiet --rm ${release} image coredns)
+MDNS_PUBLISHER_IMAGE=$(podman run --quiet --rm ${release} image mdns-publisher)
+HAPROXY_IMAGE=$(podman run --quiet --rm ${release} image haproxy-router)
+RUNTIMECFG_IMAGE=$(podman run --quiet --rm ${release} image baremetal-runtimecfg)
+
 
 # Now, as early as possible we replace the pause image and reload crio to use it, to ensure
 # that we're using the pause image from our payload just like the primary cluster.
@@ -194,19 +202,32 @@ then
 			--machine-config-operator-image=${MACHINE_CONFIG_OPERATOR_IMAGE} \
 			--machine-config-oscontent-image=${MACHINE_CONFIG_OSCONTENT} \
 			--infra-image=${MACHINE_CONFIG_INFRA_IMAGE} \
+			--keepalived-image=${KEEPALIVED_IMAGE} \
+			--coredns-image=${COREDNS_IMAGE} \
+			--mdns-publisher-image=${MDNS_PUBLISHER_IMAGE} \
+			--haproxy-image=${HAPROXY_IMAGE} \
+			--baremetal-runtimecfg-image=${RUNTIMECFG_IMAGE} \
 			--cloud-config-file=/assets/manifests/cloud-provider-config.yaml
 
 	# Bootstrap MachineConfigController uses /etc/mcc/bootstrap/manifests/ dir to
 	# 1. read the controller config rendered by MachineConfigOperator
 	# 2. read the default MachineConfigPools rendered by MachineConfigOperator
 	# 3. read any additional MachineConfigs that are needed for the default MachineConfigPools.
-	mkdir --parents /etc/mcc/bootstrap /etc/mcs/bootstrap /etc/kubernetes/manifests
+	mkdir --parents /etc/mcc/bootstrap /etc/mcs/bootstrap /etc/kubernetes/manifests /etc/kubernetes/static-pod-resources
 	cp mco-bootstrap/bootstrap/manifests/* /etc/mcc/bootstrap/
 	cp openshift/* /etc/mcc/bootstrap/
 	# 4. read ImageContentSourcePolicy objects generated by the installer
 	cp manifests/* /etc/mcc/bootstrap/
 	cp auth/kubeconfig-kubelet /etc/mcs/kubeconfig
 	cp mco-bootstrap/bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/
+    if [ -d mco-bootstrap/baremetal/manifests ]; then
+        cp mco-bootstrap/baremetal/manifests/* /etc/kubernetes/manifests/
+        cp -r mco-bootstrap/baremetal/static-pod-resources/* /etc/kubernetes/static-pod-resources/
+    fi
+    if [ -d mco-bootstrap/openstack/manifests ]; then
+        cp mco-bootstrap/openstack/manifests/* /etc/kubernetes/manifests/
+        cp -r mco-bootstrap/openstack/static-pod-resources/* /etc/kubernetes/static-pod-resources/
+    fi
 	cp mco-bootstrap/manifests/* manifests/
 
 	# /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets.

diff --git a/data/data/openstack/bootstrap/main.tf b/data/data/openstack/bootstrap/main.tf
@@ -18,40 +18,73 @@ data "ignition_config" "redirect" {
 
   files = [
     data.ignition_file.hostname.id,
-    data.ignition_file.bootstrap_ifcfg.id,
+    data.ignition_file.dns_conf.id,
+    data.ignition_file.dhcp_conf.id,
+    data.ignition_file.hosts.id,
   ]
 }
 
-data "ignition_file" "bootstrap_ifcfg" {
+data "ignition_file" "dhcp_conf" {
   filesystem = "root"
-  mode       = "420" // 0644
-  path       = "/etc/sysconfig/network-scripts/ifcfg-eth0"
+  mode       = "420"
+  path       = "/etc/NetworkManager/conf.d/dhcp-client.conf"
 
   content {
     content = <<EOF
-DEVICE="eth0"
-BOOTPROTO="dhcp"
-ONBOOT="yes"
-TYPE="Ethernet"
-PERSISTENT_DHCLIENT="yes"
-DNS1="${var.service_vm_fixed_ip}"
-PEERDNS="no"
-NM_CONTROLLED="yes"
+[main]
+dhcp=dhclient
 EOF
+  }
+}
 
+data "ignition_file" "dns_conf" {
+  filesystem = "root"
+  mode = "420"
+  path = "/etc/dhcp/dhclient.conf"
+
+  # FIXME(mandre) this will likely cause delay with bootstrap node networking
+  # until the master come up and are able to serve DNS queries.  Not sure the
+  # bootstrap is trying to resolve anything it doesn't have in its hosts
+  # file...
+  # BareMetal solved this by running coredns on the bootstrap node
+  #
+  # NOTE(shadower) bootstrap's waiting for the etcd cluster seems to
+  # always fail the first time because of this. The second attempt
+  # succeeds, but we should probably run the core dns there too so
+  # that:
+  # 1. We don't show spurious errors in the logs
+  # 2. Align better with what the baremetal platfrorm is doing
+  content {
+    content = <<EOF
+send dhcp-client-identifier = hardware;
+prepend domain-name-servers ${var.node_dns_ip};
+EOF
   }
 }
 
 data "ignition_file" "hostname" {
   filesystem = "root"
-  mode = "420" // 0644
-  path = "/etc/hostname"
+  mode       = "420" // 0644
+  path       = "/etc/hostname"
 
   content {
     content = <<EOF
 ${var.cluster_id}-bootstrap
 EOF
+  }
+}
+
+data "ignition_file" "hosts" {
+  filesystem = "root"
+  mode = "420" // 0644
+  path = "/etc/hosts"
 
+  content {
+    content = <<EOF
+127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
+${var.api_int_ip} api-int.${var.cluster_domain} api.${var.cluster_domain}
+EOF
   }
 }
 
@@ -81,4 +114,3 @@ resource "openstack_compute_instance_v2" "bootstrap" {
     openshiftClusterID = var.cluster_id
   }
 }
-
diff --git a/data/data/openstack/bootstrap/variables.tf b/data/data/openstack/bootstrap/variables.tf
@@ -33,7 +33,10 @@ variable "bootstrap_port_id" {
   description = "The subnet ID for the bootstrap node."
 }
 
-variable "service_vm_fixed_ip" {
+variable "api_int_ip" {
   type = string
 }
 
+variable "node_dns_ip" {
+  type = string
+}
diff --git a/data/data/openstack/main.tf b/data/data/openstack/main.tf
@@ -22,72 +22,54 @@ provider "openstack" {
   user_name           = var.openstack_credentials_user_name
 }
 
-module "service" {
-  source = "./service"
+module "bootstrap" {
+  source = "./bootstrap"
 
   swift_container   = openstack_objectstorage_container_v1.container.name
   cluster_id        = var.cluster_id
   cluster_domain    = var.cluster_domain
   image_name        = var.openstack_base_image
   flavor_name       = var.openstack_master_flavor_name
   ignition          = var.ignition_bootstrap
-  lb_floating_ip    = var.openstack_lb_floating_ip
-  service_port_id   = module.topology.service_port_id
-  service_port_ip   = module.topology.service_port_ip
-  master_ips        = module.topology.master_ips
-  master_port_names = module.topology.master_port_names
-  bootstrap_ip      = module.topology.bootstrap_port_ip
-}
-
-module "bootstrap" {
-  source = "./bootstrap"
-
-  swift_container     = openstack_objectstorage_container_v1.container.name
-  cluster_id          = var.cluster_id
-  cluster_domain      = var.cluster_domain
-  image_name          = var.openstack_base_image
-  flavor_name         = var.openstack_master_flavor_name
-  ignition            = var.ignition_bootstrap
-  bootstrap_port_id   = module.topology.bootstrap_port_id
-  service_vm_fixed_ip = module.topology.service_vm_fixed_ip
+  bootstrap_port_id = module.topology.bootstrap_port_id
+  api_int_ip        = var.openstack_api_int_ip
+  node_dns_ip       = var.openstack_node_dns_ip
 }
 
 module "masters" {
   source = "./masters"
 
-  base_image          = var.openstack_base_image
-  bootstrap_ip        = module.topology.bootstrap_port_ip
-  cluster_id          = var.cluster_id
-  cluster_domain      = var.cluster_domain
-  flavor_name         = var.openstack_master_flavor_name
-  instance_count      = var.master_count
-  lb_floating_ip      = var.openstack_lb_floating_ip
-  master_ips          = module.topology.master_ips
-  master_port_ids     = module.topology.master_port_ids
-  master_port_names   = module.topology.master_port_names
-  user_data_ign       = var.ignition_master
-  service_vm_fixed_ip = module.topology.service_vm_fixed_ip
-  api_int_ip          = var.openstack_api_int_ip
-  node_dns_ip         = var.openstack_node_dns_ip
+  base_image      = var.openstack_base_image
+  bootstrap_ip    = module.topology.bootstrap_port_ip
+  cluster_id      = var.cluster_id
+  cluster_domain  = var.cluster_domain
+  flavor_name     = var.openstack_master_flavor_name
+  instance_count  = var.master_count
+  lb_floating_ip  = var.openstack_lb_floating_ip
+  master_ips      = module.topology.master_ips
+  master_port_ids = module.topology.master_port_ids
+  user_data_ign   = var.ignition_master
+  api_int_ip      = var.openstack_api_int_ip
+  node_dns_ip     = var.openstack_node_dns_ip
   master_sg_ids = concat(
     var.openstack_master_extra_sg_ids,
     [module.topology.master_sg_id],
   )
 }
 
-# TODO(shadower) add a dns module here
-
 module "topology" {
   source = "./topology"
 
   cidr_block          = var.machine_cidr
   cluster_id          = var.cluster_id
+  cluster_domain      = var.cluster_domain
   external_network    = var.openstack_external_network
   external_network_id = var.openstack_external_network_id
   masters_count       = var.master_count
   lb_floating_ip      = var.openstack_lb_floating_ip
   api_int_ip          = var.openstack_api_int_ip
   node_dns_ip         = var.openstack_node_dns_ip
+  ingress_ip          = var.openstack_ingress_ip
   trunk_support       = var.openstack_trunk_support
   octavia_support     = var.openstack_octavia_support
 }
@@ -98,9 +80,11 @@ resource "openstack_objectstorage_container_v1" "container" {
   # "kubernetes.io/cluster/${var.cluster_id}" = "owned"
   metadata = merge(
     {
-      "Name"               = "${var.cluster_id}-ignition-master"
+      "Name"               = "${var.cluster_id}-ignition"
       "openshiftClusterID" = var.cluster_id
     },
+    # FIXME(mandre) the openstack_extra_tags should be applied to all resources
+    # created
     var.openstack_extra_tags,
   )
 }

diff --git a/data/data/openstack/masters/main.tf b/data/data/openstack/masters/main.tf
@@ -17,7 +17,6 @@ data "ignition_file" "hostname" {
     content = <<EOF
 ${var.cluster_id}-master-${count.index}
 EOF
-
   }
 }
 
@@ -30,9 +29,7 @@ data "ignition_file" "clustervars" {
     content = <<EOF
 export API_VIP=${var.api_int_ip}
 export DNS_VIP=${var.node_dns_ip}
-export FLOATING_IP=${var.lb_floating_ip}
 export BOOTSTRAP_IP=${var.bootstrap_ip}
-${replace(join("\n", formatlist("export MASTER_FIXED_IPS_%s=%s", var.master_port_names, var.master_ips)), "${var.cluster_id}-master-port-", "")}
 EOF
   }
 }
@@ -67,6 +64,7 @@ resource "openstack_compute_instance_v2" "master_conf" {
   }
 
   metadata = {
+    # FIXME(mandre) shouldn't it be "${var.cluster_id}-master-${count.index}" ?
     Name = "${var.cluster_id}-master"
     # "kubernetes.io/cluster/${var.cluster_id}" = "owned"
     openshiftClusterID = var.cluster_id

diff --git a/data/data/openstack/masters/variables.tf b/data/data/openstack/masters/variables.tf
@@ -43,10 +43,6 @@ variable "master_port_ids" {
   description = "List of port ids for the master nodes"
 }
 
-variable "master_port_names" {
-  type = list(string)
-}
-
 variable "user_data_ign" {
   type = string
 }
@@ -58,7 +54,3 @@ variable "api_int_ip" {
 variable "node_dns_ip" {
   type = string
 }
-
-variable "service_vm_fixed_ip" {
-  type = string
-}