From 0c1768dff7ffaefdde79fd03d71fdc5745f5d417 Mon Sep 17 00:00:00 2001 From: trown Date: Fri, 12 Jul 2019 11:05:22 -0400 Subject: [PATCH] openstack: Remove the Service VM The experimental OpenStack backend used to create an extra server running DNS and load balancer services that the cluster needed. OpenStack does not always come with DNSaaS or LBaaS so we had to provide the functionality the OpenShift cluster depends on (e.g. the etcd SRV records, the api-int records & load balancing, etc.). This approach is undesirable for two reasons: first, it adds an extra node that the other IPI platforms do not need. Second, this node is a single point of failure. The Baremetal platform has faced the same issues and they have solved them with a few virtual IP addresses managed by keepalived in combination with coredns static pod running on every node using the mDNS protocol to update records as new nodes are added or removed and a similar static pod haproxy to load balance the control plane internally. The VIPs are defined here in the installer and they use the PlatformStatus field to be passed to the necessary machine-config-operator fields: https://github.com/openshift/api/pull/374 The Bare Metal IPI Networking Infrastructure document is applicable here as well: https://github.com/openshift/installer/blob/master/docs/design/baremetal/networking-infrastructure.md There is also a great opportunity to share some of the configuration files and scripts here. This change needs several other pull requests: Keepalived plus the coredns & haproxy static pods in the MCO: https://github.com/openshift/machine-config-operator/pull/740 Co-authored-by: Emilio Garcia Co-authored-by: John Trowbridge Co-authored-by: Martin Andre Co-authored-by: Tomas Sedovic Massive thanks to the Bare Metal and oVirt people! --- .../files/usr/local/bin/bootkube.sh.template | 4 + data/data/openstack/bootstrap/main.tf | 43 +-- data/data/openstack/bootstrap/variables.tf | 10 - data/data/openstack/main.tf | 53 +-- data/data/openstack/masters/main.tf | 25 +- data/data/openstack/masters/variables.tf | 25 -- data/data/openstack/service/main.tf | 320 ------------------ data/data/openstack/service/variables.tf | 57 ---- data/data/openstack/topology/outputs.tf | 24 -- .../openstack/topology/private-network.tf | 117 +++++-- data/data/openstack/topology/sg-lb.tf | 85 ----- data/data/openstack/topology/sg-master.tf | 35 +- data/data/openstack/topology/sg-worker.tf | 6 + data/data/openstack/topology/variables.tf | 17 + data/data/openstack/variables-openstack.tf | 17 +- pkg/asset/cluster/tfvars.go | 16 + pkg/asset/ignition/bootstrap/bootstrap.go | 2 +- pkg/asset/ignition/machine/node.go | 9 + pkg/asset/manifests/infrastructure.go | 18 + pkg/asset/tls/mcscertkey.go | 9 + pkg/tfvars/openstack/openstack.go | 8 +- pkg/types/openstack/defaults/platform.go | 30 ++ pkg/types/openstack/validation/platform.go | 4 +- .../openstack/validation/platform_test.go | 2 +- pkg/types/validation/installconfig.go | 2 +- pkg/types/validation/installconfig_test.go | 19 +- 26 files changed, 313 insertions(+), 644 deletions(-) delete mode 100644 data/data/openstack/service/main.tf delete mode 100644 data/data/openstack/service/variables.tf delete mode 100644 data/data/openstack/topology/sg-lb.tf diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index d4656230224..fa7e1201fa2 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -219,6 +219,10 @@ then cp mco-bootstrap/baremetal/manifests/* /etc/kubernetes/manifests/ cp --recursive mco-bootstrap/baremetal/static-pod-resources/* /etc/kubernetes/static-pod-resources/ fi + if [ -d mco-bootstrap/openstack/manifests ]; then + cp mco-bootstrap/openstack/manifests/* /etc/kubernetes/manifests/ + cp --recursive mco-bootstrap/openstack/static-pod-resources/* /etc/kubernetes/static-pod-resources/ + fi cp mco-bootstrap/manifests/* manifests/ # /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets. diff --git a/data/data/openstack/bootstrap/main.tf b/data/data/openstack/bootstrap/main.tf index 7c456c2a1fe..b4a430f4a0a 100644 --- a/data/data/openstack/bootstrap/main.tf +++ b/data/data/openstack/bootstrap/main.tf @@ -18,45 +18,51 @@ data "ignition_config" "redirect" { files = [ data.ignition_file.hostname.id, - data.ignition_file.bootstrap_ifcfg.id, + data.ignition_file.dns_conf.id, + data.ignition_file.dhcp_conf.id, ] } -data "ignition_file" "bootstrap_ifcfg" { +data "ignition_file" "dhcp_conf" { filesystem = "root" - mode = "420" // 0644 - path = "/etc/sysconfig/network-scripts/ifcfg-eth0" + mode = "420" + path = "/etc/NetworkManager/conf.d/dhcp-client.conf" content { content = < /etc/haproxy/haproxy.cfg.new << EOF -listen ${var.cluster_id}-api-masters - bind 0.0.0.0:6443 - bind 0.0.0.0:22623 - mode tcp - balance roundrobin - server bootstrap-22623 ${var.bootstrap_ip} check port 22623 - server bootstrap-6443 ${var.bootstrap_ip} check port 6443 - ${replace(join("\n ", formatlist("server master-%s %s check port 6443", var.master_port_names, var.master_ips)), "master-port-", "")} -EOF - update_cfg_and_restart - exit 0 -fi - -for master in $MASTERS; -do - MASTER_LINES="$MASTER_LINES - server $master $master check port 6443" -done - -for worker in $WORKERS; -do - WORKER_LINES="$WORKER_LINES - server $worker $worker check port 443" -done - -cat > /etc/haproxy/haproxy.cfg.new << EOF -listen ${var.cluster_id}-api-masters - bind 0.0.0.0:6443 - bind 0.0.0.0:22623 - mode tcp - balance roundrobin$MASTER_LINES - -listen ${var.cluster_id}-api-workers - bind 0.0.0.0:80 - bind 0.0.0.0:443 - mode tcp - balance roundrobin$WORKER_LINES -EOF - -update_cfg_and_restart -TFEOF - -} -} - -data "ignition_file" "corefile" { - filesystem = "root" - mode = "420" // 0644 - path = "/etc/coredns/Corefile" - - content { - content = <