Skip to content

Commit

Permalink
openstack: Remove the Service VM
Browse files Browse the repository at this point in the history 
The experimental OpenStack backend used to create an extra server
running DNS and load balancer services that the cluster needed.
OpenStack does not always come with DNSaaS or LBaaS so we had to provide
the functionality the OpenShift cluster depends on (e.g. the etcd SRV
records, the api-int records & load balancing, etc.).

This approach is undesirable for two reasons: first, it adds an extra
node that the other IPI platforms do not need. Second, this node is a
single point of failure.

The Baremetal platform has faced the same issues and they have solved
them with a few virtual IP addresses managed by keepalived in
combination with coredns static pod running on every node using the mDNS
protocol to update records as new nodes are added or removed and a
similar static pod haproxy to load balance the control plane internally.

The VIPs are defined here in the installer and they use the
PlatformStatus field to be passed to the necessary
machine-config-operator fields:

openshift/api#374

The Bare Metal IPI Networking Infrastructure document is applicable here as
well:

https://github.com/openshift/installer/blob/master/docs/design/baremetal/networking-infrastructure.md

There is also a great opportunity to share some of the configuration
files and scripts here.

This change needs several other pull requests:

Keepalived plus the coredns & haproxy static pods in the MCO:
openshift/machine-config-operator#740

Passing the API and DNS VIPs through the installer:
#1998

Co-authored-by: Emilio Garcia <egarcia@redhat.com>
Co-authored-by: John Trowbridge <trown@redhat.com>
Co-authored-by: Martin Andre <m.andre@redhat.com>
Co-authored-by: Tomas Sedovic <tsedovic@redhat.com>

Massive thanks to the Bare Metal and oVirt people!
  • Loading branch information
@tomassedovic
trown authored and tomassedovic committed Jul 30, 2019
1 parent 19ca25b commit e5226b9
Show file tree
Hide file tree
Showing 26 changed files with 218 additions and 671 deletions.
6 changes: 5 additions & 1 deletion data/data/bootstrap/files/usr/local/bin/bootkube.sh.template
View file
Expand Up @@ -211,13 +211,17 @@ then
# 1. read the controller config rendered by MachineConfigOperator
# 2. read the default MachineConfigPools rendered by MachineConfigOperator
# 3. read any additional MachineConfigs that are needed for the default MachineConfigPools.
mkdir --parents /etc/mcc/bootstrap /etc/mcs/bootstrap /etc/kubernetes/manifests
mkdir --parents /etc/mcc/bootstrap /etc/mcs/bootstrap /etc/kubernetes/manifests /etc/kubernetes/static-pod-resources
cp mco-bootstrap/bootstrap/manifests/* /etc/mcc/bootstrap/
cp openshift/* /etc/mcc/bootstrap/
# 4. read ImageContentSourcePolicy objects generated by the installer
cp manifests/* /etc/mcc/bootstrap/
cp auth/kubeconfig-kubelet /etc/mcs/kubeconfig
cp mco-bootstrap/bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/
if [ -d mco-bootstrap/openstack/manifests ]; then
cp mco-bootstrap/openstack/manifests/* /etc/kubernetes/manifests/
cp -r mco-bootstrap/openstack/static-pod-resources/* /etc/kubernetes/static-pod-resources/
fi
cp mco-bootstrap/manifests/* manifests/

# /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets.
Expand Down
43 changes: 24 additions & 19 deletions data/data/openstack/bootstrap/main.tf
View file
Expand Up @@ -18,45 +18,51 @@ data "ignition_config" "redirect" {

files = [
data.ignition_file.hostname.id,
data.ignition_file.bootstrap_ifcfg.id,
data.ignition_file.dns_conf.id,
data.ignition_file.dhcp_conf.id,
]
}

data "ignition_file" "bootstrap_ifcfg" {
data "ignition_file" "dhcp_conf" {
filesystem = "root"
mode = "420" // 0644
path = "/etc/sysconfig/network-scripts/ifcfg-eth0"
mode = "420"
path = "/etc/NetworkManager/conf.d/dhcp-client.conf"

content {
content = <<EOF
DEVICE="eth0"
BOOTPROTO="dhcp"
ONBOOT="yes"
TYPE="Ethernet"
PERSISTENT_DHCLIENT="yes"
DNS1="${var.service_vm_fixed_ip}"
PEERDNS="no"
NM_CONTROLLED="yes"
[main]
dhcp=dhclient
EOF
}
}

data "ignition_file" "dns_conf" {
filesystem = "root"
mode = "420"
path = "/etc/dhcp/dhclient.conf"

content {
content = <<EOF
send dhcp-client-identifier = hardware;
prepend domain-name-servers 127.0.0.1;
EOF
}
}

data "ignition_file" "hostname" {
filesystem = "root"
mode = "420" // 0644
path = "/etc/hostname"
mode = "420" // 0644
path = "/etc/hostname"

content {
content = <<EOF
${var.cluster_id}-bootstrap
EOF

}
}

data "openstack_images_image_v2" "bootstrap_image" {
name = var.image_name
name = var.image_name
most_recent = true
}

Expand All @@ -65,9 +71,9 @@ data "openstack_compute_flavor_v2" "bootstrap_flavor" {
}

resource "openstack_compute_instance_v2" "bootstrap" {
name = "${var.cluster_id}-bootstrap"
name = "${var.cluster_id}-bootstrap"
flavor_id = data.openstack_compute_flavor_v2.bootstrap_flavor.id
image_id = data.openstack_images_image_v2.bootstrap_image.id
image_id = data.openstack_images_image_v2.bootstrap_image.id

user_data = data.ignition_config.redirect.rendered

Expand All @@ -81,4 +87,3 @@ resource "openstack_compute_instance_v2" "bootstrap" {
openshiftClusterID = var.cluster_id
}
}

10 changes: 0 additions & 10 deletions data/data/openstack/bootstrap/variables.tf
View file
Expand Up @@ -13,11 +13,6 @@ variable "cluster_id" {
description = "The identifier for the cluster."
}

variable "cluster_domain" {
type = string
description = "The domain name of the cluster. All DNS records must be under this domain."
}

variable "ignition" {
type = string
description = "The content of the bootstrap ignition file."
Expand All @@ -32,8 +27,3 @@ variable "bootstrap_port_id" {
type = string
description = "The subnet ID for the bootstrap node."
}

variable "service_vm_fixed_ip" {
type = string
}

53 changes: 14 additions & 39 deletions data/data/openstack/main.tf
View file
Expand Up @@ -22,72 +22,45 @@ provider "openstack" {
user_name = var.openstack_credentials_user_name
}

module "service" {
source = "./service"
module "bootstrap" {
source = "./bootstrap"

swift_container = openstack_objectstorage_container_v1.container.name
cluster_id = var.cluster_id
cluster_domain = var.cluster_domain
image_name = var.openstack_base_image
flavor_name = var.openstack_master_flavor_name
ignition = var.ignition_bootstrap
lb_floating_ip = var.openstack_lb_floating_ip
service_port_id = module.topology.service_port_id
service_port_ip = module.topology.service_port_ip
master_ips = module.topology.master_ips
master_port_names = module.topology.master_port_names
bootstrap_ip = module.topology.bootstrap_port_ip
}

module "bootstrap" {
source = "./bootstrap"

swift_container = openstack_objectstorage_container_v1.container.name
cluster_id = var.cluster_id
cluster_domain = var.cluster_domain
image_name = var.openstack_base_image
flavor_name = var.openstack_master_flavor_name
ignition = var.ignition_bootstrap
bootstrap_port_id = module.topology.bootstrap_port_id
service_vm_fixed_ip = module.topology.service_vm_fixed_ip
bootstrap_port_id = module.topology.bootstrap_port_id
}

module "masters" {
source = "./masters"

base_image = var.openstack_base_image
bootstrap_ip = module.topology.bootstrap_port_ip
cluster_id = var.cluster_id
cluster_domain = var.cluster_domain
flavor_name = var.openstack_master_flavor_name
instance_count = var.master_count
lb_floating_ip = var.openstack_lb_floating_ip
master_ips = module.topology.master_ips
master_port_ids = module.topology.master_port_ids
master_port_names = module.topology.master_port_names
user_data_ign = var.ignition_master
service_vm_fixed_ip = module.topology.service_vm_fixed_ip
api_int_ip = var.openstack_api_int_ip
node_dns_ip = var.openstack_node_dns_ip
base_image = var.openstack_base_image
cluster_id = var.cluster_id
flavor_name = var.openstack_master_flavor_name
instance_count = var.master_count
master_port_ids = module.topology.master_port_ids
user_data_ign = var.ignition_master
master_sg_ids = concat(
var.openstack_master_extra_sg_ids,
[module.topology.master_sg_id],
)
}

# TODO(shadower) add a dns module here

module "topology" {
source = "./topology"

cidr_block = var.machine_cidr
cluster_id = var.cluster_id
cluster_domain = var.cluster_domain
external_network = var.openstack_external_network
external_network_id = var.openstack_external_network_id
masters_count = var.master_count
lb_floating_ip = var.openstack_lb_floating_ip
api_int_ip = var.openstack_api_int_ip
node_dns_ip = var.openstack_node_dns_ip
ingress_ip = var.openstack_ingress_ip
trunk_support = var.openstack_trunk_support
octavia_support = var.openstack_octavia_support
}
Expand All @@ -98,9 +71,11 @@ resource "openstack_objectstorage_container_v1" "container" {
# "kubernetes.io/cluster/${var.cluster_id}" = "owned"
metadata = merge(
{
"Name" = "${var.cluster_id}-ignition-master"
"Name" = "${var.cluster_id}-ignition"
"openshiftClusterID" = var.cluster_id
},
# FIXME(mandre) the openstack_extra_tags should be applied to all resources
# created
var.openstack_extra_tags,
)
}
Expand Down
27 changes: 5 additions & 22 deletions data/data/openstack/masters/main.tf
View file
Expand Up @@ -16,23 +16,6 @@ data "ignition_file" "hostname" {
content {
content = <<EOF
${var.cluster_id}-master-${count.index}
EOF

}
}

data "ignition_file" "clustervars" {
filesystem = "root"
mode = "420" // 0644
path = "/etc/kubernetes/static-pod-resources/clustervars"

content {
content = <<EOF
export API_VIP=${var.api_int_ip}
export DNS_VIP=${var.node_dns_ip}
export FLOATING_IP=${var.lb_floating_ip}
export BOOTSTRAP_IP=${var.bootstrap_ip}
${replace(join("\n", formatlist("export MASTER_FIXED_IPS_%s=%s", var.master_port_names, var.master_ips)), "${var.cluster_id}-master-port-", "")}
EOF
}
}
Expand All @@ -45,17 +28,16 @@ data "ignition_config" "master_ignition_config" {
}

files = [
element(data.ignition_file.hostname.*.id, count.index),
data.ignition_file.clustervars.id,
element(data.ignition_file.hostname.*.id, count.index)
]
}

resource "openstack_compute_instance_v2" "master_conf" {
name = "${var.cluster_id}-master-${count.index}"
name = "${var.cluster_id}-master-${count.index}"
count = var.instance_count

flavor_id = data.openstack_compute_flavor_v2.masters_flavor.id
image_id = data.openstack_images_image_v2.masters_img.id
flavor_id = data.openstack_compute_flavor_v2.masters_flavor.id
image_id = data.openstack_images_image_v2.masters_img.id
security_groups = var.master_sg_ids
user_data = element(
data.ignition_config.master_ignition_config.*.rendered,
Expand All @@ -67,6 +49,7 @@ resource "openstack_compute_instance_v2" "master_conf" {
}

metadata = {
# FIXME(mandre) shouldn't it be "${var.cluster_id}-master-${count.index}" ?
Name = "${var.cluster_id}-master"
# "kubernetes.io/cluster/${var.cluster_id}" = "owned"
openshiftClusterID = var.cluster_id
Expand Down
33 changes: 0 additions & 33 deletions data/data/openstack/masters/variables.tf
View file
Expand Up @@ -2,20 +2,11 @@ variable "base_image" {
type = string
}

variable "bootstrap_ip" {
type = string
}

variable "cluster_id" {
type = string
description = "The identifier for the cluster."
}

variable "cluster_domain" {
type = string
description = "The domain name of the cluster. All DNS records must be under this domain."
}

variable "flavor_name" {
type = string
}
Expand All @@ -24,14 +15,6 @@ variable "instance_count" {
type = string
}

variable "lb_floating_ip" {
type = string
}

variable "master_ips" {
type = list(string)
}

variable "master_sg_ids" {
type = list(string)
default = ["default"]
Expand All @@ -43,22 +26,6 @@ variable "master_port_ids" {
description = "List of port ids for the master nodes"
}

variable "master_port_names" {
type = list(string)
}

variable "user_data_ign" {
type = string
}

variable "api_int_ip" {
type = string
}

variable "node_dns_ip" {
type = string
}

variable "service_vm_fixed_ip" {
type = string
}

0 comments on commit e5226b9

Please sign in to comment.