openstack: get ignition via IP

For platforms without a DNS as a Service available, there is a
chicken/egg scenario. This is because we cannot setup our own DNS
solution on the nodes before they are booted via ignition.

Instead, we get the ignition config via a virtual IP that is
conifgured and maintained by keepalived. The setup of keepalived
will be done via machine-config-operator, so it is not part of
this patch.

This patch only sets up the plumbing so that we can merge the
keepalived work while keeping CI green. Then we can put up a patch
that will switch to actually using this functionality.

This patch also adds an interface for other platforms that need this
functionality to work from. Specifically, these PRs can be rebased
on this work:
openshift#1873
openshift#1948

data/data/openstack/main.tf

@@ -67,6 +67,8 @@ module "masters" {
   master_port_names   = module.topology.master_port_names
   user_data_ign       = var.ignition_master
   service_vm_fixed_ip = module.topology.service_vm_fixed_ip
+  api_int_ip          = var.openstack_api_int_ip
+  node_dns_ip         = var.openstack_node_dns_ip
   master_sg_ids = concat(
     var.openstack_master_extra_sg_ids,
     [module.topology.master_sg_id],
@@ -84,6 +86,8 @@ module "topology" {
   external_network_id = var.openstack_external_network_id
   masters_count       = var.master_count
   lb_floating_ip      = var.openstack_lb_floating_ip
+  api_int_ip          = var.openstack_api_int_ip
+  node_dns_ip         = var.openstack_node_dns_ip
   trunk_support       = var.openstack_trunk_support
   octavia_support     = var.openstack_octavia_support
 }

data/data/openstack/masters/main.tf

@@ -28,6 +28,8 @@ data "ignition_file" "clustervars" {
 
   content {
     content = <<EOF
+export API_VIP=${var.api_int_ip}
+export DNS_VIP=${var.node_dns_ip}
 export FLOATING_IP=${var.lb_floating_ip}
 export BOOTSTRAP_IP=${var.bootstrap_ip}
 ${replace(join("\n", formatlist("export MASTER_FIXED_IPS_%s=%s", var.master_port_names, var.master_ips)), "${var.cluster_id}-master-port-", "")}

data/data/openstack/masters/variables.tf

@@ -51,6 +51,14 @@ variable "user_data_ign" {
   type = string
 }
 
+variable "api_int_ip" {
+  type = string
+}
+
+variable "node_dns_ip" {
+  type = string
+}
+
 variable "service_vm_fixed_ip" {
   type = string
 }

data/data/openstack/topology/private-network.tf

@@ -30,6 +30,14 @@ resource "openstack_networking_subnet_v2" "nodes" {
   network_id      = openstack_networking_network_v2.openshift-private.id
   tags            = ["openshiftClusterID=${var.cluster_id}"]
   dns_nameservers = [openstack_networking_port_v2.service_port.all_fixed_ips[0]]
+
+  # We reserve some space at the beginning of the CIDR to use for the VIPs
+  # It would be good to make this more dynamic by calculating the number of
+  # addresses in the provided CIDR. This currently assumes at least a /18.
+  allocation_pool {
+    start = cidrhost(local.nodes_cidr_block, 10)
+    end   = cidrhost(local.nodes_cidr_block, 16000)
+  }
 }
 
 resource "openstack_networking_port_v2" "masters" {
@@ -44,6 +52,14 @@ resource "openstack_networking_port_v2" "masters" {
   fixed_ip {
     subnet_id = openstack_networking_subnet_v2.nodes.id
   }
+
+  allowed_address_pairs {
+    ip_address = var.api_int_ip
+  }
+
+  allowed_address_pairs {
+    ip_address = var.node_dns_ip
+  }
 }
 
 resource "openstack_networking_trunk_v2" "masters" {

data/data/openstack/topology/variables.tf

@@ -28,6 +28,14 @@ variable "masters_count" {
   type = string
 }
 
+variable "api_int_ip" {
+  type = string
+}
+
+variable "node_dns_ip" {
+  type = string
+}
+
 variable "trunk_support" {
   type = string
 }

data/data/openstack/variables-openstack.tf

@@ -269,6 +269,16 @@ EOF
 
 }
 
+variable "openstack_api_int_ip" {
+  type        = string
+  description = "IP on the node subnet reserved for api-int VIP."
+}
+
+variable "openstack_node_dns_ip" {
+  type        = string
+  description = "IP on the nodes subnet reserved for node dns VIP."
+}
+
 variable "openstack_master_flavor_name" {
   type        = string
   description = "Instance size for the master node(s). Example: `m1.medium`."

pkg/asset/cluster/tfvars.go

@@ -248,6 +248,8 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			installConfig.Config.Platform.OpenStack.Region,
 			installConfig.Config.Platform.OpenStack.ExternalNetwork,
 			installConfig.Config.Platform.OpenStack.LbFloatingIP,
+			installConfig.Config.Platform.OpenStack.APIVIP,
+			installConfig.Config.Platform.OpenStack.DNSVIP,
 			installConfig.Config.Platform.OpenStack.TrunkSupport,
 			installConfig.Config.Platform.OpenStack.OctaviaSupport,
 		)

pkg/asset/ignition/machine/node.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/openshift/installer/pkg/types"
 	baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
+	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
 )
 
 // pointerIgnitionConfig generates a config which references the remote config
@@ -20,6 +21,11 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro
 		// Baremetal needs to point directly at the VIP because we don't have a
 		// way to configure DNS before Ignition runs.
 		ignitionHost = fmt.Sprintf("%s:22623", installConfig.BareMetal.APIVIP)
+	case openstacktypes.Name:
+		// We can't actually set this to the VIP until we have keepalived patches
+		// merged to machine-config-operator
+		// ignitionHost = fmt.Sprintf("%s:22623", installConfig.Config.OpenStack.APIVIP)
+		ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
 	default:
 		ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
 	}

pkg/asset/tls/mcscertkey.go

@@ -8,6 +8,7 @@ import (
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
+	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
 )
 
 // MCSCertKey is the asset that generates the MCS key/cert pair.
@@ -45,6 +46,9 @@ func (a *MCSCertKey) Generate(dependencies asset.Parents) error {
 	case baremetaltypes.Name:
 		cfg.IPAddresses = []net.IP{net.ParseIP(installConfig.Config.BareMetal.APIVIP)}
 		cfg.DNSNames = []string{hostname, installConfig.Config.BareMetal.APIVIP}
+	case openstacktypes.Name:
+		cfg.IPAddresses = []net.IP{net.ParseIP(installConfig.Config.OpenStack.APIVIP)}
+		cfg.DNSNames = []string{hostname, installConfig.Config.OpenStack.APIVIP}
 	default:
 		cfg.DNSNames = []string{hostname}
 	}

pkg/tfvars/openstack/openstack.go

@@ -14,19 +14,23 @@ type config struct {
 	Cloud           string `json:"openstack_credentials_cloud,omitempty"`
 	FlavorName      string `json:"openstack_master_flavor_name,omitempty"`
 	LbFloatingIP    string `json:"openstack_lb_floating_ip,omitempty"`
+	APIVIP          string `json:"openstack_api_int_ip,omitempty"`
+	DNSVIP          string `json:"openstack_node_dns_ip,omitempty"`
 	TrunkSupport    string `json:"openstack_trunk_support,omitempty"`
 	OctaviaSupport  string `json:"openstack_octavia_support,omitempty"`
 }
 
 // TFVars generates OpenStack-specific Terraform variables.
-func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, region string, externalNetwork string, lbFloatingIP string, trunkSupport string, octaviaSupport string) ([]byte, error) {
+func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, region string, externalNetwork string, lbFloatingIP string, apiVIP string, dnsVIP string, trunkSupport string, octaviaSupport string) ([]byte, error) {
 	cfg := &config{
 		Region:          region,
 		BaseImage:       masterConfig.Image,
 		ExternalNetwork: externalNetwork,
 		Cloud:           masterConfig.CloudName,
 		FlavorName:      masterConfig.Flavor,
 		LbFloatingIP:    lbFloatingIP,
+		APIVIP:          apiVIP,
+		DNSVIP:          dnsVIP,
 		TrunkSupport:    trunkSupport,
 		OctaviaSupport:  octaviaSupport,
 	}

pkg/types/defaults/installconfig.go

@@ -67,7 +67,7 @@ func SetInstallConfigDefaults(c *types.InstallConfig) {
 	case c.Platform.Libvirt != nil:
 		libvirtdefaults.SetPlatformDefaults(c.Platform.Libvirt)
 	case c.Platform.OpenStack != nil:
-		openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack)
+		openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack, c)
 	case c.Platform.VSphere != nil:
 		vspheredefaults.SetPlatformDefaults(c.Platform.VSphere, c)
 	case c.Platform.BareMetal != nil:

pkg/types/defaults/installconfig_test.go

@@ -65,7 +65,7 @@ func defaultLibvirtInstallConfig() *types.InstallConfig {
 func defaultOpenStackInstallConfig() *types.InstallConfig {
 	c := defaultInstallConfig()
 	c.Platform.OpenStack = &openstack.Platform{}
-	openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack)
+	openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack, c)
 	return c
 }
 

pkg/types/openstack/defaults/platform.go

@@ -1,9 +1,37 @@
 package defaults
 
 import (
+	"fmt"
+
+	"github.com/apparentlymart/go-cidr/cidr"
+
+	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/openstack"
 )
 
+// Defaults for the openstack platform.
+const (
+	APIVIP = ""
+	DNSVIP = ""
+)
+
 // SetPlatformDefaults sets the defaults for the platform.
-func SetPlatformDefaults(p *openstack.Platform) {
+func SetPlatformDefaults(p *openstack.Platform, c *types.InstallConfig) {
+	if p.APIVIP == "" {
+		vip, err := cidr.Host(&c.Networking.MachineCIDR.IPNet, 5)
+		if err != nil {
+			p.APIVIP = fmt.Sprintf("failed to get APIVIP from MachineCIDR: %s", err.Error())
+		} else {
+			p.APIVIP = vip.String()
+		}
+	}
+
+	if p.DNSVIP == "" {
+		vip, err := cidr.Host(&c.Networking.MachineCIDR.IPNet, 6)
+		if err != nil {
+			p.DNSVIP = fmt.Sprintf("failed to get DNSVIP from MachineCIDR: %s", err.Error())
+		} else {
+			p.DNSVIP = vip.String()
+		}
+	}
 }

pkg/types/openstack/platform.go

@@ -28,6 +28,14 @@ type Platform struct {
 	// Existing Floating IP to associate with the OpenStack load balancer.
 	LbFloatingIP string `json:"lbFloatingIP"`
 
+	// APIVIP
+	// IP in the machineCIDR to use for api-int.
+	APIVIP string `json:"apiVIP"`
+
+	// DNSVIP
+	// IP in the machineCIDR to use for simulated route53.
+	DNSVIP string `json:"dnsVIP"`
+
 	// TrunkSupport
 	// Whether OpenStack ports can be trunked
 	TrunkSupport string `json:"trunkSupport"`