baremetal: send full ignition to masters

This restores the work which was previously done via openshift#3276
but then reverted via openshift#3589 due to breaking users who customized
the pointer ignition config in IPI deployments.

A solution to that has been proposed via openshift#4413 - see openshift/enhancements#540 for more details.

Note that some additional changes beyond the initial implementation
were required due to the MCO now supporting multiple
ignition versions, thus this depends on openshift-metal3/terraform-provider-ironic#46

Co-Authored-By: Steven Hardy <shardy@redhat.com>

data/data/baremetal/main.tf

@@ -26,11 +26,13 @@ module "bootstrap" {
 module "masters" {
   source = "./masters"
 
-  master_count   = var.master_count
-  ignition       = var.ignition_master
-  hosts          = var.hosts
-  properties     = var.properties
-  root_devices   = var.root_devices
-  driver_infos   = var.driver_infos
-  instance_infos = var.instance_infos
+  master_count                = var.master_count
+  hosts                       = var.hosts
+  properties                  = var.properties
+  root_devices                = var.root_devices
+  driver_infos                = var.driver_infos
+  instance_infos              = var.instance_infos
+  master_ignition_url         = var.master_ignition_url
+  master_ignition_url_ca_cert = var.master_ignition_url_ca_cert
+  master_ignition_url_headers = var.master_ignition_url_headers
 }

data/data/baremetal/masters/main.tf

@@ -42,8 +42,10 @@ resource "ironic_deployment" "openshift-master-deployment" {
     count.index,
   )
 
-  instance_info = var.instance_infos[count.index]
-  user_data     = var.ignition
+  instance_info         = var.instance_infos[count.index]
+  user_data_url         = var.master_ignition_url
+  user_data_url_ca_cert = var.master_ignition_url_ca_cert
+  user_data_url_headers = var.master_ignition_url_headers
 }
 
 data "ironic_introspection" "openshift-master-introspection" {

data/data/baremetal/masters/variables.tf

@@ -4,11 +4,6 @@ variable "master_count" {
   default     = 3
 }
 
-variable "ignition" {
-  type        = string
-  description = "The content of the master ignition file"
-}
-
 variable "hosts" {
   type        = list(map(string))
   description = "Hardware details for hosts"
@@ -33,3 +28,18 @@ variable "instance_infos" {
   type        = list(map(string))
   description = "Instance information for hosts"
 }
+
+variable "master_ignition_url" {
+  type        = string
+  description = "The URL of the full ignition"
+}
+
+variable "master_ignition_url_ca_cert" {
+  type        = string
+  description = "Root CA cert of the full ignition URL"
+}
+
+variable "master_ignition_url_headers" {
+  type        = map(string)
+  description = "Headers to use when retrieving master_ignition_url"
+}

data/data/baremetal/variables-baremetal.tf

@@ -52,3 +52,18 @@ variable "instance_infos" {
   type        = list(map(string))
   description = "Instance information for hosts"
 }
+
+variable "master_ignition_url" {
+  type        = string
+  description = "The URL of the full ignition"
+}
+
+variable "master_ignition_url_ca_cert" {
+  type        = string
+  description = "Root CA cert of the full ignition URL"
+}
+
+variable "master_ignition_url_headers" {
+  type        = map(string)
+  description = "Headers to pass when retrieving master_ignition_url"
+}

pkg/asset/cluster/tfvars.go

@@ -442,6 +442,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			string(*rhcosImage),
 			ironicCreds.Username,
 			ironicCreds.Password,
+			masterIgn,
 		)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get %s Terraform variables", platform)

pkg/tfvars/baremetal/baremetal.go

@@ -9,6 +9,8 @@ import (
 	"path"
 	"strings"
 
+	igntypes "github.com/coreos/ignition/v2/config/v3_1/types"
+
 	"github.com/metal3-io/baremetal-operator/pkg/bmc"
 	"github.com/metal3-io/baremetal-operator/pkg/hardware"
 	"github.com/openshift/installer/pkg/tfvars/internal/cache"
@@ -25,6 +27,10 @@ type config struct {
 	IronicUsername string `json:"ironic_username"`
 	IronicPassword string `json:"ironic_password"`
 
+	MasterIgnitionURL        string            `json:"master_ignition_url,omitempty"`
+	MasterIgnitionURLCACert  string            `json:"master_ignition_url_ca_cert,omitempty"`
+	MasterIgnitionURLHeaders map[string]string `json:"master_ignition_url_headers,omitempty"`
+
 	// Data required for control plane deployment - several maps per host, because of terraform's limitations
 	Hosts         []map[string]interface{} `json:"hosts"`
 	RootDevices   []map[string]interface{} `json:"root_devices"`
@@ -34,7 +40,7 @@ type config struct {
 }
 
 // TFVars generates bare metal specific Terraform variables.
-func TFVars(libvirtURI, bootstrapProvisioningIP, bootstrapOSImage, externalBridge, externalMAC, provisioningBridge, provisioningMAC string, platformHosts []*baremetal.Host, image, ironicUsername, ironicPassword string) ([]byte, error) {
+func TFVars(libvirtURI, bootstrapProvisioningIP, bootstrapOSImage, externalBridge, externalMAC, provisioningBridge, provisioningMAC string, platformHosts []*baremetal.Host, image, ironicUsername, ironicPassword, ignition string) ([]byte, error) {
 	bootstrapOSImage, err := cache.DownloadImageFile(bootstrapOSImage)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to use cached bootstrap libvirt image")
@@ -156,18 +162,43 @@ func TFVars(libvirtURI, bootstrapProvisioningIP, bootstrapOSImage, externalBridg
 			})
 	}
 
+	var masterIgn igntypes.Config
+	if err := json.Unmarshal([]byte(ignition), &masterIgn); err != nil {
+		return nil, err
+	}
+	if len(masterIgn.Ignition.Config.Merge) == 0 {
+		return nil, errors.Wrap(err, "Empty Merge section in master pointer ignition")
+	}
+	ignitionURL := *masterIgn.Ignition.Config.Merge[0].Source
+	if len(masterIgn.Ignition.Security.TLS.CertificateAuthorities) == 0 {
+		return nil, errors.Wrap(err, "Empty CertificateAuthorities section in master pointer ignition")
+	}
+	ignitionURLCACert := strings.TrimPrefix(
+		*masterIgn.Ignition.Security.TLS.CertificateAuthorities[0].Source,
+		"data:text/plain;charset=utf-8;base64,")
+	// To return the same version as the stub config, the MCS requires a
+	// header, otherwise we get 2.2.0, e.g:
+	// "Accept: application/vnd.coreos.ignition+json; version=3.1.0"
+	ignitionURLHeaders := map[string]string{
+		"Accept": fmt.Sprintf("application/vnd.coreos.ignition+json;version=%s",
+			masterIgn.Ignition.Version),
+	}
+
 	cfg := &config{
-		LibvirtURI:              libvirtURI,
-		BootstrapProvisioningIP: bootstrapProvisioningIP,
-		BootstrapOSImage:        bootstrapOSImage,
-		IronicUsername:          ironicUsername,
-		IronicPassword:          ironicPassword,
-		Hosts:                   hosts,
-		Bridges:                 bridges,
-		Properties:              properties,
-		DriverInfos:             driverInfos,
-		RootDevices:             rootDevices,
-		InstanceInfos:           instanceInfos,
+		LibvirtURI:               libvirtURI,
+		BootstrapProvisioningIP:  bootstrapProvisioningIP,
+		BootstrapOSImage:         bootstrapOSImage,
+		IronicUsername:           ironicUsername,
+		IronicPassword:           ironicPassword,
+		Hosts:                    hosts,
+		Bridges:                  bridges,
+		Properties:               properties,
+		DriverInfos:              driverInfos,
+		RootDevices:              rootDevices,
+		InstanceInfos:            instanceInfos,
+		MasterIgnitionURL:        ignitionURL,
+		MasterIgnitionURLCACert:  ignitionURLCACert,
+		MasterIgnitionURLHeaders: ignitionURLHeaders,
 	}
 
 	return json.MarshalIndent(cfg, "", "  ")