openstack: Run preprovision steps for CAPI

pkg/asset/cluster/cluster.go

@@ -19,9 +19,11 @@ import (
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/asset/kubeconfig"
 	"github.com/openshift/installer/pkg/asset/machines"
+	"github.com/openshift/installer/pkg/asset/manifests"
 	capimanifests "github.com/openshift/installer/pkg/asset/manifests/clusterapi"
 	"github.com/openshift/installer/pkg/asset/password"
 	"github.com/openshift/installer/pkg/asset/quota"
+	"github.com/openshift/installer/pkg/asset/rhcos"
 	infra "github.com/openshift/installer/pkg/infrastructure/platform"
 	typesaws "github.com/openshift/installer/pkg/types/aws"
 	typesazure "github.com/openshift/installer/pkg/types/azure"
@@ -59,9 +61,11 @@ func (c *Cluster) Dependencies() []asset.Asset {
 		&installconfig.PlatformCredsCheck{},
 		&installconfig.PlatformPermsCheck{},
 		&installconfig.PlatformProvisionCheck{},
+		new(rhcos.Image),
 		&quota.PlatformQuotaCheck{},
 		&tfvars.TerraformVariables{},
 		&password.KubeadminPassword{},
+		&manifests.Manifests{},
 		&capimanifests.Cluster{},
 		&kubeconfig.AdminClient{},
 		&bootstrap.Bootstrap{},
@@ -78,8 +82,9 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 
 	clusterID := &installconfig.ClusterID{}
 	installConfig := &installconfig.InstallConfig{}
+	rhcosImage := new(rhcos.Image)
 	terraformVariables := &tfvars.TerraformVariables{}
-	parents.Get(clusterID, installConfig, terraformVariables)
+	parents.Get(clusterID, installConfig, terraformVariables, rhcosImage)
 
 	if fs := installConfig.Config.FeatureSet; strings.HasSuffix(string(fs), "NoUpgrade") {
 		logrus.Warnf("FeatureSet %q is enabled. This FeatureSet does not allow upgrades and may affect the supportability of the cluster.", fs)
@@ -112,7 +117,14 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 			return err
 		}
 	case typesopenstack.Name:
-		if err := openstack.PreTerraform(); err != nil {
+		var tfvarsFile *asset.File
+		for _, f := range terraformVariables.Files() {
+			if f.Filename == tfvars.TfPlatformVarsFileName {
+				tfvarsFile = f
+				break
+			}
+		}
+		if err := openstack.PreTerraform(context.TODO(), tfvarsFile, installConfig, clusterID, rhcosImage); err != nil {
 			return err
 		}
 	}

pkg/asset/cluster/openstack/openstack.go

@@ -3,42 +3,17 @@
 package openstack
 
 import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
+	"context"
 
+	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	rhcos_asset "github.com/openshift/installer/pkg/asset/rhcos"
+	"github.com/openshift/installer/pkg/infrastructure/openstack/preprovision"
+	"github.com/openshift/installer/pkg/rhcos"
 	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/openstack"
 )
 
-// PreTerraform performs any infrastructure initialization which must
-// happen before Terraform creates the remaining infrastructure.
-func PreTerraform() error {
-	// Terraform runs in a different directory but we want to allow people to
-	// use clouds.yaml files in their local directory. Emulate this by setting
-	// the necessary environment variable to point to this file if (a) the user
-	// hasn't already set this environment variable and (b) there is actually
-	// a local file
-	if path := os.Getenv("OS_CLIENT_CONFIG_FILE"); path != "" {
-		return nil
-	}
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		return fmt.Errorf("unable to determine working directory: %w", err)
-	}
-
-	cloudsYAML := filepath.Join(cwd, "clouds.yaml")
-	if _, err = os.Stat(cloudsYAML); err == nil {
-		os.Setenv("OS_CLIENT_CONFIG_FILE", cloudsYAML)
-	} else if !errors.Is(err, os.ErrNotExist) {
-		return fmt.Errorf("unable to determine if clouds.yaml exists: %w", err)
-	}
-
-	return nil
-}
-
 // Metadata converts an install configuration to OpenStack metadata.
 func Metadata(infraID string, config *types.InstallConfig) *openstack.Metadata {
 	return &openstack.Metadata{
@@ -48,3 +23,26 @@ func Metadata(infraID string, config *types.InstallConfig) *openstack.Metadata {
 		},
 	}
 }
+
+// PreTerraform performs any infrastructure initialization which must
+// happen before Terraform creates the remaining infrastructure.
+func PreTerraform(ctx context.Context, tfvarsFile *asset.File, installConfig *installconfig.InstallConfig, clusterID *installconfig.ClusterID, rhcosImage *rhcos_asset.Image) error {
+	// if err := preprovision.ReplaceBootstrapIgnitionInTFVars(ctx, tfvarsFile, installConfig, clusterID); err != nil {
+	// 	return err
+	// }
+
+	// if err := preprovision.TagVIPPorts(ctx, installConfig, clusterID.InfraID); err != nil {
+	// 	return err
+	// }
+
+	// upload the corresponding image to Glance if rhcosImage contains a
+	// URL. If rhcosImage contains a name, then that points to an existing
+	// Glance image.
+	if imageName, isURL := rhcos.GenerateOpenStackImageName(string(*rhcosImage), clusterID.InfraID); isURL {
+		if err := preprovision.UploadBaseImage(ctx, installConfig.Config.Platform.OpenStack.Cloud, string(*rhcosImage), imageName, clusterID.InfraID, installConfig.Config.Platform.OpenStack.ClusterOSImageProperties); err != nil {
+			return err
+		}
+	}
+
+	return preprovision.SetTerraformEnvironment()
+}

pkg/destroy/bootstrap/bootstrap.go

@@ -15,10 +15,10 @@ import (
 
 	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/installer/pkg/asset/cluster/metadata"
-	openstackasset "github.com/openshift/installer/pkg/asset/cluster/openstack"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/clusterapi"
 	osp "github.com/openshift/installer/pkg/destroy/openstack"
+	"github.com/openshift/installer/pkg/infrastructure/openstack/preprovision"
 	infra "github.com/openshift/installer/pkg/infrastructure/platform"
 	ibmcloudtfvars "github.com/openshift/installer/pkg/tfvars/ibmcloud"
 	"github.com/openshift/installer/pkg/types"
@@ -45,7 +45,7 @@ func Destroy(ctx context.Context, dir string) (err error) {
 	}
 
 	if platform == openstack.Name {
-		if err := openstackasset.PreTerraform(); err != nil {
+		if err := preprovision.SetTerraformEnvironment(); err != nil {
 			return errors.Wrapf(err, "Failed to  initialize infrastructure")
 		}
 

pkg/infrastructure/clusterapi/clusterapi.go

@@ -23,8 +23,10 @@ import (
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/asset/kubeconfig"
 	"github.com/openshift/installer/pkg/asset/machines"
+	"github.com/openshift/installer/pkg/asset/manifests"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	capimanifests "github.com/openshift/installer/pkg/asset/manifests/clusterapi"
+	"github.com/openshift/installer/pkg/asset/rhcos"
 	"github.com/openshift/installer/pkg/clusterapi"
 	"github.com/openshift/installer/pkg/infrastructure"
 	"github.com/openshift/installer/pkg/types"
@@ -52,18 +54,22 @@ func InitializeProvider(platform Provider) infrastructure.Provider {
 //
 //nolint:gocyclo
 func (i InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.File, error) {
+	manifestsAsset := &manifests.Manifests{}
 	capiManifestsAsset := &capimanifests.Cluster{}
 	capiMachinesAsset := &machines.ClusterAPI{}
 	clusterKubeconfigAsset := &kubeconfig.AdminClient{}
 	clusterID := &installconfig.ClusterID{}
 	installConfig := &installconfig.InstallConfig{}
+	rhcosImage := new(rhcos.Image)
 	bootstrapIgnAsset := &bootstrap.Bootstrap{}
 	masterIgnAsset := &machine.Master{}
 	parents.Get(
 		capiManifestsAsset,
 		clusterKubeconfigAsset,
 		clusterID,
 		installConfig,
+		manifestsAsset,
+		rhcosImage,
 		bootstrapIgnAsset,
 		masterIgnAsset,
 		capiMachinesAsset,
@@ -99,6 +105,8 @@ func (i InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.Fi
 		preProvisionInput := PreProvisionInput{
 			InfraID:       clusterID.InfraID,
 			InstallConfig: installConfig,
+			RhcosImage:    rhcosImage,
+			Manifests:     manifestsAsset,
 		}
 
 		if err := p.PreProvision(ctx, preProvisionInput); err != nil {
@@ -164,16 +172,12 @@ func (i InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.Fi
 				return false, err
 			}
 			cluster = c
+			logrus.Warnf("cluster spec:\n%+v", cluster.Spec)
+			logrus.Warnf("cluster status:\n%+v", cluster.Status)
 			return cluster.Status.InfrastructureReady, nil
 		}); err != nil {
 			return fileList, err
 		}
-		if cluster == nil {
-			return fileList, fmt.Errorf("error occurred during load balancer ready check")
-		}
-		if cluster.Spec.ControlPlaneEndpoint.Host == "" {
-			return fileList, fmt.Errorf("control plane endpoint is not set")
-		}
 	}
 
 	if p, ok := i.capiProvider.(InfraReadyProvider); ok {
@@ -275,6 +279,41 @@ func (i InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.Fi
 	manifests := []client.Object{}
 	manifests = append(manifests, infraManifests...)
 	manifests = append(manifests, machineManifests...)
+
+	// Wait for the load balancer to be ready by checking the control plane endpoint
+	// on the cluster object.
+	{
+		if err := wait.ExponentialBackoff(wait.Backoff{
+			Duration: time.Second * 10,
+			Factor:   float64(1.5),
+			Steps:    32,
+		}, func() (bool, error) {
+			c := &clusterv1.Cluster{}
+			if err := cl.Get(context.Background(), client.ObjectKey{
+				Name:      clusterID.InfraID,
+				Namespace: capiutils.Namespace,
+			}, c); err != nil {
+				if apierrors.IsNotFound(err) {
+					return false, nil
+				}
+				return false, err
+			}
+			cluster = c
+			ready := cluster.Spec.ControlPlaneEndpoint.IsValid() && cluster.Status.ControlPlaneReady
+			logrus.Warnf("cluster spec:\n%+v", cluster.Spec)
+			logrus.Warnf("cluster status:\n%+v", cluster.Status)
+			return ready, nil
+		}); err != nil {
+			return fileList, err
+		}
+		if cluster == nil {
+			return fileList, fmt.Errorf("error occurred during load balancer ready check")
+		}
+		if cluster.Spec.ControlPlaneEndpoint.Host == "" {
+			return fileList, fmt.Errorf("control plane endpoint is not set")
+		}
+	}
+
 	for _, m := range manifests {
 		key := client.ObjectKey{
 			Name:      m.GetName(),

pkg/infrastructure/clusterapi/types.go

@@ -6,6 +6,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/asset/manifests"
+	"github.com/openshift/installer/pkg/asset/rhcos"
 )
 
 // Provider is the base interface that cloud platforms
@@ -28,6 +30,8 @@ type PreProvider interface {
 type PreProvisionInput struct {
 	InfraID       string
 	InstallConfig *installconfig.InstallConfig
+	RhcosImage    *rhcos.Image
+	Manifests     *manifests.Manifests
 }
 
 // IgnitionProvider handles preconditions for bootstrap ignition and