Skip to content

Commit

Permalink
openstack: Run preprovision steps for CAPI
Browse files Browse the repository at this point in the history 
Co-Authored-By: Martin André <m.andre@redhat.com>
  • Loading branch information
@pierreprinetti @mandre @MaysaMacedo
2 people authored and MaysaMacedo committed Feb 2, 2024
1 parent 902b8c2 commit 3531aa5
Show file tree
Hide file tree
Showing 20 changed files with 896 additions and 1,039 deletions.
16 changes: 14 additions & 2 deletions pkg/asset/cluster/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ import (
"github.com/openshift/installer/pkg/asset/installconfig"
"github.com/openshift/installer/pkg/asset/kubeconfig"
"github.com/openshift/installer/pkg/asset/machines"
"github.com/openshift/installer/pkg/asset/manifests"
capimanifests "github.com/openshift/installer/pkg/asset/manifests/clusterapi"
"github.com/openshift/installer/pkg/asset/password"
"github.com/openshift/installer/pkg/asset/quota"
"github.com/openshift/installer/pkg/asset/rhcos"
infra "github.com/openshift/installer/pkg/infrastructure/platform"
typesaws "github.com/openshift/installer/pkg/types/aws"
typesazure "github.com/openshift/installer/pkg/types/azure"
Expand Down Expand Up @@ -59,9 +61,11 @@ func (c *Cluster) Dependencies() []asset.Asset {
&installconfig.PlatformCredsCheck{},
&installconfig.PlatformPermsCheck{},
&installconfig.PlatformProvisionCheck{},
new(rhcos.Image),
&quota.PlatformQuotaCheck{},
&tfvars.TerraformVariables{},
&password.KubeadminPassword{},
&manifests.Manifests{},
&capimanifests.Cluster{},
&kubeconfig.AdminClient{},
&bootstrap.Bootstrap{},
Expand All @@ -78,8 +82,9 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {

clusterID := &installconfig.ClusterID{}
installConfig := &installconfig.InstallConfig{}
rhcosImage := new(rhcos.Image)
terraformVariables := &tfvars.TerraformVariables{}
parents.Get(clusterID, installConfig, terraformVariables)
parents.Get(clusterID, installConfig, terraformVariables, rhcosImage)

if fs := installConfig.Config.FeatureSet; strings.HasSuffix(string(fs), "NoUpgrade") {
logrus.Warnf("FeatureSet %q is enabled. This FeatureSet does not allow upgrades and may affect the supportability of the cluster.", fs)
Expand Down Expand Up @@ -112,7 +117,14 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
return err
}
case typesopenstack.Name:
if err := openstack.PreTerraform(); err != nil {
var tfvarsFile *asset.File
for _, f := range terraformVariables.Files() {
if f.Filename == tfvars.TfPlatformVarsFileName {
tfvarsFile = f
break
}
}
if err := openstack.PreTerraform(context.TODO(), tfvarsFile, installConfig, clusterID, rhcosImage); err != nil {
return err
}
}
Expand Down
63 changes: 32 additions & 31 deletions pkg/asset/cluster/openstack/openstack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,42 +3,18 @@
package openstack

import (
"errors"
"fmt"
"os"
"path/filepath"
"context"

"github.com/openshift/installer/pkg/asset"
"github.com/openshift/installer/pkg/asset/installconfig"
"github.com/openshift/installer/pkg/asset/manifests/capiutils"
rhcos_asset "github.com/openshift/installer/pkg/asset/rhcos"
"github.com/openshift/installer/pkg/infrastructure/openstack/preprovision"
"github.com/openshift/installer/pkg/rhcos"
"github.com/openshift/installer/pkg/types"
"github.com/openshift/installer/pkg/types/openstack"
)

// PreTerraform performs any infrastructure initialization which must
// happen before Terraform creates the remaining infrastructure.
func PreTerraform() error {
// Terraform runs in a different directory but we want to allow people to
// use clouds.yaml files in their local directory. Emulate this by setting
// the necessary environment variable to point to this file if (a) the user
// hasn't already set this environment variable and (b) there is actually
// a local file
if path := os.Getenv("OS_CLIENT_CONFIG_FILE"); path != "" {
return nil
}

cwd, err := os.Getwd()
if err != nil {
return fmt.Errorf("unable to determine working directory: %w", err)
}

cloudsYAML := filepath.Join(cwd, "clouds.yaml")
if _, err = os.Stat(cloudsYAML); err == nil {
os.Setenv("OS_CLIENT_CONFIG_FILE", cloudsYAML)
} else if !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("unable to determine if clouds.yaml exists: %w", err)
}

return nil
}

// Metadata converts an install configuration to OpenStack metadata.
func Metadata(infraID string, config *types.InstallConfig) *openstack.Metadata {
return &openstack.Metadata{
Expand All @@ -48,3 +24,28 @@ func Metadata(infraID string, config *types.InstallConfig) *openstack.Metadata {
},
}
}

// PreTerraform performs any infrastructure initialization which must
// happen before Terraform creates the remaining infrastructure.
func PreTerraform(ctx context.Context, tfvarsFile *asset.File, installConfig *installconfig.InstallConfig, clusterID *installconfig.ClusterID, rhcosImage *rhcos_asset.Image) error {
if !capiutils.IsEnabled(installConfig) {
if err := preprovision.ReplaceBootstrapIgnitionInTFVars(ctx, tfvarsFile, installConfig, clusterID); err != nil {
return err
}

if err := preprovision.TagVIPPorts(ctx, installConfig, clusterID.InfraID); err != nil {
return err
}

// upload the corresponding image to Glance if rhcosImage contains a
// URL. If rhcosImage contains a name, then that points to an existing
// Glance image.
if imageName, isURL := rhcos.GenerateOpenStackImageName(string(*rhcosImage), clusterID.InfraID); isURL {
if err := preprovision.UploadBaseImage(ctx, installConfig.Config.Platform.OpenStack.Cloud, string(*rhcosImage), imageName, clusterID.InfraID, installConfig.Config.Platform.OpenStack.ClusterOSImageProperties); err != nil {
return err
}
}
}

return preprovision.SetTerraformEnvironment()
}
4 changes: 2 additions & 2 deletions pkg/destroy/bootstrap/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import (

configv1 "github.com/openshift/api/config/v1"
"github.com/openshift/installer/pkg/asset/cluster/metadata"
openstackasset "github.com/openshift/installer/pkg/asset/cluster/openstack"
osp "github.com/openshift/installer/pkg/destroy/openstack"
"github.com/openshift/installer/pkg/infrastructure/openstack/preprovision"
infra "github.com/openshift/installer/pkg/infrastructure/platform"
ibmcloudtfvars "github.com/openshift/installer/pkg/tfvars/ibmcloud"
typesazure "github.com/openshift/installer/pkg/types/azure"
Expand All @@ -35,7 +35,7 @@ func Destroy(ctx context.Context, dir string) (err error) {
}

if platform == openstack.Name {
if err := openstackasset.PreTerraform(); err != nil {
if err := preprovision.SetTerraformEnvironment(); err != nil {
return errors.Wrapf(err, "Failed to initialize infrastructure")
}

Expand Down
9 changes: 9 additions & 0 deletions pkg/infrastructure/clusterapi/clusterapi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@ import (
"github.com/openshift/installer/pkg/asset/installconfig"
"github.com/openshift/installer/pkg/asset/kubeconfig"
"github.com/openshift/installer/pkg/asset/machines"
"github.com/openshift/installer/pkg/asset/manifests"
"github.com/openshift/installer/pkg/asset/manifests/capiutils"
capimanifests "github.com/openshift/installer/pkg/asset/manifests/clusterapi"
"github.com/openshift/installer/pkg/asset/rhcos"
"github.com/openshift/installer/pkg/clusterapi"
"github.com/openshift/installer/pkg/infrastructure"
"github.com/openshift/installer/pkg/types"
Expand Down Expand Up @@ -52,18 +54,22 @@ func InitializeProvider(platform Provider) infrastructure.Provider {
//
//nolint:gocyclo
func (i *InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.File, error) {
manifestsAsset := &manifests.Manifests{}
capiManifestsAsset := &capimanifests.Cluster{}
capiMachinesAsset := &machines.ClusterAPI{}
clusterKubeconfigAsset := &kubeconfig.AdminClient{}
clusterID := &installconfig.ClusterID{}
installConfig := &installconfig.InstallConfig{}
rhcosImage := new(rhcos.Image)
bootstrapIgnAsset := &bootstrap.Bootstrap{}
masterIgnAsset := &machine.Master{}
parents.Get(
capiManifestsAsset,
clusterKubeconfigAsset,
clusterID,
installConfig,
manifestsAsset,
rhcosImage,
bootstrapIgnAsset,
masterIgnAsset,
capiMachinesAsset,
Expand Down Expand Up @@ -98,6 +104,8 @@ func (i *InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.F
preProvisionInput := PreProvisionInput{
InfraID: clusterID.InfraID,
InstallConfig: installConfig,
RhcosImage: rhcosImage,
Manifests: manifestsAsset,
}

if err := p.PreProvision(ctx, preProvisionInput); err != nil {
Expand Down Expand Up @@ -276,6 +284,7 @@ func (i *InfraProvider) Provision(dir string, parents asset.Parents) ([]*asset.F
manifests := []client.Object{}
manifests = append(manifests, infraManifests...)
manifests = append(manifests, machineManifests...)

for _, m := range manifests {
key := client.ObjectKey{
Name: m.GetName(),
Expand Down
4 changes: 4 additions & 0 deletions pkg/infrastructure/clusterapi/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/openshift/installer/pkg/asset/installconfig"
"github.com/openshift/installer/pkg/asset/manifests"
"github.com/openshift/installer/pkg/asset/rhcos"
)

// Provider is the base interface that cloud platforms
Expand All @@ -28,6 +30,8 @@ type PreProvider interface {
type PreProvisionInput struct {
InfraID string
InstallConfig *installconfig.InstallConfig
RhcosImage *rhcos.Image
Manifests *manifests.Manifests
}

// IgnitionProvider handles preconditions for bootstrap ignition and
Expand Down
102 changes: 86 additions & 16 deletions pkg/infrastructure/openstack/clusterapi/clusterapi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,58 +4,110 @@ import (
"context"
"fmt"

configv1 "github.com/openshift/api/config/v1"
"gopkg.in/yaml.v2"
capo "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha7"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags"
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips"
"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
"github.com/openshift/installer/pkg/asset/installconfig"
"github.com/openshift/installer/pkg/asset/manifests"
"github.com/openshift/installer/pkg/asset/manifests/capiutils"
"github.com/openshift/installer/pkg/infrastructure/clusterapi"
"github.com/openshift/installer/pkg/infrastructure/openstack/preprovision"
"github.com/openshift/installer/pkg/rhcos"
"github.com/openshift/installer/pkg/types/openstack"
openstackdefaults "github.com/openshift/installer/pkg/types/openstack/defaults"
"github.com/sirupsen/logrus"
)

type Provider struct {
clusterapi.InfraProvider
}

// func (p Provider) PreProvision(in clusterapi.PreProvisionInput) error {
// return nil
// }
func (p Provider) Name() string {
return openstack.Name
}

var _ clusterapi.PreProvider = Provider{}

func (p Provider) PreProvision(ctx context.Context, in clusterapi.PreProvisionInput) error {
var (
infraID string = in.InfraID
installConfig *installconfig.InstallConfig = in.InstallConfig
rhcosImage string = string(*in.RhcosImage)
mastersSchedulable bool
)

for _, f := range in.Manifests.Files() {
if f.Filename == manifests.SchedulerCfgFilename {
schedulerConfig := configv1.Scheduler{}
if err := yaml.Unmarshal(f.Data, &schedulerConfig); err != nil {
return fmt.Errorf("unable to decode the scheduler manifest: %w", err)
}
mastersSchedulable = schedulerConfig.Spec.MastersSchedulable
break
}
}

if err := preprovision.TagVIPPorts(ctx, installConfig, infraID); err != nil {
return err
}

// upload the corresponding image to Glance if rhcosImage contains a
// URL. If rhcosImage contains a name, then that points to an existing
// Glance image.
if imageName, isURL := rhcos.GenerateOpenStackImageName(rhcosImage, infraID); isURL {
if err := preprovision.UploadBaseImage(ctx, installConfig.Config.Platform.OpenStack.Cloud, rhcosImage, imageName, infraID, installConfig.Config.Platform.OpenStack.ClusterOSImageProperties); err != nil {
return err
}
}

return preprovision.SecurityGroups(ctx, installConfig, infraID, mastersSchedulable)
}

var _ clusterapi.InfraReadyProvider = Provider{}

func (p Provider) ControlPlaneAvailable(in clusterapi.ControlPlaneAvailableInput) error {
func (p Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput) error {
var (
k8sClient client.Client = in.Client
infraID string = in.InfraID
installConfig *installconfig.InstallConfig = in.InstallConfig
)
ospCluster := &capo.OpenStackCluster{}
key := client.ObjectKey{
Name: in.InfraID,
Name: infraID,
Namespace: capiutils.Namespace,
}
if err := in.Client.Get(context.Background(), key, ospCluster); err != nil {
if err := k8sClient.Get(ctx, key, ospCluster); err != nil {
return fmt.Errorf("failed to get OSPCluster: %w", err)
}

networkClient, err := openstackdefaults.NewServiceClient("network", openstackdefaults.DefaultClientOpts(in.InstallConfig.Config.Platform.OpenStack.Cloud))
networkClient, err := openstackdefaults.NewServiceClient("network", openstackdefaults.DefaultClientOpts(installConfig.Config.Platform.OpenStack.Cloud))
if err != nil {
return err
}

apiPort, err := createPort(networkClient, "api", in.InfraID, ospCluster.Status.Network.ID)
apiPort, err := createPort(networkClient, "api", infraID, ospCluster.Status.Network.ID, ospCluster.Status.Network.Subnets[0].ID, installConfig.Config.Platform.OpenStack.APIVIPs[0])
if err != nil {
return err
}
if in.InstallConfig.Config.OpenStack.APIFloatingIP != "" {
err = assignFIP(networkClient, in.InstallConfig.Config.OpenStack.APIFloatingIP, apiPort)
if installConfig.Config.OpenStack.APIFloatingIP != "" {
err = assignFIP(networkClient, installConfig.Config.OpenStack.APIFloatingIP, apiPort)
if err != nil {
return err
}
}

ingressPort, err := createPort(networkClient, "ingress", in.InfraID, ospCluster.Status.Network.ID)
ingressPort, err := createPort(networkClient, "ingress", infraID, ospCluster.Status.Network.ID, ospCluster.Status.Network.Subnets[0].ID, installConfig.Config.Platform.OpenStack.IngressVIPs[0])
if err != nil {
return err
}
if in.InstallConfig.Config.OpenStack.IngressFloatingIP != "" {
err = assignFIP(networkClient, in.InstallConfig.Config.OpenStack.IngressFloatingIP, ingressPort)
if installConfig.Config.OpenStack.IngressFloatingIP != "" {
err = assignFIP(networkClient, installConfig.Config.OpenStack.IngressFloatingIP, ingressPort)
if err != nil {
return err
}
Expand All @@ -64,14 +116,32 @@ func (p Provider) ControlPlaneAvailable(in clusterapi.ControlPlaneAvailableInput
return nil
}

func createPort(client *gophercloud.ServiceClient, role, infraID, networkID string) (*ports.Port, error) {
createOtps := ports.CreateOpts{
var _ clusterapi.IgnitionProvider = Provider{}

func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]byte, error) {
logrus.Debugf("Uploading the bootstrap machine's Ignition file to OpenStack")
var (
bootstrapIgnData []byte = in.BootstrapIgnData
infraID string = in.InfraID
installConfig *installconfig.InstallConfig = in.InstallConfig
)

return preprovision.UploadIgnitionAndBuildShim(ctx, installConfig.Config.Platform.OpenStack.Cloud, infraID, installConfig.Config.Proxy, bootstrapIgnData)
}

func createPort(client *gophercloud.ServiceClient, role, infraID, networkID, subnetID, fixedIP string) (*ports.Port, error) {
createOpts := ports.CreateOpts{
Name: fmt.Sprintf("%s-%s-port", infraID, role),
NetworkID: networkID,
Description: "Created By OpenShift Installer",
FixedIPs: []ports.IP{
{
IPAddress: fixedIP,
SubnetID: subnetID,
}},
}

port, err := ports.Create(client, createOtps).Extract()
port, err := ports.Create(client, createOpts).Extract()
if err != nil {
return nil, err
}
Expand Down

0 comments on commit 3531aa5

Please sign in to comment.