workload.go

package framework

import (
	"context"
	"errors"
	"fmt"
	"path/filepath"
	"time"

	"github.com/aws/eks-anywhere/internal/pkg/api"
	"github.com/aws/eks-anywhere/pkg/constants"
	"github.com/aws/eks-anywhere/pkg/filewriter"
	"github.com/aws/eks-anywhere/pkg/retrier"
)

const (
	kubectlDeleteTimeout = "20m"
)

type WorkloadCluster struct {
	*ClusterE2ETest
	ManagementClusterKubeconfigFile func() string
}

type WorkloadClusters map[string]*WorkloadCluster

func (w *WorkloadCluster) CreateCluster(opts ...CommandOpt) {
	opts = append(opts, withKubeconfig(w.ManagementClusterKubeconfigFile()))
	w.createCluster(opts...)
}

func (w *WorkloadCluster) UpgradeCluster(clusterOpts []ClusterE2ETestOpt, commandOpts ...CommandOpt) {
	commandOpts = append(commandOpts, withKubeconfig(w.ManagementClusterKubeconfigFile()))
	w.upgradeCluster(clusterOpts, commandOpts...)
}

func (w *WorkloadCluster) DeleteCluster(opts ...CommandOpt) {
	opts = append(opts, withKubeconfig(w.ManagementClusterKubeconfigFile()))
	w.deleteCluster(opts...)
}

// ApplyClusterManifest uses client-side logic to create/update objects defined in a cluster yaml manifest.
func (w *WorkloadCluster) ApplyClusterManifest() {
	ctx := context.Background()
	w.T.Logf("Applying workload cluster %s spec located at %s", w.ClusterName, w.ClusterConfigLocation)
	if err := w.KubectlClient.ApplyManifest(ctx, w.ManagementClusterKubeconfigFile(), w.ClusterConfigLocation); err != nil {
		w.T.Fatalf("Failed to apply workload cluster config: %s", err)
	}
	w.StopIfFailed()
}

// DeleteClusterWithKubectl uses client-side logic to delete a cluster.
func (w *WorkloadCluster) DeleteClusterWithKubectl() {
	ctx := context.Background()
	w.T.Logf("Deleting workload cluster %s with kubectl", w.ClusterName)
	opts := func(params *[]string) {
		*params = append(*params, "--timeout", kubectlDeleteTimeout)
	}
	if err := w.KubectlClient.DeleteManifest(ctx, w.ManagementClusterKubeconfigFile(), w.ClusterConfigLocation, opts); err != nil {
		w.T.Fatalf("Failed to delete workload cluster config: %s", err)
	}
	w.StopIfFailed()
}

// WaitForAvailableHardware waits for workload cluster hardware to be available.
func (w *WorkloadCluster) WaitForAvailableHardware() {
	ctx := context.Background()
	w.T.Logf("Waiting for workload cluster %s hardware to be available", w.ClusterName)
	err := retrier.Retry(240, 5*time.Second, func() error {
		return w.availableHardware(ctx)
	})
	if err != nil {
		w.T.Fatalf("Failed waiting for cluster hardware: %s", err)
	}
}

// WaitForKubeconfig waits for the kubeconfig for the workload cluster to be available and then writes it to disk.
func (w *WorkloadCluster) WaitForKubeconfig() {
	ctx := context.Background()
	w.T.Logf("Waiting for workload cluster %s kubeconfig to be available", w.ClusterName)
	err := retrier.Retry(120, 5*time.Second, func() error {
		return w.writeKubeconfigToDisk(ctx, fmt.Sprintf("%s-kubeconfig", w.ClusterName), w.KubeconfigFilePath())
	})
	if err != nil {
		w.T.Fatalf("Failed waiting for cluster kubeconfig: %s", err)
	}

	if len(w.ClusterConfig.AWSIAMConfigs) != 0 {
		w.T.Logf("Waiting for workload cluster %s iam auth kubeconfig to be available", w.ClusterName)
		err := retrier.Retry(120, 5*time.Second, func() error {
			return w.writeKubeconfigToDisk(ctx, fmt.Sprintf("%s-aws-iam-kubeconfig", w.ClusterName), w.iamAuthKubeconfigFilePath())
		})
		if err != nil {
			w.T.Fatalf("Failed waiting for cluster kubeconfig: %s", err)
		}
	}

	w.T.Logf("Waiting for workload cluster %s control plane to be ready", w.ClusterName)
	if err := w.KubectlClient.WaitForControlPlaneReady(ctx, w.managementCluster(), "15m", w.ClusterName); err != nil {
		w.T.Errorf("Failed waiting for control plane ready: %s", err)
	}
}

// ValidateClusterDelete verifies the cluster has been deleted.
func (w *WorkloadCluster) ValidateClusterDelete() {
	ctx := context.Background()
	w.T.Logf("Validating cluster deletion %s", w.ClusterName)
	clusterStateValidator := newClusterStateValidator(w.clusterStateValidationConfig)
	clusterStateValidator.WithValidations(
		validationsForClusterDoesNotExist()...,
	)
	if err := clusterStateValidator.Validate(ctx); err != nil {
		w.T.Fatalf("failed to validate cluster deletion %v", err)
	}
}

func (w *WorkloadCluster) writeKubeconfigToDisk(ctx context.Context, secretName string, filePath string) error {
	secret, err := w.KubectlClient.GetSecretFromNamespace(ctx, w.ManagementClusterKubeconfigFile(), secretName, constants.EksaSystemNamespace)
	if err != nil {
		return fmt.Errorf("failed to get kubeconfig for cluster: %s", err)
	}
	kubeconfig := secret.Data["value"]
	if err := w.Provider.UpdateKubeConfig(&kubeconfig, w.ClusterName); err != nil {
		return fmt.Errorf("failed to update kubeconfig for cluster: %s", err)
	}
	writer, err := filewriter.NewWriter(w.ClusterConfigFolder)
	if err != nil {
		return fmt.Errorf("failed to write kubeconfig to disk: %v", err)
	}

	_, err = writer.Write(filepath.Base(filePath), kubeconfig, func(op *filewriter.FileOptions) {
		op.IsTemp = false
	})
	if err != nil {
		return fmt.Errorf("failed to write kubeconfig to disk: %v", err)
	}
	return err
}

func (w *WorkloadCluster) availableHardware(ctx context.Context) error {
	hardwareList, err := w.KubectlClient.GetUnprovisionedTinkerbellHardware(ctx, w.ManagementClusterKubeconfigFile(), constants.EksaSystemNamespace)
	if err != nil {
		return fmt.Errorf("failed to get unprovisioned hardware: %s", err)
	}

	cpHardwareRequired := w.ClusterConfig.Cluster.Spec.ControlPlaneConfiguration.Count
	var workerHardwareRequired int
	for _, workerNodeGroup := range w.ClusterConfig.Cluster.Spec.WorkerNodeGroupConfigurations {
		workerHardwareRequired += *workerNodeGroup.Count
	}

	var cpHardwareAvailable int
	var workerHardwareAvailable int

	for _, hardware := range hardwareList {
		switch hardware.Labels[api.HardwareLabelTypeKeyName] {
		case api.ControlPlane:
			cpHardwareAvailable++
		case api.Worker:
			workerHardwareAvailable++
		}
	}

	if cpHardwareAvailable < cpHardwareRequired || workerHardwareAvailable < workerHardwareRequired {
		return errors.New("Insufficient hardware available for cluster")
	}

	return nil
}