tinkerbell.go

package framework

import (
	"fmt"
	"os"
	"strings"
	"testing"

	"github.com/aws/eks-anywhere/internal/pkg/api"
	anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
	releasev1 "github.com/aws/eks-anywhere/release/api/v1alpha1"
	clusterf "github.com/aws/eks-anywhere/test/framework/cluster"
)

const (
	tinkerbellProviderName                       = "tinkerbell"
	tinkerbellBootstrapIPEnvVar                  = "T_TINKERBELL_BOOTSTRAP_IP"
	tinkerbellControlPlaneNetworkCidrEnvVar      = "T_TINKERBELL_CP_NETWORK_CIDR"
	tinkerbellImageUbuntu123EnvVar               = "T_TINKERBELL_IMAGE_UBUNTU_1_23"
	tinkerbellImageUbuntu124EnvVar               = "T_TINKERBELL_IMAGE_UBUNTU_1_24"
	tinkerbellImageUbuntu125EnvVar               = "T_TINKERBELL_IMAGE_UBUNTU_1_25"
	tinkerbellImageUbuntu126EnvVar               = "T_TINKERBELL_IMAGE_UBUNTU_1_26"
	tinkerbellImageUbuntu127EnvVar               = "T_TINKERBELL_IMAGE_UBUNTU_1_27"
	tinkerbellImageUbuntu2204Kubernetes123EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_2204_1_23"
	tinkerbellImageUbuntu2204Kubernetes124EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_2204_1_24"
	tinkerbellImageUbuntu2204Kubernetes125EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_2204_1_25"
	tinkerbellImageUbuntu2204Kubernetes126EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_2204_1_26"
	tinkerbellImageUbuntu2204Kubernetes127EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_2204_1_27"
	tinkerbellImageRedHat123EnvVar               = "T_TINKERBELL_IMAGE_REDHAT_1_23"
	tinkerbellImageRedHat124EnvVar               = "T_TINKERBELL_IMAGE_REDHAT_1_24"
	tinkerbellImageRedHat125EnvVar               = "T_TINKERBELL_IMAGE_REDHAT_1_25"
	tinkerbellImageRedHat126EnvVar               = "T_TINKERBELL_IMAGE_REDHAT_1_26"
	tinkerbellImageRedHat127EnvVar               = "T_TINKERBELL_IMAGE_REDHAT_1_27"
	tinkerbellInventoryCsvFilePathEnvVar         = "T_TINKERBELL_INVENTORY_CSV"
	tinkerbellSSHAuthorizedKey                   = "T_TINKERBELL_SSH_AUTHORIZED_KEY"
	TinkerbellCIEnvironment                      = "T_TINKERBELL_CI_ENVIRONMENT"
)

var requiredTinkerbellEnvVars = []string{
	tinkerbellControlPlaneNetworkCidrEnvVar,
	tinkerbellImageUbuntu123EnvVar,
	tinkerbellImageUbuntu124EnvVar,
	tinkerbellImageUbuntu125EnvVar,
	tinkerbellImageUbuntu126EnvVar,
	tinkerbellImageUbuntu127EnvVar,
	tinkerbellImageUbuntu2204Kubernetes123EnvVar,
	tinkerbellImageUbuntu2204Kubernetes124EnvVar,
	tinkerbellImageUbuntu2204Kubernetes125EnvVar,
	tinkerbellImageUbuntu2204Kubernetes126EnvVar,
	tinkerbellImageUbuntu2204Kubernetes127EnvVar,
	tinkerbellImageRedHat123EnvVar,
	tinkerbellImageRedHat124EnvVar,
	tinkerbellImageRedHat125EnvVar,
	tinkerbellImageRedHat126EnvVar,
	tinkerbellImageRedHat127EnvVar,
	tinkerbellInventoryCsvFilePathEnvVar,
	tinkerbellSSHAuthorizedKey,
}

func RequiredTinkerbellEnvVars() []string {
	return requiredTinkerbellEnvVars
}

type TinkerbellOpt func(*Tinkerbell)

type Tinkerbell struct {
	t                    *testing.T
	fillers              []api.TinkerbellFiller
	clusterFillers       []api.ClusterFiller
	serverIP             string
	cidr                 string
	inventoryCsvFilePath string
}

// UpdateTinkerbellMachineSSHAuthorizedKey updates a tinkerbell machine configs SSHAuthorizedKey.
func UpdateTinkerbellMachineSSHAuthorizedKey() api.TinkerbellMachineFiller {
	return api.WithStringFromEnvVarTinkerbellMachineFiller(tinkerbellSSHAuthorizedKey, api.WithSSHAuthorizedKeyForTinkerbellMachineConfig)
}

func NewTinkerbell(t *testing.T, opts ...TinkerbellOpt) *Tinkerbell {
	checkRequiredEnvVars(t, requiredTinkerbellEnvVars)
	cidr := os.Getenv(tinkerbellControlPlaneNetworkCidrEnvVar)

	serverIP, err := GetIP(cidr, ClusterIPPoolEnvVar)
	if err != nil {
		t.Fatalf("failed to get tinkerbell ip for test environment: %v", err)
	}

	tink := &Tinkerbell{
		t: t,
		fillers: []api.TinkerbellFiller{
			api.WithTinkerbellServer(serverIP),
			api.WithStringFromEnvVarTinkerbell(tinkerbellSSHAuthorizedKey, api.WithSSHAuthorizedKeyForAllTinkerbellMachines),
			api.WithHardwareSelectorLabels(),
		},
	}

	tink.serverIP = serverIP

	tink.cidr = cidr
	tink.inventoryCsvFilePath = os.Getenv(tinkerbellInventoryCsvFilePathEnvVar)

	for _, opt := range opts {
		opt(tink)
	}

	return tink
}

func (t *Tinkerbell) Name() string {
	return tinkerbellProviderName
}

func (t *Tinkerbell) Setup() {}

// UpdateKubeConfig customizes generated kubeconfig for the provider.
func (t *Tinkerbell) UpdateKubeConfig(content *[]byte, clusterName string) error {
	return nil
}

// ClusterConfigUpdates satisfies the test framework Provider.
func (t *Tinkerbell) ClusterConfigUpdates() []api.ClusterConfigFiller {
	clusterIP, err := GetIP(t.cidr, ClusterIPPoolEnvVar)
	if err != nil {
		t.t.Fatalf("failed to get cluster ip for test environment: %v", err)
	}

	f := make([]api.ClusterFiller, 0, len(t.clusterFillers)+1)
	f = append(f, t.clusterFillers...)
	f = append(f, api.WithControlPlaneEndpointIP(clusterIP))

	return []api.ClusterConfigFiller{api.ClusterToConfigFiller(f...), api.TinkerbellToConfigFiller(t.fillers...)}
}

func (t *Tinkerbell) WithProviderUpgrade(fillers ...api.TinkerbellFiller) ClusterE2ETestOpt {
	return func(e *ClusterE2ETest) {
		e.UpdateClusterConfig(api.TinkerbellToConfigFiller(fillers...))
	}
}

func (t *Tinkerbell) CleanupVMs(_ string) error {
	return nil
}

// WithKubeVersionAndOS returns a cluster config filler that sets the cluster kube version and the right image for all
// tinkerbell machine configs.
func (t *Tinkerbell) WithKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, release *releasev1.EksARelease) api.ClusterConfigFiller {
	return api.JoinClusterConfigFillers(
		api.ClusterToConfigFiller(api.WithKubernetesVersion(kubeVersion)),
		api.TinkerbellToConfigFiller(
			imageForKubeVersionAndOS(kubeVersion, os),
			api.WithOsFamilyForAllTinkerbellMachines(osFamiliesForOS[os]),
		),
	)
}

// WithNewWorkerNodeGroup returns an api.ClusterFiller that adds a new workerNodeGroupConfiguration and
// a corresponding TinkerbellMachineConfig to the cluster config.
func (t *Tinkerbell) WithNewWorkerNodeGroup(name string, workerNodeGroup *WorkerNodeGroup) api.ClusterConfigFiller {
	// TODO: Implement for Tinkerbell provider
	panic("Not implemented for Tinkerbell yet")
}

func envVarForImage(os OS, kubeVersion anywherev1.KubernetesVersion) string {
	imageEnvVar := fmt.Sprintf("T_TINKERBELL_IMAGE_%s_%s", strings.ToUpper(strings.ReplaceAll(string(os), "-", "_")), strings.ReplaceAll(string(kubeVersion), ".", "_"))
	return imageEnvVar
}

// withKubeVersionAndOS returns a cluster config filler that sets the cluster kube version and the right image for all
// tinkerbell machine configs.
func withKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, os OS, release *releasev1.EksARelease) TinkerbellOpt {
	return func(t *Tinkerbell) {
		t.fillers = append(t.fillers,
			imageForKubeVersionAndOS(kubeVersion, os),
			api.WithOsFamilyForAllTinkerbellMachines(osFamiliesForOS[os]),
		)
	}
}

// WithUbuntu123Tinkerbell tink test with ubuntu 1.23.
func WithUbuntu123Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube123, Ubuntu2004, nil)
}

// WithUbuntu124Tinkerbell tink test with ubuntu 1.24.
func WithUbuntu124Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004, nil)
}

// WithUbuntu125Tinkerbell tink test with ubuntu 1.25.
func WithUbuntu125Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004, nil)
}

// WithUbuntu126Tinkerbell tink test with ubuntu 1.26.
func WithUbuntu126Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004, nil)
}

// WithUbuntu127Tinkerbell tink test with ubuntu 1.27.
func WithUbuntu127Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004, nil)
}

// WithRedHat123Tinkerbell tink test with redhat 1.23.
func WithRedHat123Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube123, RedHat8, nil)
}

// WithRedHat124Tinkerbell tink test with redhat 1.24.
func WithRedHat124Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube124, RedHat8, nil)
}

// WithRedHat125Tinkerbell tink test with redhat 1.25.
func WithRedHat125Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube125, RedHat8, nil)
}

// WithRedHat126Tinkerbell tink test with redhat 1.26.
func WithRedHat126Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube126, RedHat8, nil)
}

// WithRedHat127Tinkerbell tink test with redhat 1.27.
func WithRedHat127Tinkerbell() TinkerbellOpt {
	return withKubeVersionAndOS(anywherev1.Kube127, RedHat8, nil)
}

func WithBottleRocketTinkerbell() TinkerbellOpt {
	return func(t *Tinkerbell) {
		t.fillers = append(t.fillers,
			api.WithOsFamilyForAllTinkerbellMachines(anywherev1.Bottlerocket),
		)
	}
}

func WithTinkerbellExternalEtcdTopology(count int) TinkerbellOpt {
	return func(t *Tinkerbell) {
		t.fillers = append([]api.TinkerbellFiller{api.WithTinkerbellEtcdMachineConfig()}, t.fillers...)
		t.clusterFillers = append(t.clusterFillers, api.WithExternalEtcdTopology(count), api.WithExternalEtcdMachineRef(anywherev1.TinkerbellMachineConfigKind))
	}
}

func WithCustomTinkerbellMachineConfig(selector string) TinkerbellOpt {
	return func(t *Tinkerbell) {
		t.fillers = append([]api.TinkerbellFiller{api.WithCustomTinkerbellMachineConfig(selector)}, t.fillers...)
	}
}

// ClusterStateValidations returns a list of provider specific validations.
func (t *Tinkerbell) ClusterStateValidations() []clusterf.StateValidation {
	return []clusterf.StateValidation{}
}

// WithOSImageURL Modify OS Image url.
func WithOSImageURL(url string) TinkerbellOpt {
	return func(t *Tinkerbell) {
		t.fillers = append(t.fillers,
			api.WithTinkerbellOSImageURL(url),
		)
	}
}

// WithHookImagesURLPath Modify Hook OS Image url.
func WithHookImagesURLPath(url string) TinkerbellOpt {
	return func(t *Tinkerbell) {
		t.fillers = append(t.fillers,
			api.WithHookImagesURLPath(url),
		)
	}
}

func imageForKubeVersionAndOS(kubeVersion anywherev1.KubernetesVersion, operatingSystem OS) api.TinkerbellFiller {
	return api.WithTinkerbellOSImageURL(os.Getenv(envVarForImage(operatingSystem, kubeVersion)))
}

// Ubuntu123Image represents an Ubuntu raw image corresponding to Kubernetes 1.23.
func Ubuntu123Image() api.TinkerbellFiller {
	return imageForKubeVersionAndOS(anywherev1.Kube123, Ubuntu2004)
}

// Ubuntu124Image represents an Ubuntu raw image corresponding to Kubernetes 1.24.
func Ubuntu124Image() api.TinkerbellFiller {
	return imageForKubeVersionAndOS(anywherev1.Kube124, Ubuntu2004)
}

// Ubuntu125Image represents an Ubuntu raw image corresponding to Kubernetes 1.25.
func Ubuntu125Image() api.TinkerbellFiller {
	return imageForKubeVersionAndOS(anywherev1.Kube125, Ubuntu2004)
}

// Ubuntu126Image represents an Ubuntu raw image corresponding to Kubernetes 1.26.
func Ubuntu126Image() api.TinkerbellFiller {
	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2004)
}

// Ubuntu127Image represents an Ubuntu raw image corresponding to Kubernetes 1.27.
func Ubuntu127Image() api.TinkerbellFiller {
	return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2004)
}

// Ubuntu2204Kubernetes126Image represents an Ubuntu 22.04 raw image corresponding to Kubernetes 1.26.
func Ubuntu2204Kubernetes126Image() api.TinkerbellFiller {
	return imageForKubeVersionAndOS(anywherev1.Kube126, Ubuntu2204)
}

// Ubuntu2204Kubernetes127Image represents an Ubuntu 22.04 raw image corresponding to Kubernetes 1.27.
func Ubuntu2204Kubernetes127Image() api.TinkerbellFiller {
	return imageForKubeVersionAndOS(anywherev1.Kube127, Ubuntu2204)
}