client.go

/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clusterctl

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client"
	clusterctllog "sigs.k8s.io/cluster-api/cmd/clusterctl/log"

	"github.com/kubesphere/kubekey/v3/test/e2e/framework/clusterctl/logger"
	"github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log"
)

// Provide E2E friendly wrappers for the clusterctl client library.

const (
	// DefaultFlavor for ConfigClusterInput; use it for getting the cluster-template.yaml file.
	DefaultFlavor = ""
)

const (
	// DefaultInfrastructureProvider for ConfigClusterInput; use it for using the only infrastructure provider installed in a cluster.
	DefaultInfrastructureProvider = ""
)

// InitInput is the input for Init.
type InitInput struct {
	LogFolder               string
	ClusterctlConfigPath    string
	KubeconfigPath          string
	CoreProvider            string
	BootstrapProviders      []string
	ControlPlaneProviders   []string
	InfrastructureProviders []string
}

// Init calls clusterctl init with the list of providers defined in the local repository.
func Init(_ context.Context, input InitInput) {
	log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s --config %s --kubeconfig %s",
		input.CoreProvider,
		strings.Join(input.BootstrapProviders, ","),
		strings.Join(input.ControlPlaneProviders, ","),
		strings.Join(input.InfrastructureProviders, ","),
		input.ClusterctlConfigPath,
		input.KubeconfigPath,
	)

	initOpt := clusterctlclient.InitOptions{
		Kubeconfig: clusterctlclient.Kubeconfig{
			Path:    input.KubeconfigPath,
			Context: "",
		},
		CoreProvider:            input.CoreProvider,
		BootstrapProviders:      input.BootstrapProviders,
		ControlPlaneProviders:   input.ControlPlaneProviders,
		InfrastructureProviders: input.InfrastructureProviders,
		LogUsageInstructions:    true,
		WaitProviders:           true,
	}

	clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-init.log", input.LogFolder)
	defer log.Close()

	_, err := clusterctlClient.Init(initOpt)
	Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl init")
}

// InitWithBinary uses clusterctl binary to run init with the list of providers defined in the local repository.
func InitWithBinary(_ context.Context, binary string, input InitInput) {
	log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s --config %s --kubeconfig %s",
		input.CoreProvider,
		strings.Join(input.BootstrapProviders, ","),
		strings.Join(input.ControlPlaneProviders, ","),
		strings.Join(input.InfrastructureProviders, ","),
		input.ClusterctlConfigPath,
		input.KubeconfigPath,
	)

	args := []string{"init", "--config", input.ClusterctlConfigPath, "--kubeconfig", input.KubeconfigPath}
	if input.CoreProvider != "" {
		args = append(args, "--core", input.CoreProvider)
	}
	if len(input.BootstrapProviders) > 0 {
		args = append(args, "--bootstrap", strings.Join(input.BootstrapProviders, ","))
	}
	if len(input.InfrastructureProviders) > 0 {
		args = append(args, "--infrastructure", strings.Join(input.InfrastructureProviders, ","))
	}

	cmd := exec.Command(binary, args...) //nolint:gosec // We don't care about command injection here.

	out, err := cmd.CombinedOutput()
	_ = os.WriteFile(filepath.Join(input.LogFolder, "clusterctl-init.log"), out, 0644) //nolint:gosec // this is a log file to be shared via prow artifacts
	var stdErr string
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			stdErr = string(exitErr.Stderr)
		}
	}
	Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl init:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr)
}

// UpgradeInput is the input for Upgrade.
type UpgradeInput struct {
	LogFolder                 string
	ClusterctlConfigPath      string
	ClusterctlVariables       map[string]string
	ClusterName               string
	KubeconfigPath            string
	Contract                  string
	CoreProvider              string
	BootstrapProviders        []string
	ControlPlaneProviders     []string
	InfrastructureProviders   []string
	IPAMProviders             []string
	RuntimeExtensionProviders []string
}

// Upgrade calls clusterctl upgrade apply with the list of providers defined in the local repository.
func Upgrade(ctx context.Context, input UpgradeInput) {
	if len(input.ClusterctlVariables) > 0 {
		outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName))
		copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{
			ClusterctlConfigPath: input.ClusterctlConfigPath,
			OutputPath:           outputPath,
			Variables:            input.ClusterctlVariables,
		})
		input.ClusterctlConfigPath = outputPath
	}

	// Check if the user want a custom upgrade
	isCustomUpgrade := input.CoreProvider != "" ||
		len(input.BootstrapProviders) > 0 ||
		len(input.ControlPlaneProviders) > 0 ||
		len(input.InfrastructureProviders) > 0 ||
		len(input.IPAMProviders) > 0 ||
		len(input.RuntimeExtensionProviders) > 0

	Expect((input.Contract != "" && !isCustomUpgrade) || (input.Contract == "" && isCustomUpgrade)).To(BeTrue(), `Invalid arguments. Either the input.Contract parameter or at least one of the following providers has to be set:
		input.CoreProvider, input.BootstrapProviders, input.ControlPlaneProviders, input.InfrastructureProviders, input.IPAMProviders, input.RuntimeExtensionProviders`)

	if isCustomUpgrade {
		log.Logf("clusterctl upgrade apply --core %s --bootstrap %s --control-plane %s --infrastructure %s --ipam %s --runtime-extension %s --config %s --kubeconfig %s",
			input.CoreProvider,
			strings.Join(input.BootstrapProviders, ","),
			strings.Join(input.ControlPlaneProviders, ","),
			strings.Join(input.InfrastructureProviders, ","),
			strings.Join(input.IPAMProviders, ","),
			strings.Join(input.RuntimeExtensionProviders, ","),
			input.ClusterctlConfigPath,
			input.KubeconfigPath,
		)
	} else {
		log.Logf("clusterctl upgrade apply --contract %s --config %s --kubeconfig %s",
			input.Contract,
			input.ClusterctlConfigPath,
			input.KubeconfigPath,
		)
	}

	upgradeOpt := clusterctlclient.ApplyUpgradeOptions{
		Kubeconfig: clusterctlclient.Kubeconfig{
			Path:    input.KubeconfigPath,
			Context: "",
		},
		Contract:                input.Contract,
		CoreProvider:            input.CoreProvider,
		BootstrapProviders:      input.BootstrapProviders,
		ControlPlaneProviders:   input.ControlPlaneProviders,
		InfrastructureProviders: input.InfrastructureProviders,
		WaitProviders:           true,
	}

	clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-upgrade.log", input.LogFolder)
	defer log.Close()

	err := clusterctlClient.ApplyUpgrade(upgradeOpt)
	Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade")
}

// DeleteInput is the input for Delete.
type DeleteInput struct {
	LogFolder            string
	ClusterctlConfigPath string
	KubeconfigPath       string
}

// Delete calls clusterctl delete --all.
func Delete(_ context.Context, input DeleteInput) {
	log.Logf("clusterctl delete --all")

	deleteOpts := clusterctlclient.DeleteOptions{
		Kubeconfig: clusterctlclient.Kubeconfig{
			Path:    input.KubeconfigPath,
			Context: "",
		},
		DeleteAll: true,
	}

	clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-delete.log", input.LogFolder)
	defer log.Close()

	err := clusterctlClient.Delete(deleteOpts)
	Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade")
}

// ConfigClusterInput is the input for ConfigCluster.
type ConfigClusterInput struct {
	LogFolder                string
	ClusterctlConfigPath     string
	KubeconfigPath           string
	InfrastructureProvider   string
	Namespace                string
	ClusterName              string
	KubernetesVersion        string
	ControlPlaneMachineCount *int64
	WorkerMachineCount       *int64
	Flavor                   string
	ClusterctlVariables      map[string]string
}

// ConfigCluster gets a workload cluster based on a template.
func ConfigCluster(ctx context.Context, input ConfigClusterInput) []byte {
	log.Logf("clusterctl config cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s",
		input.ClusterName,
		valueOrDefault(input.InfrastructureProvider),
		input.KubernetesVersion,
		*input.ControlPlaneMachineCount,
		*input.WorkerMachineCount,
		valueOrDefault(input.Flavor),
	)

	templateOptions := clusterctlclient.GetClusterTemplateOptions{
		Kubeconfig: clusterctlclient.Kubeconfig{
			Path:    input.KubeconfigPath,
			Context: "",
		},
		ProviderRepositorySource: &clusterctlclient.ProviderRepositorySourceOptions{
			InfrastructureProvider: input.InfrastructureProvider,
			Flavor:                 input.Flavor,
		},
		ClusterName:              input.ClusterName,
		KubernetesVersion:        input.KubernetesVersion,
		ControlPlaneMachineCount: input.ControlPlaneMachineCount,
		WorkerMachineCount:       input.WorkerMachineCount,
		TargetNamespace:          input.Namespace,
	}

	if len(input.ClusterctlVariables) > 0 {
		outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName))
		copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{
			ClusterctlConfigPath: input.ClusterctlConfigPath,
			OutputPath:           outputPath,
			Variables:            input.ClusterctlVariables,
		})
		input.ClusterctlConfigPath = outputPath
	}

	clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, fmt.Sprintf("%s-cluster-template.yaml", input.ClusterName), input.LogFolder)
	defer log.Close()

	template, err := clusterctlClient.GetClusterTemplate(templateOptions)
	Expect(err).ToNot(HaveOccurred(), "Failed to run clusterctl config cluster")

	yaml, err := template.Yaml()
	Expect(err).ToNot(HaveOccurred(), "Failed to generate yaml for the workload cluster template")

	_, _ = log.WriteString(string(yaml))
	return yaml
}

// ConfigClusterWithBinary uses clusterctl binary to run config cluster or generate cluster.
// NOTE: This func detects the clusterctl version and uses config cluster or generate cluster
// accordingly. We can drop the detection when we don't have to support clusterctl v0.3.x anymore.
func ConfigClusterWithBinary(_ context.Context, clusterctlBinaryPath string, input ConfigClusterInput) []byte {
	log.Logf("Detect clusterctl version via: clusterctl version")

	out, err := exec.Command(clusterctlBinaryPath, "version").Output()
	Expect(err).ToNot(HaveOccurred(), "error running clusterctl version")
	var clusterctlSupportsGenerateCluster bool
	if strings.Contains(string(out), "Major:\"1\"") {
		log.Logf("Detected clusterctl v1.x")
		clusterctlSupportsGenerateCluster = true
	}

	var cmd *exec.Cmd
	if clusterctlSupportsGenerateCluster {
		log.Logf("clusterctl generate cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s",
			input.ClusterName,
			valueOrDefault(input.InfrastructureProvider),
			input.KubernetesVersion,
			*input.ControlPlaneMachineCount,
			*input.WorkerMachineCount,
			valueOrDefault(input.Flavor),
		)
		cmd = exec.Command(clusterctlBinaryPath, "generate", "cluster", //nolint:gosec // We don't care about command injection here.
			input.ClusterName,
			"--infrastructure", input.InfrastructureProvider,
			"--kubernetes-version", input.KubernetesVersion,
			"--control-plane-machine-count", fmt.Sprint(*input.ControlPlaneMachineCount),
			"--worker-machine-count", fmt.Sprint(*input.WorkerMachineCount),
			"--flavor", input.Flavor,
			"--target-namespace", input.Namespace,
			"--config", input.ClusterctlConfigPath,
			"--kubeconfig", input.KubeconfigPath,
		)
	} else {
		log.Logf("clusterctl config cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s",
			input.ClusterName,
			valueOrDefault(input.InfrastructureProvider),
			input.KubernetesVersion,
			*input.ControlPlaneMachineCount,
			*input.WorkerMachineCount,
			valueOrDefault(input.Flavor),
		)
		cmd = exec.Command(clusterctlBinaryPath, "config", "cluster", //nolint:gosec // We don't care about command injection here.
			input.ClusterName,
			"--infrastructure", input.InfrastructureProvider,
			"--kubernetes-version", input.KubernetesVersion,
			"--control-plane-machine-count", fmt.Sprint(*input.ControlPlaneMachineCount),
			"--worker-machine-count", fmt.Sprint(*input.WorkerMachineCount),
			"--flavor", input.Flavor,
			"--target-namespace", input.Namespace,
			"--config", input.ClusterctlConfigPath,
			"--kubeconfig", input.KubeconfigPath,
		)
	}

	out, err = cmd.Output()
	_ = os.WriteFile(filepath.Join(input.LogFolder, fmt.Sprintf("%s-cluster-template.yaml", input.ClusterName)), out, 0644) //nolint:gosec // this is a log file to be shared via prow artifacts
	var stdErr string
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			stdErr = string(exitErr.Stderr)
		}
	}
	Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl config cluster:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr)

	return out
}

// MoveInput is the input for ClusterctlMove.
type MoveInput struct {
	LogFolder            string
	ClusterctlConfigPath string
	FromKubeconfigPath   string
	ToKubeconfigPath     string
	Namespace            string
}

// Move moves workload clusters.
func Move(ctx context.Context, input MoveInput) {
	Expect(ctx).NotTo(BeNil(), "ctx is required for Move")
	Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling Move")
	Expect(input.FromKubeconfigPath).To(BeAnExistingFile(), "Invalid argument. input.FromKubeconfigPath must be an existing file when calling Move")
	Expect(input.ToKubeconfigPath).To(BeAnExistingFile(), "Invalid argument. input.ToKubeconfigPath must be an existing file when calling Move")
	logDir := path.Join(input.LogFolder, "logs", input.Namespace)
	Expect(os.MkdirAll(logDir, 0750)).To(Succeed(), "Invalid argument. input.LogFolder can't be created for Move")

	By("Moving workload clusters")
	log.Logf("clusterctl move --from-kubeconfig %s --to-kubeconfig %s --namespace %s",
		input.FromKubeconfigPath,
		input.ToKubeconfigPath,
		input.Namespace,
	)

	clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-move.log", logDir)
	defer log.Close()
	options := clusterctlclient.MoveOptions{
		FromKubeconfig: clusterctlclient.Kubeconfig{Path: input.FromKubeconfigPath, Context: ""},
		ToKubeconfig:   clusterctlclient.Kubeconfig{Path: input.ToKubeconfigPath, Context: ""},
		Namespace:      input.Namespace,
	}

	Expect(clusterctlClient.Move(options)).To(Succeed(), "Failed to run clusterctl move")
}

func getClusterctlClientWithLogger(configPath, logName, logFolder string) (clusterctlclient.Client, *logger.LogFile) {
	log := logger.OpenLogFile(logger.OpenLogFileInput{
		LogFolder: logFolder,
		Name:      logName,
	})
	clusterctllog.SetLogger(log.Logger())

	c, err := clusterctlclient.New(configPath)
	Expect(err).ToNot(HaveOccurred(), "Failed to create the clusterctl client library")
	return c, log
}

func valueOrDefault(v string) string {
	if v != "" {
		return v
	}
	return "(default)"
}