cluster.go

/*
Copyright © 2020-2023 The k3d Author(s)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package client

import (
	"bytes"
	"context"
	_ "embed"
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/docker/go-connections/nat"
	"github.com/imdario/mergo"
	copystruct "github.com/mitchellh/copystructure"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
	"k8s.io/utils/strings/slices"
	"sigs.k8s.io/yaml"

	wharfie "github.com/rancher/wharfie/pkg/registries"

	"github.com/k3d-io/k3d/v5/pkg/actions"
	config "github.com/k3d-io/k3d/v5/pkg/config/v1alpha5"
	l "github.com/k3d-io/k3d/v5/pkg/logger"
	k3drt "github.com/k3d-io/k3d/v5/pkg/runtimes"
	runtimeErr "github.com/k3d-io/k3d/v5/pkg/runtimes/errors"
	k3d "github.com/k3d-io/k3d/v5/pkg/types"
	"github.com/k3d-io/k3d/v5/pkg/types/k3s"
	"github.com/k3d-io/k3d/v5/pkg/util"
	goyaml "gopkg.in/yaml.v2"
)

// ClusterRun orchestrates the steps of cluster creation, configuration and starting
func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *config.ClusterConfig) error {
	/*
	 * Step 0: (Infrastructure) Preparation
	 */
	if err := ClusterPrep(ctx, runtime, clusterConfig); err != nil {
		return fmt.Errorf("Failed Cluster Preparation: %+v", err)
	}

	// Create tools-node for later steps

	g, _ := errgroup.WithContext(ctx)
	g.Go(func() error {
		_, err := EnsureToolsNode(ctx, runtime, &clusterConfig.Cluster)
		return err
	})

	/*
	 * Step 1: Create Containers
	 */
	if err := ClusterCreate(ctx, runtime, &clusterConfig.Cluster, &clusterConfig.ClusterCreateOpts); err != nil {
		return fmt.Errorf("failed Cluster Creation: %+v", err)
	}

	// Wait for tools node to be available
	if err := g.Wait(); err != nil {
		return fmt.Errorf("failed to ensure tools node: %w", err)
	}

	/*
	 * Step 2: Pre-Start Configuration
	 */
	// Gather Environment information, e.g. the host gateway address
	envInfo, err := GatherEnvironmentInfo(ctx, runtime, &clusterConfig.Cluster)
	if err != nil {
		return fmt.Errorf("failed to gather environment information used for cluster creation: %w", err)
	}

	/*
	 * Step 3: Start Containers
	 */
	if err := ClusterStart(ctx, runtime, &clusterConfig.Cluster, k3d.ClusterStartOpts{
		WaitForServer:   clusterConfig.ClusterCreateOpts.WaitForServer,
		Timeout:         clusterConfig.ClusterCreateOpts.Timeout, // TODO: here we should consider the time used so far
		NodeHooks:       clusterConfig.ClusterCreateOpts.NodeHooks,
		EnvironmentInfo: envInfo,
		Intent:          k3d.IntentClusterCreate,
		HostAliases:     clusterConfig.ClusterCreateOpts.HostAliases,
	}); err != nil {
		return fmt.Errorf("Failed Cluster Start: %+v", err)
	}

	/*
	 * Post-Start Configuration
	 */
	/**********************************
	 * Additional Cluster Preparation *
	 **********************************/

	// create the registry hosting configmap
	if len(clusterConfig.ClusterCreateOpts.Registries.Use) > 0 {
		if err := prepCreateLocalRegistryHostingConfigMap(ctx, runtime, &clusterConfig.Cluster); err != nil {
			l.Log().Warnf("Failed to create LocalRegistryHosting ConfigMap: %+v", err)
		}
	}

	return nil
}

// ClusterPrep takes care of the steps required before creating/starting the cluster containers
func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *config.ClusterConfig) error {
	/*
	 * Set up contexts
	 * Used for (early) termination (across API boundaries)
	 */
	clusterPrepCtx := ctx
	if clusterConfig.ClusterCreateOpts.Timeout > 0*time.Second {
		var cancelClusterPrepCtx context.CancelFunc
		clusterPrepCtx, cancelClusterPrepCtx = context.WithTimeout(ctx, clusterConfig.ClusterCreateOpts.Timeout)
		defer cancelClusterPrepCtx()
	}

	/*
	 * Step 0: Pre-Pull Images
	 */
	// TODO: ClusterPrep: add image pre-pulling step

	/*
	 * Step 1: Network
	 */
	if err := ClusterPrepNetwork(clusterPrepCtx, runtime, &clusterConfig.Cluster, &clusterConfig.ClusterCreateOpts); err != nil {
		return fmt.Errorf("Failed Network Preparation: %+v", err)
	}

	/*
	 * Step 2: Volume(s)
	 */
	if !clusterConfig.ClusterCreateOpts.DisableImageVolume {
		if err := ClusterPrepImageVolume(ctx, runtime, &clusterConfig.Cluster, &clusterConfig.ClusterCreateOpts); err != nil {
			return fmt.Errorf("Failed Image Volume Preparation: %+v", err)
		}
	}

	/*
	 * Step 3: Registries
	 */

	// Ensure referenced registries
	for _, reg := range clusterConfig.ClusterCreateOpts.Registries.Use {
		l.Log().Debugf("Trying to find registry %s", reg.Host)
		regNode, err := runtime.GetNode(ctx, &k3d.Node{Name: reg.Host})
		if err != nil {
			return fmt.Errorf("Failed to find registry node '%s': %+v", reg.Host, err)
		}
		regFromNode, err := RegistryFromNode(regNode)
		if err != nil {
			return fmt.Errorf("failed to translate node to registry spec: %w", err)
		}
		*reg = *regFromNode
	}

	// Create managed registry bound to this cluster
	if clusterConfig.ClusterCreateOpts.Registries.Create != nil {
		registryNode, err := RegistryCreate(ctx, runtime, clusterConfig.ClusterCreateOpts.Registries.Create)
		if err != nil {
			return fmt.Errorf("Failed to create registry: %+v", err)
		}

		clusterConfig.Cluster.Nodes = append(clusterConfig.Cluster.Nodes, registryNode)

		clusterConfig.ClusterCreateOpts.Registries.Use = append(clusterConfig.ClusterCreateOpts.Registries.Use, clusterConfig.ClusterCreateOpts.Registries.Create)
	}

	// Use existing registries (including the new one, if created)
	l.Log().Tracef("Using Registries: %+v", clusterConfig.ClusterCreateOpts.Registries.Use)

	var registryConfig *wharfie.Registry

	if len(clusterConfig.ClusterCreateOpts.Registries.Use) > 0 {
		// ensure that all selected registries exist and connect them to the cluster network
		for _, externalReg := range clusterConfig.ClusterCreateOpts.Registries.Use {
			regNode, err := runtime.GetNode(ctx, &k3d.Node{Name: externalReg.Host})
			if err != nil {
				return fmt.Errorf("Failed to find registry node '%s': %+v", externalReg.Host, err)
			}
			if err := RegistryConnectNetworks(ctx, runtime, regNode, []string{clusterConfig.Cluster.Network.Name}); err != nil {
				return fmt.Errorf("Failed to connect registry node '%s' to cluster network: %+v", regNode.Name, err)
			}
		}

		// generate the registries.yaml
		regConf, err := RegistryGenerateK3sConfig(ctx, clusterConfig.ClusterCreateOpts.Registries.Use)
		if err != nil {
			return fmt.Errorf("Failed to generate registry config file for k3s: %+v", err)
		}

		// generate the LocalRegistryHosting configmap
		regCm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx, runtime, clusterConfig.ClusterCreateOpts.Registries.Use)
		if err != nil {
			return fmt.Errorf("Failed to generate LocalRegistryHosting configmap: %+v", err)
		}
		l.Log().Tracef("Writing LocalRegistryHosting YAML:\n%s", string(regCm))
		clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{
			Stage: k3d.LifecycleStagePreStart,
			Action: actions.WriteFileAction{
				Runtime:     runtime,
				Content:     regCm,
				Dest:        k3d.DefaultLocalRegistryHostingConfigmapTempPath,
				Mode:        0644,
				Description: "Write LocalRegistryHosting Configmap",
			},
		})

		registryConfig = regConf
	}
	// merge with pre-existing, referenced registries.yaml
	if clusterConfig.ClusterCreateOpts.Registries.Config != nil {
		if registryConfig != nil {
			if err := RegistryMergeConfig(ctx, registryConfig, clusterConfig.ClusterCreateOpts.Registries.Config); err != nil {
				return err
			}
			l.Log().Tracef("Merged registry config: %+v", registryConfig)
		} else {
			registryConfig = clusterConfig.ClusterCreateOpts.Registries.Config
		}
	}
	if registryConfig != nil {
		regConfBytes, err := goyaml.Marshal(&registryConfig)
		if err != nil {
			return fmt.Errorf("Failed to marshal registry configuration: %+v", err)
		}
		clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{
			Stage: k3d.LifecycleStagePreStart,
			Action: actions.WriteFileAction{
				Runtime:     runtime,
				Content:     regConfBytes,
				Dest:        k3d.DefaultRegistriesFilePath,
				Mode:        0644,
				Description: "Write Registry Configuration",
			},
		})
	}

	/*
	 * Step 4: Files
	 */

	for id, node := range clusterConfig.Nodes {
		for _, nodefile := range node.Files {
			clusterConfig.Nodes[id].HookActions = append(clusterConfig.Nodes[id].HookActions, k3d.NodeHook{
				Stage: k3d.LifecycleStagePreStart,
				Action: actions.WriteFileAction{
					Runtime:     runtime,
					Content:     nodefile.Content,
					Dest:        nodefile.Destination,
					Mode:        0644,
					Description: nodefile.Description,
				},
			})
		}
	}

	return nil
}

// ClusterPrepNetwork creates a new cluster network, if needed or sets everything up to re-use an existing network
func ClusterPrepNetwork(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterCreateOpts *k3d.ClusterCreateOpts) error {
	l.Log().Infoln("Prep: Network")

	// error out if external cluster network should be used but no name was set
	if cluster.Network.Name == "" && cluster.Network.External {
		return fmt.Errorf("Failed to use external network because no name was specified")
	}

	if cluster.Network.Name != "" && cluster.Network.External && cluster.Network.IPAM.IPPrefix.IsValid() {
		return fmt.Errorf("cannot specify subnet for exiting network")
	}

	// generate cluster network name, if not set
	if cluster.Network.Name == "" && !cluster.Network.External {
		cluster.Network.Name = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, cluster.Name)
	}

	// handle hostnetwork
	if cluster.Network.Name == "host" {
		if len(cluster.Nodes) > 1 {
			return fmt.Errorf("only one server node supported when using host network")
		}
	}

	// create cluster network or use an existing one
	network, networkExists, err := runtime.CreateNetworkIfNotPresent(ctx, &cluster.Network)
	if err != nil {
		return fmt.Errorf("failed to create cluster network: %w", err)
	}
	cluster.Network = *network
	clusterCreateOpts.GlobalLabels[k3d.LabelNetworkID] = network.ID
	clusterCreateOpts.GlobalLabels[k3d.LabelNetwork] = cluster.Network.Name
	clusterCreateOpts.GlobalLabels[k3d.LabelNetworkIPRange] = cluster.Network.IPAM.IPPrefix.String()
	clusterCreateOpts.GlobalLabels[k3d.LabelNetworkExternal] = strconv.FormatBool(cluster.Network.External)
	if networkExists {
		l.Log().Infof("Re-using existing network '%s' (%s)", network.Name, network.ID)
		clusterCreateOpts.GlobalLabels[k3d.LabelNetworkExternal] = "true" // if the network wasn't created, we say that it's managed externally (important for cluster deletion)
	}

	// just reserve some IPs for k3d (e.g. k3d-tools container), so we don't try to use them again
	if cluster.Network.IPAM.Managed {
		reservedIP, err := GetIP(ctx, runtime, &cluster.Network)
		if err != nil {
			return fmt.Errorf("error reserving IP in new cluster network %s", network.Name)
		}
		cluster.Network.IPAM.IPsUsed = append(cluster.Network.IPAM.IPsUsed, reservedIP)
	}

	return nil
}

func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterCreateOpts *k3d.ClusterCreateOpts) error {
	/*
	 * Cluster-Wide volumes
	 * - image volume (for importing images)
	 */
	imageVolumeName := fmt.Sprintf("%s-%s-images", k3d.DefaultObjectNamePrefix, cluster.Name)
	if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
		return fmt.Errorf("failed to create image volume '%s' for cluster '%s': %w", imageVolumeName, cluster.Name, err)
	}
	l.Log().Infof("Created image volume %s", imageVolumeName)

	clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName
	cluster.ImageVolume = imageVolumeName
	cluster.Volumes = append(cluster.Volumes, imageVolumeName)

	// attach volume to nodes
	for _, node := range cluster.Nodes {
		node.Volumes = append(node.Volumes, fmt.Sprintf("%s:%s", imageVolumeName, k3d.DefaultImageVolumeMountPath))
	}
	return nil
}

// ClusterCreate creates a new cluster consisting of
// - some containerized k3s nodes
// - a docker network
func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterCreateOpts *k3d.ClusterCreateOpts) error {
	l.Log().Tracef(`
===== Creating Cluster =====

Runtime:
%+v

Cluster:
%+v

ClusterCreatOpts:
%+v

============================
	`, runtime, cluster, clusterCreateOpts)

	/*
	 * Set up contexts
	 * Used for (early) termination (across API boundaries)
	 */
	clusterCreateCtx := ctx
	if clusterCreateOpts.Timeout > 0*time.Second {
		var cancelClusterCreateCtx context.CancelFunc
		clusterCreateCtx, cancelClusterCreateCtx = context.WithTimeout(ctx, clusterCreateOpts.Timeout)
		defer cancelClusterCreateCtx()
	}

	/*
	 * Docker Machine Special Configuration
	 */
	if cluster.KubeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker {
		// If the runtime is docker, attempt to use the docker host
		if runtime == k3drt.Docker {
			dockerHost := runtime.GetHost()
			if dockerHost != "" {
				dockerHost = strings.Split(dockerHost, ":")[0] // remove the port
				l.Log().Tracef("Using docker host %s", dockerHost)
				cluster.KubeAPI.Host = dockerHost
			}
		}
	}

	/*
	 * Cluster Token
	 */

	if cluster.Token == "" {
		cluster.Token = GenerateClusterToken()
	}
	clusterCreateOpts.GlobalLabels[k3d.LabelClusterToken] = cluster.Token

	/*
	 * Extra Labels
	 */
	if len(clusterCreateOpts.HostAliases) > 0 {
		hostAliasesJSON, err := json.Marshal(clusterCreateOpts.HostAliases)
		if err != nil {
			return fmt.Errorf("error marshalling hostaliases: %w", err)
		}

		clusterCreateOpts.GlobalLabels[k3d.LabelClusterStartHostAliases] = string(hostAliasesJSON)
	}

	/*
	 * Nodes
	 */

	clusterCreateOpts.GlobalLabels[k3d.LabelClusterName] = cluster.Name
	// Add serverlb url to be used as tls-san value
	// This is used to avoid a fatal error on registering server nodes
	// using loadbalancer
	if clusterCreateOpts.DisableLoadBalancer {
		clusterCreateOpts.GlobalLabels[k3d.LabelServerLoadBalancer] = ""
	} else {
		clusterCreateOpts.GlobalLabels[k3d.LabelServerLoadBalancer] = fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name)
	}

	// agent defaults (per cluster)
	// connection url is always the name of the first server node (index 0) // TODO: change this to the server loadbalancer
	connectionURL := fmt.Sprintf("https://%s:%s", GenerateNodeName(cluster.Name, k3d.ServerRole, 0), k3d.DefaultAPIPort)
	clusterCreateOpts.GlobalLabels[k3d.LabelClusterURL] = connectionURL
	clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("%s=%s", k3s.EnvClusterToken, cluster.Token))

	nodeSetup := func(node *k3d.Node) error {
		// cluster specific settings
		if node.RuntimeLabels == nil {
			node.RuntimeLabels = make(map[string]string) // TODO: maybe create an init function?
		}

		// ensure global labels
		for k, v := range clusterCreateOpts.GlobalLabels {
			node.RuntimeLabels[k] = v
		}

		// ensure global env
		node.Env = append(node.Env, clusterCreateOpts.GlobalEnv...)

		// node role specific settings
		if node.Role == k3d.ServerRole {
			if cluster.Network.IPAM.Managed {
				ip, err := GetIP(ctx, runtime, &cluster.Network)
				if err != nil {
					return fmt.Errorf("failed to find free IP in network %s: %w", cluster.Network.Name, err)
				}
				cluster.Network.IPAM.IPsUsed = append(cluster.Network.IPAM.IPsUsed, ip) // make sure that we're not reusing the same IP next time
				node.IP.Static = true
				node.IP.IP = ip
				node.RuntimeLabels[k3d.LabelNodeStaticIP] = ip.String()
			}

			node.ServerOpts.KubeAPI = cluster.KubeAPI

			// the cluster has an init server node, but its not this one, so connect it to the init node
			if cluster.InitNode != nil && !node.ServerOpts.IsInit {
				node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterConnectURL, connectionURL))
				node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
			}
		} else if node.Role == k3d.AgentRole {
			node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterConnectURL, connectionURL))
		}

		node.Networks = []string{cluster.Network.Name}
		node.Restart = true
		node.GPURequest = clusterCreateOpts.GPURequest

		// create node
		l.Log().Infof("Creating node '%s'", node.Name)
		if err := NodeCreate(clusterCreateCtx, runtime, node, k3d.NodeCreateOpts{}); err != nil {
			return fmt.Errorf("failed to create node: %w", err)
		}
		l.Log().Debugf("Created node '%s'", node.Name)
		return nil
	}

	// used for node suffices
	serverCount := 0

	// create init node first
	if cluster.InitNode != nil {
		l.Log().Infoln("Creating initializing server node")
		cluster.InitNode.Args = append(cluster.InitNode.Args, "--cluster-init")
		if cluster.InitNode.RuntimeLabels == nil {
			cluster.InitNode.RuntimeLabels = map[string]string{}
		}
		cluster.InitNode.RuntimeLabels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server

		// in case the LoadBalancer was disabled, expose the API Port on the initializing server node
		if clusterCreateOpts.DisableLoadBalancer {
			if cluster.InitNode.Ports == nil {
				cluster.InitNode.Ports = nat.PortMap{}
			}
			cluster.InitNode.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
		}

		if err := nodeSetup(cluster.InitNode); err != nil {
			return fmt.Errorf("failed init node setup: %w", err)
		}
		serverCount++
	}

	// create all other nodes, but skip the init node
	for _, node := range cluster.Nodes {
		if node.Role == k3d.ServerRole {
			// skip the init node here
			if node == cluster.InitNode {
				continue
			} else if serverCount == 0 && clusterCreateOpts.DisableLoadBalancer {
				// if this is the first server node and the server loadbalancer is disabled, expose the API Port on this server node
				if node.Ports == nil {
					node.Ports = nat.PortMap{}
				}
				node.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
			}

			time.Sleep(1 * time.Second) // FIXME: arbitrary wait for one second to avoid race conditions of servers registering

			serverCount++
		}
		if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
			if err := nodeSetup(node); err != nil {
				return fmt.Errorf("failed setup of server/agent node %s: %w", node.Name, err)
			}
		}
	}

	// WARN, if there are exactly two server nodes: that means we're using etcd, but don't have fault tolerance
	if serverCount == 2 {
		l.Log().Warnln("You're creating 2 server nodes: Please consider creating at least 3 to achieve etcd quorum & fault tolerance")
	}

	/*
	 * Auxiliary Containers
	 */
	// *** ServerLoadBalancer ***
	if !clusterCreateOpts.DisableLoadBalancer {
		if cluster.ServerLoadBalancer == nil {
			l.Log().Infof("No loadbalancer specified, creating a default one...")
			cluster.ServerLoadBalancer = k3d.NewLoadbalancer()
			var err error
			cluster.ServerLoadBalancer.Node, err = LoadbalancerPrepare(ctx, runtime, cluster, &k3d.LoadbalancerCreateOpts{Labels: clusterCreateOpts.GlobalLabels})
			if err != nil {
				return fmt.Errorf("failed to prepare loadbalancer: %w", err)
			}
			cluster.Nodes = append(cluster.Nodes, cluster.ServerLoadBalancer.Node) // append lbNode to list of cluster nodes, so it will be considered during rollback
		}

		if len(cluster.ServerLoadBalancer.Config.Ports) == 0 {
			lbConfig, err := LoadbalancerGenerateConfig(cluster)
			if err != nil {
				return fmt.Errorf("error generating loadbalancer config: %v", err)
			}
			cluster.ServerLoadBalancer.Config = &lbConfig
		}

		// ensure labels
		cluster.ServerLoadBalancer.Node.FillRuntimeLabels()
		for k, v := range clusterCreateOpts.GlobalLabels {
			cluster.ServerLoadBalancer.Node.RuntimeLabels[k] = v
		}

		// prepare to write config to lb container
		configyaml, err := yaml.Marshal(cluster.ServerLoadBalancer.Config)
		if err != nil {
			return fmt.Errorf("failed to marshal loadbalancer config: %w", err)
		}

		writeLbConfigAction := k3d.NodeHook{
			Stage: k3d.LifecycleStagePreStart,
			Action: actions.WriteFileAction{
				Runtime:     runtime,
				Dest:        k3d.DefaultLoadbalancerConfigPath,
				Mode:        0744,
				Content:     configyaml,
				Description: "Write Loadbalancer Configuration",
			},
		}

		cluster.ServerLoadBalancer.Node.HookActions = append(cluster.ServerLoadBalancer.Node.HookActions, writeLbConfigAction)
		cluster.ServerLoadBalancer.Node.Restart = true

		l.Log().Infof("Creating LoadBalancer '%s'", cluster.ServerLoadBalancer.Node.Name)
		if err := NodeCreate(ctx, runtime, cluster.ServerLoadBalancer.Node, k3d.NodeCreateOpts{}); err != nil {
			return fmt.Errorf("error creating loadbalancer: %v", err)
		}
		l.Log().Debugf("Created loadbalancer '%s'", cluster.ServerLoadBalancer.Node.Name)
	}

	return nil
}

// ClusterDelete deletes an existing cluster
func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, opts k3d.ClusterDeleteOpts) error {
	l.Log().Infof("Deleting cluster '%s'", cluster.Name)
	cluster, err := ClusterGet(ctx, runtime, cluster)
	if err != nil {
		return fmt.Errorf("failed to get cluster: %w", err)
	}
	l.Log().Debugf("Cluster Details: %+v", cluster)

	failed := 0
	for _, node := range cluster.Nodes {
		// registry: only delete, if not connected to other networks
		if node.Role == k3d.RegistryRole && !opts.SkipRegistryCheck {
			l.Log().Tracef("Registry Node has %d networks: %+v", len(node.Networks), node)

			// check if node is connected to other networks, that are not
			// - the cluster network
			// - default docker networks
			// -> if so, disconnect it from the cluster network and continue
			connectedToOtherNet := false
			for _, net := range node.Networks {
				if net == cluster.Network.Name {
					continue
				}
				if net == k3d.DefaultRuntimeNetwork || net == "host" {
					continue
				}
				l.Log().Tracef("net: %s", net)
				connectedToOtherNet = true
				break
			}
			if connectedToOtherNet {
				l.Log().Infof("Registry %s is also connected to other (non-default) networks (%+v), not deleting it...", node.Name, node.Networks)
				if err := runtime.DisconnectNodeFromNetwork(ctx, node, cluster.Network.Name); err != nil {
					l.Log().Warnf("Failed to disconnect registry %s from cluster network %s", node.Name, cluster.Network.Name)
				}
				continue
			}
		}

		if err := NodeDelete(ctx, runtime, node, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
			l.Log().Warningf("Failed to delete node '%s': Try to delete it manually", node.Name)
			failed++
			continue
		}
	}

	// Delete the cluster network, if it was created for/by this cluster (and if it's not in use anymore)
	if cluster.Network.Name != "" {
		if !cluster.Network.External {
			l.Log().Infof("Deleting cluster network '%s'", cluster.Network.Name)
			if err := runtime.DeleteNetwork(ctx, cluster.Network.Name); err != nil {
				if errors.Is(err, runtimeErr.ErrRuntimeNetworkNotEmpty) { // there are still containers connected to that network
					connectedNodes, err := runtime.GetNodesInNetwork(ctx, cluster.Network.Name) // check, if there are any k3d nodes connected to the cluster
					if err != nil {
						l.Log().Warningf("Failed to check cluster network for connected nodes: %+v", err)
					}

					if len(connectedNodes) > 0 { // there are still k3d-managed containers (aka nodes) connected to the network
						connectedRegistryNodes := util.FilterNodesByRole(connectedNodes, k3d.RegistryRole)
						if len(connectedRegistryNodes) == len(connectedNodes) { // only registry node(s) left in the network
							for _, node := range connectedRegistryNodes {
								l.Log().Debugf("Disconnecting registry node %s from the network...", node.Name)
								if err := runtime.DisconnectNodeFromNetwork(ctx, node, cluster.Network.Name); err != nil {
									l.Log().Warnf("Failed to disconnect registry %s from network %s", node.Name, cluster.Network.Name)
								} else {
									if err := runtime.DeleteNetwork(ctx, cluster.Network.Name); err != nil {
										l.Log().Warningf("Failed to delete cluster network, even after disconnecting registry node(s): %+v", err)
									}
								}
							}
						} else { // besides the registry node(s), there are still other nodes... maybe they still need a registry
							l.Log().Debugf("There are some non-registry nodes left in the network")
						}
					} else {
						l.Log().Warningf("Failed to delete cluster network '%s' because it's still in use: is there another cluster using it?", cluster.Network.Name)
					}
				} else {
					l.Log().Warningf("Failed to delete cluster network '%s': '%+v'", cluster.Network.Name, err)
				}
			}
		} else if cluster.Network.External {
			l.Log().Debugf("Skip deletion of cluster network '%s' because it's managed externally", cluster.Network.Name)
		}
	}

	// delete managed volumes attached to this cluster
	l.Log().Infof("Deleting %d attached volumes...", len(cluster.Volumes))
	for _, vol := range cluster.Volumes {
		l.Log().Debugf("Deleting volume %s...", vol)
		if err := runtime.DeleteVolume(ctx, vol); err != nil {
			l.Log().Warningf("Failed to delete volume '%s' of cluster '%s': %v -> Try to delete it manually", cluster.ImageVolume, cluster.Name, err)
		}
	}

	// return error if we failed to delete a node
	if failed > 0 {
		return fmt.Errorf("Failed to delete %d nodes: Try to delete them manually", failed)
	}
	return nil
}

// ClusterList returns a list of all existing clusters
func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, error) {
	l.Log().Traceln("Listing Clusters...")
	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels)
	if err != nil {
		return nil, fmt.Errorf("runtime failed to list nodes: %w", err)
	}

	l.Log().Debugf("Found %d nodes", len(nodes))
	if l.Log().GetLevel() == logrus.TraceLevel {
		for _, node := range nodes {
			l.Log().Tracef("Found node %s of role %s", node.Name, node.Role)
		}
	}

	nodes = NodeFilterByRoles(nodes, k3d.ClusterInternalNodeRoles, k3d.ClusterExternalNodeRoles)

	l.Log().Tracef("Found %d cluster-internal nodes", len(nodes))
	if l.Log().GetLevel() == logrus.TraceLevel {
		for _, node := range nodes {
			l.Log().Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.RuntimeLabels[k3d.LabelClusterName])
		}
	}

	clusters := []*k3d.Cluster{}
	// for each node, check, if we can add it to a cluster or add the cluster if it doesn't exist yet
	for _, node := range nodes {
		clusterExists := false
		for _, cluster := range clusters {
			if node.RuntimeLabels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist
				cluster.Nodes = append(cluster.Nodes, node)
				clusterExists = true
				break
			}
		}
		// cluster is not in the list yet, so we add it with the current node as its first member
		if !clusterExists {
			clusters = append(clusters, &k3d.Cluster{
				Name:  node.RuntimeLabels[k3d.LabelClusterName],
				Nodes: []*k3d.Node{node},
			})
		}
	}

	// enrich cluster structs with label values
	for _, cluster := range clusters {
		if err := populateClusterFieldsFromLabels(cluster); err != nil {
			l.Log().Warnf("Failed to populate cluster fields from node label values for cluster '%s'", cluster.Name)
			l.Log().Warnln(err)
		}
	}
	l.Log().Debugf("Found %d clusters", len(clusters))
	return clusters, nil
}

// populateClusterFieldsFromLabels inspects labels attached to nodes and translates them to struct fields
func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
	networkExternalSet := false

	for _, node := range cluster.Nodes {
		// get the name of the cluster network
		if cluster.Network.Name == "" {
			if networkName, ok := node.RuntimeLabels[k3d.LabelNetwork]; ok {
				cluster.Network.Name = networkName
			}
		}

		// check if the network is external
		// since the struct value is a bool, initialized as false, we cannot check if it's unset
		if !cluster.Network.External && !networkExternalSet {
			if networkExternalString, ok := node.RuntimeLabels[k3d.LabelNetworkExternal]; ok {
				if networkExternal, err := strconv.ParseBool(networkExternalString); err == nil {
					cluster.Network.External = networkExternal
					networkExternalSet = true
				}
			}
		}

		// get image volume // TODO: enable external image volumes the same way we do it with networks
		if cluster.ImageVolume == "" {
			if imageVolumeName, ok := node.RuntimeLabels[k3d.LabelImageVolume]; ok {
				cluster.ImageVolume = imageVolumeName
			}
		}

		// get k3s cluster's token
		if cluster.Token == "" {
			if token, ok := node.RuntimeLabels[k3d.LabelClusterToken]; ok {
				cluster.Token = token
			}
		}
	}

	return nil
}

func GetClusterStartOptsFromLabels(cluster *k3d.Cluster) (k3d.ClusterStartOpts, error) {
	clusterStartOpts := k3d.ClusterStartOpts{
		HostAliases: []k3d.HostAlias{},
	}
	for _, node := range cluster.Nodes {
		if len(clusterStartOpts.HostAliases) == 0 {
			if hostAliasesJSON, ok := node.RuntimeLabels[k3d.LabelClusterStartHostAliases]; ok {
				if err := json.Unmarshal([]byte(hostAliasesJSON), &clusterStartOpts.HostAliases); err != nil {
					return clusterStartOpts, fmt.Errorf("error unmarshalling hostaliases JSON from node %s label: %w", node.Name, err)
				}
			}
		}
	}
	return clusterStartOpts, nil
}

var ClusterGetNoNodesFoundError = errors.New("No nodes found for given cluster")

// ClusterGet returns an existing cluster with all fields and node lists populated
func ClusterGet(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) (*k3d.Cluster, error) {
	// get nodes that belong to the selected cluster
	nodes, err := runtime.GetNodesByLabel(ctx, map[string]string{k3d.LabelClusterName: cluster.Name})
	if err != nil {
		l.Log().Errorf("Failed to get nodes for cluster '%s': %v", cluster.Name, err)
	}

	if len(nodes) == 0 {
		return nil, ClusterGetNoNodesFoundError
	}

	// append nodes
	for _, node := range nodes {
		// check if there's already a node in the struct
		overwroteExisting := false
		for _, existingNode := range cluster.Nodes {
			// overwrite existing node
			if existingNode.Name == node.Name {
				if err := mergo.Merge(existingNode, node, mergo.WithOverride); err != nil {
					return nil, fmt.Errorf("failed to merge node %s into cluster: %v", node.Name, err)
				}
				overwroteExisting = true
			}
		}

		// no existing node overwritten: append new node
		if !overwroteExisting {
			cluster.Nodes = append(cluster.Nodes, node)
		}
	}

	// Loadbalancer
	if cluster.ServerLoadBalancer == nil {
		for _, node := range cluster.Nodes {
			if node.Role == k3d.LoadBalancerRole {
				cluster.ServerLoadBalancer = &k3d.Loadbalancer{
					Node: node,
				}
			}
		}

		if cluster.ServerLoadBalancer != nil && cluster.ServerLoadBalancer.Node != nil {
			lbcfg, err := GetLoadbalancerConfig(ctx, runtime, cluster)
			if err != nil {
				l.Log().Errorf("error getting loadbalancer config from %s: %v", cluster.ServerLoadBalancer.Node.Name, err)
			}
			cluster.ServerLoadBalancer.Config = &lbcfg
		}
	}

	vols, err := runtime.GetVolumesByLabel(ctx, map[string]string{k3d.LabelClusterName: cluster.Name})
	if err != nil {
		return nil, err
	}
	for _, vol := range vols {
		if !slices.Contains(cluster.Volumes, vol) {
			cluster.Volumes = append(cluster.Volumes, vol)
		}
	}

	if err := populateClusterFieldsFromLabels(cluster); err != nil {
		l.Log().Warnf("Failed to populate cluster fields from node labels: %v", err)
	}

	return cluster, nil
}

// GenerateClusterToken generates a random 20 character string
func GenerateClusterToken() string {
	return util.GenerateRandomString(20)
}

func GenerateNodeName(cluster string, role k3d.Role, suffix int) string {
	return fmt.Sprintf("%s-%s-%s-%d", k3d.DefaultObjectNamePrefix, cluster, role, suffix)
}

// ClusterStart starts a whole cluster (i.e. all nodes of the cluster)
func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts k3d.ClusterStartOpts) error {
	l.Log().Infof("Starting cluster '%s'", cluster.Name)

	if clusterStartOpts.Intent == "" {
		clusterStartOpts.Intent = k3d.IntentClusterStart
	}

	if clusterStartOpts.Timeout > 0*time.Second {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, clusterStartOpts.Timeout)
		defer cancel()
	}

	// sort the nodes into categories
	var initNode *k3d.Node
	var servers []*k3d.Node
	var agents []*k3d.Node
	var aux []*k3d.Node
	for _, n := range cluster.Nodes {
		if !n.State.Running {
			if n.Role == k3d.ServerRole {
				if n.ServerOpts.IsInit {
					initNode = n
					continue
				}
				servers = append(servers, n)
			} else if n.Role == k3d.AgentRole {
				agents = append(agents, n)
			} else {
				aux = append(aux, n)
			}
		} else {
			l.Log().Tracef("Node %s already running.", n.Name)
		}
	}

	// sort list of servers for properly ordered sequential start
	sort.Slice(servers, func(i, j int) bool {
		return servers[i].Name < servers[j].Name
	})

	/*
	 * Init Node
	 */
	if initNode != nil {
		l.Log().Infoln("Starting the initializing server...")
		if err := NodeStart(ctx, runtime, initNode, &k3d.NodeStartOpts{
			Wait:            true, // always wait for the init node
			NodeHooks:       clusterStartOpts.NodeHooks,
			ReadyLogMessage: k3d.GetReadyLogMessage(initNode, clusterStartOpts.Intent), // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
			EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
		}); err != nil {
			return fmt.Errorf("Failed to start initializing server node: %+v", err)
		}
	}

	/*
	 * Server Nodes
	 */
	if len(servers) > 0 {
		l.Log().Infoln("Starting servers...")
		for _, serverNode := range servers {
			if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{
				Wait:            true,
				NodeHooks:       append(clusterStartOpts.NodeHooks, serverNode.HookActions...),
				EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
			}); err != nil {
				return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err)
			}
		}
	} else {
		l.Log().Infoln("All servers already running.")
	}

	/*
	 * Agent Nodes
	 */
	if len(agents) > 0 {
		agentWG, aCtx := errgroup.WithContext(ctx)

		l.Log().Infoln("Starting agents...")
		for _, agentNode := range agents {
			currentAgentNode := agentNode
			agentWG.Go(func() error {
				return NodeStart(aCtx, runtime, currentAgentNode, &k3d.NodeStartOpts{
					Wait:            true,
					NodeHooks:       append(clusterStartOpts.NodeHooks, agentNode.HookActions...),
					EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
				})
			})
		}
		if err := agentWG.Wait(); err != nil {
			return fmt.Errorf("Failed to add one or more agents: %w", err)
		}
	} else {
		l.Log().Infoln("All agents already running.")
	}

	/*
	 * Auxiliary/Helper Nodes
	 */

	if len(aux) > 0 {
		helperWG, hCtx := errgroup.WithContext(ctx)
		l.Log().Infoln("Starting helpers...")
		for _, helperNode := range aux {
			currentHelperNode := helperNode

			helperWG.Go(func() error {
				nodeStartOpts := &k3d.NodeStartOpts{
					NodeHooks:       currentHelperNode.HookActions,
					EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
				}
				if currentHelperNode.Role == k3d.LoadBalancerRole {
					nodeStartOpts.Wait = true
				}

				return NodeStart(hCtx, runtime, currentHelperNode, nodeStartOpts)
			})
		}

		if err := helperWG.Wait(); err != nil {
			return fmt.Errorf("Failed to add one or more helper nodes: %w", err)
		}
	} else {
		l.Log().Infoln("All helpers already running.")
	}

	/*
	 * Additional Cluster Preparation (post start)
	 */

	if len(servers) > 0 || len(agents) > 0 { // TODO: make checks for required cluster start actions cleaner
		postStartErrgrp, postStartErrgrpCtx := errgroup.WithContext(ctx)

		/*** DNS ***/

		// -> skip if hostnetwork mode
		if cluster.Network.Name == "host" {
			l.Log().Debugf("Not injecting hostAliases into /etc/hosts and CoreDNS as clusternetwork is 'host'")
		} else {
			// -> add hostAliases to /etc/hosts in all nodes
			// --> inject host-gateway as host.k3d.internal
			clusterStartOpts.HostAliases = append(clusterStartOpts.HostAliases, k3d.HostAlias{
				IP:        clusterStartOpts.EnvironmentInfo.HostGateway.String(),
				Hostnames: []string{"host.k3d.internal"},
			})

			for _, node := range append(servers, agents...) {
				currNode := node
				postStartErrgrp.Go(func() error {
					return NewHostAliasesInjectEtcHostsAction(runtime, clusterStartOpts.HostAliases).Run(postStartErrgrpCtx, currNode)
				})
			}

			// -> inject hostAliases and network members into CoreDNS configmap
			if len(servers) > 0 {
				postStartErrgrp.Go(func() error {
					hosts := ""

					// hosts: hostAliases (including host.k3d.internal)
					for _, hostAlias := range clusterStartOpts.HostAliases {
						hosts += fmt.Sprintf("%s %s\n", hostAlias.IP, strings.Join(hostAlias.Hostnames, " "))
					}

					// more hosts: network members ("neighbor" containers)
					net, err := runtime.GetNetwork(postStartErrgrpCtx, &cluster.Network)
					if err != nil {
						return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err)
					}
					for _, member := range net.Members {
						hosts += fmt.Sprintf("%s %s\n", member.IP.String(), member.Name)
					}

					// inject CoreDNS configmap
					l.Log().Infof("Injecting records for hostAliases (incl. host.k3d.internal) and for %d network members into CoreDNS configmap...", len(net.Members))
					act := actions.RewriteFileAction{
						Runtime: runtime,
						Path:    "/var/lib/rancher/k3s/server/manifests/coredns.yaml",
						Mode:    0744,
						RewriteFunc: func(input []byte) ([]byte, error) {
							split, err := util.SplitYAML(input)
							if err != nil {
								return nil, fmt.Errorf("error splitting yaml: %w", err)
							}

							var outputBuf bytes.Buffer
							outputEncoder := util.NewYAMLEncoder(&outputBuf)

							for _, d := range split {
								var doc map[string]interface{}
								if err := yaml.Unmarshal(d, &doc); err != nil {
									return nil, err
								}
								if kind, ok := doc["kind"]; ok {
									if strings.ToLower(kind.(string)) == "configmap" {
										configmapData, ok := doc["data"].(map[string]interface{})
										if !ok {
											return nil, fmt.Errorf("invalid ConfigMap data type: %T", doc["data"])
										}
										configmapData["NodeHosts"] = hosts
									}
								}
								if err := outputEncoder.Encode(doc); err != nil {
									return nil, err
								}
							}
							_ = outputEncoder.Close()
							return outputBuf.Bytes(), nil
						},
					}

					// get the first server in the list and run action on it once it's ready for it
					for _, n := range servers {
						// do not try to run the action, if CoreDNS is disabled on K3s level
						for _, flag := range n.Args {
							if strings.HasPrefix(flag, "--disable") && strings.Contains(flag, "coredns") {
								l.Log().Debugf("CoreDNS disabled in K3s via flag `%s`. Not trying to use it.", flag)
								return nil
							}
						}
						ts, err := time.Parse("2006-01-02T15:04:05.999999999Z", n.State.Started)
						if err != nil {
							return err
						}
						if err := NodeWaitForLogMessage(postStartErrgrpCtx, runtime, n, "Cluster dns configmap", ts.Truncate(time.Second)); err != nil {
							return err
						}
						return act.Run(postStartErrgrpCtx, n) // nolint:staticcheck // FIXME: Does this loop really only concern the first server? (SA4004: the surrounding loop is unconditionally terminated (staticcheck))
					}
					return nil
				})
			}
		}

		if err := postStartErrgrp.Wait(); err != nil {
			return fmt.Errorf("error during post-start cluster preparation: %w", err)
		}
	}

	return nil
}

// ClusterStop stops a whole cluster (i.e. all nodes of the cluster)
func ClusterStop(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
	l.Log().Infof("Stopping cluster '%s'", cluster.Name)

	failed := 0
	for _, node := range cluster.Nodes {
		if err := runtime.StopNode(ctx, node); err != nil {
			l.Log().Warningf("Failed to stop node '%s': Try to stop it manually", node.Name)
			failed++
			continue
		}
	}

	if failed > 0 {
		return fmt.Errorf("Failed to stop %d nodes: Try to stop them manually", failed)
	}

	l.Log().Infof("Stopped cluster '%s'", cluster.Name)
	return nil
}

// SortClusters : in place sort cluster list by cluster name alphabetical order
func SortClusters(clusters []*k3d.Cluster) []*k3d.Cluster {
	sort.Slice(clusters, func(i, j int) bool {
		return clusters[i].Name < clusters[j].Name
	})
	return clusters
}

func prepCreateLocalRegistryHostingConfigMap(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
	success := false
	for _, node := range cluster.Nodes {
		if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole {
			err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("kubectl apply -f %s", k3d.DefaultLocalRegistryHostingConfigmapTempPath)})
			if err == nil {
				success = true
				break
			} else {
				l.Log().Debugf("Failed to create LocalRegistryHosting ConfigMap in node %s: %+v", node.Name, err)
			}
		}
	}
	if !success {
		l.Log().Warnf("Failed to create LocalRegistryHosting ConfigMap")
	}
	return nil
}

// ClusterEditChangesetSimple modifies an existing cluster with a given SimpleConfig changeset
func ClusterEditChangesetSimple(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, changeset *config.SimpleConfig) error {
	// nodeCount := len(cluster.Nodes)
	nodeList := cluster.Nodes

	// === Ports ===

	existingLB := cluster.ServerLoadBalancer
	lbChangeset := &k3d.Loadbalancer{}

	// copy existing loadbalancer
	lbChangesetNode, err := CopyNode(ctx, existingLB.Node, CopyNodeOpts{keepState: false})
	if err != nil {
		return fmt.Errorf("error copying existing loadbalancer: %w", err)
	}

	lbChangeset.Node = lbChangesetNode

	// copy config from existing loadbalancer
	lbChangesetConfig, err := copystruct.Copy(existingLB.Config)
	if err != nil {
		return fmt.Errorf("error copying config from existing loadbalancer: %w", err)
	}

	lbChangeset.Config = lbChangesetConfig.(*k3d.LoadbalancerConfig)

	// loop over ports
	if len(changeset.Ports) > 0 {
		// 1. ensure that there are only supported suffices in the node filters // TODO: overly complex right now, needs simplification
		for _, portWithNodeFilters := range changeset.Ports {
			filteredNodes, err := util.FilterNodesWithSuffix(nodeList, portWithNodeFilters.NodeFilters)
			if err != nil {
				return fmt.Errorf("failed to filter nodes: %w", err)
			}

			for suffix := range filteredNodes {
				switch suffix {
				case "proxy", util.NodeFilterSuffixNone, util.NodeFilterMapKeyAll:
					continue
				default:
					return fmt.Errorf("error: 'cluster edit' does not (yet) support the '%s' opt/suffix for adding ports", suffix)
				}
			}
		}

		// 2. transform
		cluster.ServerLoadBalancer = lbChangeset // we're working with pointers, so let's point to the changeset here to not update the original that we keep as a reference
		if err := TransformPorts(ctx, runtime, cluster, changeset.Ports); err != nil {
			return fmt.Errorf("error transforming port config %s: %w", changeset.Ports, err)
		}
	}

	l.Log().Debugf("ORIGINAL:\n> Ports: %+v\n> Config: %+v\nCHANGESET:\n> Ports: %+v\n> Config: %+v", existingLB.Node.Ports, existingLB.Config, lbChangeset.Node.Ports, lbChangeset.Config)

	// prepare to write config to lb container
	configyaml, err := yaml.Marshal(lbChangeset.Config)
	if err != nil {
		return fmt.Errorf("failed to marshal loadbalancer config changeset: %w", err)
	}
	writeLbConfigAction := k3d.NodeHook{
		Stage: k3d.LifecycleStagePreStart,
		Action: actions.WriteFileAction{
			Runtime:     runtime,
			Dest:        k3d.DefaultLoadbalancerConfigPath,
			Mode:        0744,
			Content:     configyaml,
			Description: "Write Loadbalancer Configuration",
		},
	}
	if lbChangeset.Node.HookActions == nil {
		lbChangeset.Node.HookActions = []k3d.NodeHook{}
	}
	lbChangeset.Node.HookActions = append(lbChangeset.Node.HookActions, writeLbConfigAction)

	if err := NodeReplace(ctx, runtime, existingLB.Node, lbChangeset.Node); err != nil {
		return fmt.Errorf("error replacing loadbalancer node: %w", err)
	}

	return nil
}