talosconfig_controller.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package controllers

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	jsonpatch "github.com/evanphx/json-patch"
	"github.com/go-logr/logr"
	"github.com/talos-systems/talos/pkg/machinery/config"
	"github.com/talos-systems/talos/pkg/machinery/config/configpatcher"
	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/generate"
	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
	"github.com/talos-systems/talos/pkg/machinery/constants"
	"github.com/talos-systems/talos/pkg/machinery/role"
	"gopkg.in/yaml.v2"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	capiv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
	bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
	expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
	"sigs.k8s.io/cluster-api/feature"
	"sigs.k8s.io/cluster-api/util"
	"sigs.k8s.io/cluster-api/util/annotations"
	"sigs.k8s.io/cluster-api/util/conditions"
	"sigs.k8s.io/cluster-api/util/patch"
	"sigs.k8s.io/cluster-api/util/predicates"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/source"

	bootstrapv1alpha3 "github.com/talos-systems/cluster-api-bootstrap-provider-talos/api/v1alpha3"
	// +kubebuilder:scaffold:imports
)

const (
	controllerName = "cabpt-controller"
)

var (
	defaultVersionContract = config.TalosVersionCurrent
)

// TalosConfigReconciler reconciles a TalosConfig object
type TalosConfigReconciler struct {
	client.Client
	Log              logr.Logger
	Scheme           *runtime.Scheme
	WatchFilterValue string
}

type TalosConfigScope struct {
	Config      *bootstrapv1alpha3.TalosConfig
	ConfigOwner *bsutil.ConfigOwner
	Cluster     *capiv1.Cluster
}

type TalosConfigBundle struct {
	BootstrapData string
	TalosConfig   string
}

func (r *TalosConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
	r.Scheme = mgr.GetScheme()

	b := ctrl.NewControllerManagedBy(mgr).
		For(&bootstrapv1alpha3.TalosConfig{}).
		WithOptions(options).
		WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)).
		Watches(
			&source.Kind{Type: &capiv1.Machine{}},
			handler.EnqueueRequestsFromMapFunc(r.MachineToBootstrapMapFunc),
		)

	if feature.Gates.Enabled(feature.MachinePool) {
		b = b.Watches(
			&source.Kind{Type: &expv1.MachinePool{}},
			handler.EnqueueRequestsFromMapFunc(r.MachinePoolToBootstrapMapFunc),
		).WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue))

	}

	c, err := b.Build(r)
	if err != nil {
		return err
	}

	err = c.Watch(
		&source.Kind{Type: &capiv1.Cluster{}},
		handler.EnqueueRequestsFromMapFunc(r.ClusterToTalosConfigs),
		predicates.All(ctrl.LoggerFrom(ctx),
			predicates.ClusterUnpausedAndInfrastructureReady(ctrl.LoggerFrom(ctx)),
			predicates.ResourceHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue),
		),
	)
	if err != nil {
		return err
	}

	return nil
}

// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=talosconfigs,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=talosconfigs/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machines;machines/status,verbs=get;list;watch
// +kubebuilder:rbac:groups=exp.cluster.x-k8s.io,resources=machinepools;machinepools/status,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete

func (r *TalosConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, rerr error) {
	log := r.Log.WithName(controllerName).
		WithName(fmt.Sprintf("namespace=%s", req.Namespace)).
		WithName(fmt.Sprintf("talosconfig=%s", req.Name))

	// Lookup the talosconfig config
	config := &bootstrapv1alpha3.TalosConfig{}
	if err := r.Client.Get(ctx, req.NamespacedName, config); err != nil {
		if apierrors.IsNotFound(err) {
			return ctrl.Result{}, nil
		}
		log.Error(err, "failed to get config")
		return ctrl.Result{}, err
	}

	// Initialize the patch helper
	patchHelper, err := patch.NewHelper(config, r.Client)
	if err != nil {
		return ctrl.Result{}, err
	}

	// Always attempt to Patch the TalosConfig object and status after each reconciliation.
	defer func() {
		// always update the readyCondition; the summary is represented using the "1 of x completed" notation.
		conditions.SetSummary(config,
			conditions.WithConditions(
				bootstrapv1alpha3.DataSecretAvailableCondition,
			),
		)

		patchOpts := []patch.Option{
			patch.WithOwnedConditions{
				Conditions: []capiv1.ConditionType{
					bootstrapv1alpha3.DataSecretAvailableCondition,
				},
			},
		}

		// Patch ObservedGeneration only if the reconciliation completed successfully
		if rerr == nil {
			patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{})
		}

		if err := patchHelper.Patch(ctx, config, patchOpts...); err != nil {
			log.Error(err, "failed to patch config")
			if rerr == nil {
				rerr = err
			}
		}
	}()

	// Handle deleted talosconfigs
	// We no longer set finalizers on talosconfigs, but we have to remove previously set finalizers
	if !config.ObjectMeta.DeletionTimestamp.IsZero() {
		return r.reconcileDelete(ctx, config)
	}

	// Look up the resource that owns this talosconfig if there is one
	owner, err := bsutil.GetConfigOwner(ctx, r.Client, config)
	if err != nil {
		log.Error(err, "could not get owner resource")
		return ctrl.Result{}, err
	}

	if owner == nil {
		log.Info("Waiting for OwnerRef on the talosconfig")
		return ctrl.Result{}, nil
	}

	log = log.WithName(fmt.Sprintf("owner-name=%s", owner.GetName()))

	// Lookup the cluster the machine is associated with
	cluster, err := util.GetClusterByName(ctx, r.Client, owner.GetNamespace(), owner.ClusterName())
	if err != nil {
		if errors.Is(err, util.ErrNoCluster) {
			log.Info(fmt.Sprintf("%s does not belong to a cluster yet, waiting until it's part of a cluster", owner.GetKind()))
			return ctrl.Result{}, nil
		}

		if apierrors.IsNotFound(err) {
			log.Info("Cluster does not exist yet, waiting until it is created")
			return ctrl.Result{}, nil
		}

		log.Error(err, "could not get cluster by machine metadata")

		return ctrl.Result{}, err
	}

	if annotations.IsPaused(cluster, config) {
		log.Info("Reconciliation is paused for this object")
		return ctrl.Result{}, nil
	}

	tcScope := &TalosConfigScope{
		Config:      config,
		ConfigOwner: owner,
		Cluster:     cluster,
	}

	// bail super early if it's already ready
	if config.Status.Ready {
		log.Info("ignoring an already ready config")
		conditions.MarkTrue(config, bootstrapv1alpha3.DataSecretAvailableCondition)

		// reconcile cluster-wide talosconfig
		err = r.reconcileClientConfig(ctx, log, tcScope)

		if err == nil {
			conditions.MarkTrue(config, bootstrapv1alpha3.ClientConfigAvailableCondition)
		} else {
			conditions.MarkFalse(config, bootstrapv1alpha3.ClientConfigAvailableCondition, bootstrapv1alpha3.ClientConfigGenerationFailedReason, capiv1.ConditionSeverityError, "talosconfig generation failure: %s", err)
		}

		return ctrl.Result{}, err
	}

	// Wait patiently for the infrastructure to be ready
	if !cluster.Status.InfrastructureReady {
		log.Info("Infrastructure is not ready, waiting until ready.")
		conditions.MarkFalse(config, bootstrapv1alpha3.DataSecretAvailableCondition, bootstrapv1alpha3.WaitingForClusterInfrastructureReason, capiv1.ConditionSeverityInfo, "")

		return ctrl.Result{}, nil
	}

	// Reconcile status for machines that already have a secret reference, but our status isn't up to date.
	// This case solves the pivoting scenario (or a backup restore) which doesn't preserve the status subresource on objects.
	if owner.DataSecretName() != nil && (!config.Status.Ready || config.Status.DataSecretName == nil) {
		config.Status.Ready = true
		config.Status.DataSecretName = owner.DataSecretName()
		conditions.MarkTrue(config, bootstrapv1alpha3.DataSecretAvailableCondition)

		return ctrl.Result{}, nil
	}

	if err = r.reconcileGenerate(ctx, tcScope); err != nil {
		conditions.MarkFalse(config, bootstrapv1alpha3.DataSecretAvailableCondition, bootstrapv1alpha3.DataSecretGenerationFailedReason, capiv1.ConditionSeverityError, err.Error())

		return ctrl.Result{}, err
	}

	config.Status.Ready = true
	conditions.MarkTrue(config, bootstrapv1alpha3.DataSecretAvailableCondition)

	return ctrl.Result{}, nil
}

func (r *TalosConfigReconciler) reconcileGenerate(ctx context.Context, tcScope *TalosConfigScope) error {
	var (
		retData *TalosConfigBundle
		err     error
	)

	config := tcScope.Config

	machineType, _ := machine.ParseType(config.Spec.GenerateType) //nolint:errcheck // handle errors later

	switch {
	// Slurp and use user-supplied configs
	case config.Spec.GenerateType == "none":
		if config.Spec.Data == "" {
			return errors.New("failed to specify config data with none generate type")
		}

		retData, err = r.userConfigs(ctx, tcScope)
		if err != nil {
			return err
		}

	// Generate configs on the fly
	case machineType != machine.TypeUnknown:
		retData, err = r.genConfigs(ctx, tcScope)
		if err != nil {
			return err
		}

	default:
		return fmt.Errorf("unknown generate type specified: %q", config.Spec.GenerateType)
	}

	// Handle patches to the machine config if they were specified
	// Note this will patch both pre-generated and user-provided configs.
	if len(config.Spec.ConfigPatches) > 0 {
		marshalledPatches, err := json.Marshal(config.Spec.ConfigPatches)
		if err != nil {
			return fmt.Errorf("failure marshalling config patches: %s", err)
		}

		patch, err := jsonpatch.DecodePatch(marshalledPatches)
		if err != nil {
			return fmt.Errorf("failure decoding config patches from talosconfig to rfc6902 patch: %s", err)
		}

		patchedBytes, err := configpatcher.JSON6902([]byte(retData.BootstrapData), patch)
		if err != nil {
			return err
		}

		retData.BootstrapData = string(patchedBytes)
	}

	// Packet acts a fool if you don't prepend #!talos to the userdata
	// so we try to suss out if that's the type of machine/machinePool getting created.
	if tcScope.ConfigOwner.IsMachinePool() {
		mp := &expv1.MachinePool{}
		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(tcScope.ConfigOwner.Object, mp); err != nil {
			return err
		}

		if mp.Spec.Template.Spec.InfrastructureRef.Kind == "PacketMachinePool" {
			retData.BootstrapData = "#!talos\n" + retData.BootstrapData
		}
	} else {
		machine := &capiv1.Machine{}
		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(tcScope.ConfigOwner.Object, machine); err != nil {
			return err
		}

		if machine.Spec.InfrastructureRef.Name == "PacketMachine" {
			retData.BootstrapData = "#!talos\n" + retData.BootstrapData
		}
	}

	var dataSecretName string

	dataSecretName, err = r.writeBootstrapData(ctx, tcScope, []byte(retData.BootstrapData))
	if err != nil {
		return err
	}

	config.Status.DataSecretName = &dataSecretName
	config.Status.TalosConfig = retData.TalosConfig //nolint:staticcheck // deprecated, for backwards compatibility only

	return nil
}

func (r *TalosConfigReconciler) reconcileDelete(ctx context.Context, config *bootstrapv1alpha3.TalosConfig) (ctrl.Result, error) {
	controllerutil.RemoveFinalizer(config, bootstrapv1alpha3.ConfigFinalizer)

	return ctrl.Result{}, nil
}

func genTalosConfigFile(clusterName string, bundle *generate.SecretsBundle, endpoints []string) (string, error) {
	in := &generate.Input{
		ClusterName: clusterName,
		Certs: &generate.Certs{
			OS: bundle.Certs.OS,
		},
	}

	var err error

	in.Certs.Admin, err = generate.NewAdminCertificateAndKey(
		bundle.Clock.Now(),
		bundle.Certs.OS,
		role.MakeSet(role.Admin),
		87600*time.Hour,
	)
	if err != nil {
		return "", err
	}

	talosConfig, err := generate.Talosconfig(in, generate.WithEndpointList(endpoints))
	if err != nil {
		return "", err
	}

	talosConfigBytes, err := yaml.Marshal(talosConfig)
	if err != nil {
		return "", err
	}

	return string(talosConfigBytes), nil
}

// userConfigs will fetch and make use of user-supplied bootstrap configs to return
func (r *TalosConfigReconciler) userConfigs(ctx context.Context, scope *TalosConfigScope) (*TalosConfigBundle, error) {
	retBundle := &TalosConfigBundle{}

	userConfig := &v1alpha1.Config{}
	err := yaml.Unmarshal([]byte(scope.Config.Spec.Data), userConfig)
	if err != nil {
		return retBundle, err
	}

	// Create the secret with kubernetes certs so a kubeconfig can be generated
	// but do this only when machineconfig contains full Kubernetes CA secret (controlplane nodes)
	if userConfig.Cluster().CA() != nil && len(userConfig.Cluster().CA().Crt) > 0 && len(userConfig.Cluster().CA().Key) > 0 {
		if err = r.writeK8sCASecret(ctx, scope, userConfig.Cluster().CA()); err != nil {
			return retBundle, err
		}
	}

	userConfigStr, err := userConfig.String()
	if err != nil {
		return retBundle, err
	}

	retBundle.BootstrapData = userConfigStr

	if userConfig.Machine().Security().CA() != nil && len(userConfig.Machine().Security().CA().Crt) > 0 && len(userConfig.Machine().Security().CA().Key) > 0 {
		bundle := generate.NewSecretsBundleFromConfig(generate.NewClock(), userConfig)

		retBundle.TalosConfig, err = genTalosConfigFile(userConfig.Cluster().Name(), bundle, nil)
		if err != nil {
			r.Log.Error(err, "failed generating talosconfig for user-supplied machine configuration")
		}
	}

	return retBundle, nil
}

// genConfigs will generate a bootstrap config and a talosconfig to return
func (r *TalosConfigReconciler) genConfigs(ctx context.Context, scope *TalosConfigScope) (*TalosConfigBundle, error) {
	retBundle := &TalosConfigBundle{}

	// Determine what type of node this is
	machineType, err := machine.ParseType(scope.Config.Spec.GenerateType)
	if err != nil {
		machineType = machine.TypeWorker
	}

	// Allow user to override default kube version.
	// This also handles version being formatted like "vX.Y.Z" instead of without leading 'v'
	// TrimPrefix returns the string unchanged if the prefix isn't present.
	k8sVersion := constants.DefaultKubernetesVersion
	if scope.ConfigOwner.IsMachinePool() {
		mp := &expv1.MachinePool{}
		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, mp); err != nil {
			return retBundle, err
		}
		if mp.Spec.Template.Spec.Version != nil {
			k8sVersion = strings.TrimPrefix(*mp.Spec.Template.Spec.Version, "v")
		}
	} else {
		machine := &capiv1.Machine{}
		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, machine); err != nil {
			return retBundle, err
		}
		if machine.Spec.Version != nil {
			k8sVersion = strings.TrimPrefix(*machine.Spec.Version, "v")
		}
	}

	clusterDNS := constants.DefaultDNSDomain
	if scope.Cluster.Spec.ClusterNetwork.ServiceDomain != "" {
		clusterDNS = scope.Cluster.Spec.ClusterNetwork.ServiceDomain
	}

	genOptions := []generate.GenOption{generate.WithDNSDomain(clusterDNS)}

	// default version contract to v0.8 unless user specifies something different.
	versionContract := defaultVersionContract

	if scope.Config.Spec.TalosVersion != "" {
		var err error
		versionContract, err = config.ParseContractFromVersion(scope.Config.Spec.TalosVersion)
		if err != nil {
			return retBundle, fmt.Errorf("invalid talos-version: %w", err)
		}
	}

	genOptions = append(genOptions, generate.WithVersionContract(versionContract))

	secretBundle, err := r.getSecretsBundle(ctx, scope, true, genOptions...)
	if err != nil {
		return retBundle, err
	}

	APIEndpointPort := strconv.Itoa(int(scope.Cluster.Spec.ControlPlaneEndpoint.Port))

	input, err := generate.NewInput(
		scope.Cluster.Name,
		"https://"+scope.Cluster.Spec.ControlPlaneEndpoint.Host+":"+APIEndpointPort,
		k8sVersion,
		secretBundle,
		genOptions...,
	)
	if err != nil {
		return retBundle, err
	}

	// Create the secret with kubernetes certs so a kubeconfig can be generated
	if err = r.writeK8sCASecret(ctx, scope, input.Certs.K8s); err != nil {
		return retBundle, err
	}

	tcString, err := genTalosConfigFile(input.ClusterName, secretBundle, nil)
	if err != nil {
		return retBundle, err
	}

	retBundle.TalosConfig = tcString

	data, err := generate.Config(machineType, input)
	if err != nil {
		return retBundle, err
	}

	if scope.Cluster.Spec.ClusterNetwork.Pods != nil {
		data.ClusterConfig.ClusterNetwork.PodSubnet = scope.Cluster.Spec.ClusterNetwork.Pods.CIDRBlocks
	}
	if scope.Cluster.Spec.ClusterNetwork.Services != nil {
		data.ClusterConfig.ClusterNetwork.ServiceSubnet = scope.Cluster.Spec.ClusterNetwork.Services.CIDRBlocks
	}

	dataOut, err := data.String()
	if err != nil {
		return retBundle, err
	}

	retBundle.BootstrapData = dataOut

	return retBundle, nil
}

// MachineToBootstrapMapFunc is a handler.ToRequestsFunc to be used to enqueue
// request for reconciliation of TalosConfig.
func (r *TalosConfigReconciler) MachineToBootstrapMapFunc(o client.Object) []ctrl.Request {
	m, ok := o.(*capiv1.Machine)
	if !ok {
		panic(fmt.Sprintf("Expected a Machine but got a %T", o))
	}

	result := []ctrl.Request{}
	if m.Spec.Bootstrap.ConfigRef != nil && m.Spec.Bootstrap.ConfigRef.GroupVersionKind() == bootstrapv1alpha3.GroupVersion.WithKind("TalosConfig") {
		name := client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.Bootstrap.ConfigRef.Name}
		result = append(result, ctrl.Request{NamespacedName: name})
	}
	return result
}

// MachinePoolToBootstrapMapFunc is a handler.ToRequestsFunc to be used to enqueue
// request for reconciliation of TalosConfig.
func (r *TalosConfigReconciler) MachinePoolToBootstrapMapFunc(o client.Object) []ctrl.Request {
	m, ok := o.(*expv1.MachinePool)
	if !ok {
		panic(fmt.Sprintf("Expected a MachinePool but got a %T", o))
	}

	result := []ctrl.Request{}
	configRef := m.Spec.Template.Spec.Bootstrap.ConfigRef
	if configRef != nil && configRef.GroupVersionKind().GroupKind() == bootstrapv1alpha3.GroupVersion.WithKind("TalosConfig").GroupKind() {
		name := client.ObjectKey{Namespace: m.Namespace, Name: configRef.Name}
		result = append(result, ctrl.Request{NamespacedName: name})
	}
	return result
}

// ClusterToTalosConfigs is a handler.ToRequestsFunc to be used to enqeue
// requests for reconciliation of TalosConfigs.
func (r *TalosConfigReconciler) ClusterToTalosConfigs(o client.Object) []ctrl.Request {
	result := []ctrl.Request{}

	c, ok := o.(*capiv1.Cluster)
	if !ok {
		panic(fmt.Sprintf("Expected a Cluster but got a %T", o))
	}

	selectors := []client.ListOption{
		client.InNamespace(c.Namespace),
		client.MatchingLabels{
			capiv1.ClusterLabelName: c.Name,
		},
	}

	machineList := &capiv1.MachineList{}
	if err := r.Client.List(context.TODO(), machineList, selectors...); err != nil {
		return nil
	}

	for _, m := range machineList.Items {
		if m.Spec.Bootstrap.ConfigRef != nil &&
			m.Spec.Bootstrap.ConfigRef.GroupVersionKind().GroupKind() == bootstrapv1alpha3.GroupVersion.WithKind("TalosConfig").GroupKind() {
			name := client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.Bootstrap.ConfigRef.Name}
			result = append(result, ctrl.Request{NamespacedName: name})
		}
	}

	if feature.Gates.Enabled(feature.MachinePool) {
		machinePoolList := &expv1.MachinePoolList{}
		if err := r.Client.List(context.TODO(), machinePoolList, selectors...); err != nil {
			return nil
		}

		for _, mp := range machinePoolList.Items {
			if mp.Spec.Template.Spec.Bootstrap.ConfigRef != nil &&
				mp.Spec.Template.Spec.Bootstrap.ConfigRef.GroupVersionKind().GroupKind() == bootstrapv1alpha3.GroupVersion.WithKind("TalosConfig").GroupKind() {
				name := client.ObjectKey{Namespace: mp.Namespace, Name: mp.Spec.Template.Spec.Bootstrap.ConfigRef.Name}
				result = append(result, ctrl.Request{NamespacedName: name})
			}
		}
	}

	return result
}