Virtual kubelet: allow to disable API server support for offloaded pods

cmd/virtual-kubelet/root/flag.go

@@ -47,7 +47,7 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
 	flags.UintVar(&o.IngressWorkers, "ingress-reflection-workers", o.IngressWorkers, "The number of ingress reflection workers")
 	flags.UintVar(&o.ConfigMapWorkers, "configmap-reflection-workers", o.ConfigMapWorkers, "The number of configmap reflection workers")
 	flags.UintVar(&o.SecretWorkers, "secret-reflection-workers", o.SecretWorkers, "The number of secret reflection workers")
-	flags.UintVar(&o.PersistenVolumeClaimWorkers, "persistentvolumeclaim-reflection-workers", o.PersistenVolumeClaimWorkers,
+	flags.UintVar(&o.PersistentVolumeClaimWorkers, "persistentvolumeclaim-reflection-workers", o.PersistentVolumeClaimWorkers,
 		"The number of persistentvolumeclaim reflection workers")
 
 	flags.DurationVar(&o.NodeLeaseDuration, "node-lease-duration", o.NodeLeaseDuration, "The duration of the node leases")
@@ -59,6 +59,8 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
 	flags.Var(&o.NodeExtraAnnotations, "node-extra-annotations", "Extra annotations to add to the Virtual Node")
 	flags.Var(&o.NodeExtraLabels, "node-extra-labels", "Extra labels to add to the Virtual Node")
 
+	flags.BoolVar(&o.EnableAPIServerSupport, "enable-apiserver-support", false,
+		"Enable offloaded pods to interact back with the local Kubernetes API server")
 	flags.BoolVar(&o.EnableStorage, "enable-storage", false, "Enable the Liqo storage reflection")
 	flags.StringVar(&o.VirtualStorageClassName, "virtual-storage-class-name", "liqo", "Name of the virtual storage class")
 	flags.StringVar(&o.RemoteRealStorageClassName, "remote-real-storage-class-name", "", "Name of the real storage class to use for the actual volumes")

cmd/virtual-kubelet/root/opts.go

@@ -65,13 +65,13 @@ type Opts struct {
 	EnableProfiling bool
 
 	// Number of workers to use to handle pod notifications and resource reflection
-	PodWorkers                  uint
-	ServiceWorkers              uint
-	EndpointSliceWorkers        uint
-	IngressWorkers              uint
-	ConfigMapWorkers            uint
-	SecretWorkers               uint
-	PersistenVolumeClaimWorkers uint
+	PodWorkers                   uint
+	ServiceWorkers               uint
+	EndpointSliceWorkers         uint
+	IngressWorkers               uint
+	ConfigMapWorkers             uint
+	SecretWorkers                uint
+	PersistentVolumeClaimWorkers uint
 
 	NodeLeaseDuration time.Duration
 	NodePingInterval  time.Duration
@@ -80,6 +80,7 @@ type Opts struct {
 	NodeExtraAnnotations argsutils.StringMap
 	NodeExtraLabels      argsutils.StringMap
 
+	EnableAPIServerSupport     bool
 	EnableStorage              bool
 	VirtualStorageClassName    string
 	RemoteRealStorageClassName string
@@ -99,13 +100,13 @@ func NewOpts() *Opts {
 		MetricsAddress:  DefaultMetricsAddress,
 		EnableProfiling: false,
 
-		PodWorkers:                  DefaultPodWorkers,
-		ServiceWorkers:              DefaultServiceWorkers,
-		EndpointSliceWorkers:        DefaultEndpointSliceWorkers,
-		IngressWorkers:              DefaultIngressWorkers,
-		ConfigMapWorkers:            DefaultConfigMapWorkers,
-		SecretWorkers:               DefaultSecretWorkers,
-		PersistenVolumeClaimWorkers: DefaultPersistenVolumeClaimWorkers,
+		PodWorkers:                   DefaultPodWorkers,
+		ServiceWorkers:               DefaultServiceWorkers,
+		EndpointSliceWorkers:         DefaultEndpointSliceWorkers,
+		IngressWorkers:               DefaultIngressWorkers,
+		ConfigMapWorkers:             DefaultConfigMapWorkers,
+		SecretWorkers:                DefaultSecretWorkers,
+		PersistentVolumeClaimWorkers: DefaultPersistenVolumeClaimWorkers,
 
 		NodeLeaseDuration: node.DefaultLeaseDuration * time.Second,
 		NodePingInterval:  node.DefaultPingInterval,

cmd/virtual-kubelet/root/root.go

@@ -103,8 +103,9 @@ func runRootCommand(ctx context.Context, c *Opts) error {
 		IngressWorkers:              c.IngressWorkers,
 		ConfigMapWorkers:            c.ConfigMapWorkers,
 		SecretWorkers:               c.SecretWorkers,
-		PersistenVolumeClaimWorkers: c.PersistenVolumeClaimWorkers,
+		PersistenVolumeClaimWorkers: c.PersistentVolumeClaimWorkers,
 
+		EnableAPIServerSupport:     c.EnableAPIServerSupport,
 		EnableStorage:              c.EnableStorage,
 		VirtualStorageClassName:    c.VirtualStorageClassName,
 		RemoteRealStorageClassName: c.RemoteRealStorageClassName,

deployments/liqo/templates/liqo-controller-manager-deployment.yaml

@@ -2,6 +2,14 @@
 {{- $ctrlManagerConfig := (merge (dict "name" "controller-manager" "module" "controller-manager") .) -}}
 {{- $webhookConfig := (merge (dict "name" "webhook" "module" "webhook") .) -}}
 
+{{- /* Enable the API support only in for Kubernetes versions < 1.24 (due to lack of support for third party tokens), if not overridden by the user */ -}}
+{{- $vkargs := .Values.virtualKubelet.extra.args }}
+{{- if semverCompare "< 1.24.0" .Capabilities.KubeVersion.Version }}
+{{- if not (or (has "--enable-apiserver-support" $vkargs ) (has "--enable-apiserver-support=true" $vkargs ) (has "--enable-apiserver-support=false" $vkargs )) }}
+{{- $vkargs = append $vkargs "--enable-apiserver-support=true" }}
+{{- end }}
+{{- end }}
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -78,8 +86,8 @@ spec:
           {{- $d := dict "commandName" "--kubelet-extra-labels" "dictionary" .Values.virtualKubelet.extra.labels }}
           {{- include "liqo.concatenateMap" $d | nindent 10 }}
           {{- end }}
-          {{- if .Values.virtualKubelet.extra.args }}
-          {{- $d := dict "commandName" "--kubelet-extra-args" "list" .Values.virtualKubelet.extra.args }}
+          {{- if ge (len $vkargs) 1 }}
+          {{- $d := dict "commandName" "--kubelet-extra-args" "list" $vkargs }}
           {{- include "liqo.concatenateList" $d | nindent 10 }}
           {{- end }}
           {{- if .Values.virtualKubelet.virtualNode.extra.annotations }}

docs/usage/reflection.md

@@ -100,4 +100,5 @@ In this respect, Liqo features also the propagation of Secrets holding **Service
 
 ```{warning}
 Currently, Liqo supports only the propagation of *ServiceAccount* tokens contained in the respective *Secret* object (i.e., *first party tokens*), and not of those to be retrieved from the *TokenRequest* API (i.e., *third party tokens*).
+Due to this limitation, service account reflection is currently *disabled* by default in Kubernetes v1.24+, as ServiceAccounts do not longer automatically generate the corresponding Secret.
 ```

pkg/virtualKubelet/forge/events.go

@@ -75,3 +75,8 @@ func EventReflectionDisabledMsg(namespace string) string {
 func EventReflectionDisabledErrorMsg(namespace string, err error) string {
 	return fmt.Sprintf("Reflection to cluster %q disabled for namespace %q: error updating status: %v", RemoteCluster.ClusterName, namespace, err)
 }
+
+// EventSAReflectionDisabledMsg returns the message for the event when service account reflection is disabled.
+func EventSAReflectionDisabledMsg() string {
+	return fmt.Sprintf("Reflection to cluster %q disabled for secrets holding service account tokens", RemoteCluster.ClusterName)
+}

pkg/virtualKubelet/forge/pods.go

@@ -118,7 +118,7 @@ func LocalRejectedPodStatus(local *corev1.PodStatus, phase corev1.PodPhase, reas
 }
 
 // RemoteShadowPod forges the reflected shadowpod, given the local one.
-func RemoteShadowPod(local *corev1.Pod, remote *vkv1alpha1.ShadowPod, targetNamespace string,
+func RemoteShadowPod(local *corev1.Pod, remote *vkv1alpha1.ShadowPod, targetNamespace string, enableAPIServerSupport bool,
 	saSecretRetriever SASecretRetriever, kubernetesServiceIPRetriever KubernetesServiceIPGetter) *vkv1alpha1.ShadowPod {
 	if remote == nil {
 		// The remote is nil if not already created.
@@ -135,26 +135,27 @@ func RemoteShadowPod(local *corev1.Pod, remote *vkv1alpha1.ShadowPod, targetName
 	return &vkv1alpha1.ShadowPod{
 		ObjectMeta: RemoteObjectMeta(FilterLocalPodOffloadedLabel(&local.ObjectMeta), &remote.ObjectMeta),
 		Spec: vkv1alpha1.ShadowPodSpec{
-			Pod: RemotePodSpec(local.Spec.DeepCopy(), remote.Spec.Pod.DeepCopy(), saSecretRetriever, kubernetesServiceIPRetriever),
+			Pod: RemotePodSpec(local.Spec.DeepCopy(), remote.Spec.Pod.DeepCopy(), enableAPIServerSupport,
+				saSecretRetriever, kubernetesServiceIPRetriever),
 		},
 	}
 }
 
 // RemotePodSpec forges the specs of the reflected pod specs, given the local ones.
 // It expects the local and remote objects to be deepcopies, as they are mutated.
-func RemotePodSpec(local, remote *corev1.PodSpec, saSecretRetriever SASecretRetriever,
-	kubernetesServiceIPRetriever KubernetesServiceIPGetter) corev1.PodSpec {
+func RemotePodSpec(local, remote *corev1.PodSpec, enableAPIServerSupport bool,
+	saSecretRetriever SASecretRetriever, kubernetesServiceIPRetriever KubernetesServiceIPGetter) corev1.PodSpec {
 	// The ServiceAccountName field in the pod specifications is optional, and empty means default.
 	if local.ServiceAccountName == "" {
 		local.ServiceAccountName = "default"
 	}
 
-	remote.Containers = RemoteContainers(local.Containers, local.ServiceAccountName)
-	remote.InitContainers = RemoteContainers(local.InitContainers, local.ServiceAccountName)
+	remote.Containers = RemoteContainers(local.Containers, enableAPIServerSupport, local.ServiceAccountName)
+	remote.InitContainers = RemoteContainers(local.InitContainers, enableAPIServerSupport, local.ServiceAccountName)
 
-	remote.HostAliases = RemoteHostAliases(local.HostAliases, kubernetesServiceIPRetriever)
+	remote.HostAliases = RemoteHostAliases(local.HostAliases, enableAPIServerSupport, kubernetesServiceIPRetriever)
 	remote.Tolerations = RemoteTolerations(local.Tolerations)
-	remote.Volumes = RemoteVolumes(local.Volumes, func() string { return saSecretRetriever(local.ServiceAccountName) })
+	remote.Volumes = RemoteVolumes(local.Volumes, enableAPIServerSupport, func() string { return saSecretRetriever(local.ServiceAccountName) })
 
 	remote.ActiveDeadlineSeconds = local.ActiveDeadlineSeconds
 	remote.DNSConfig = local.DNSConfig
@@ -185,7 +186,11 @@ func RemotePodSpec(local, remote *corev1.PodSpec, saSecretRetriever SASecretRetr
 
 // RemoteContainers forges the containers for a reflected pod, appropriately adding the environment variables
 // to enable the offloaded containers to contact back the local API server, instead of the remote one.
-func RemoteContainers(containers []corev1.Container, saName string) []corev1.Container {
+func RemoteContainers(containers []corev1.Container, enableAPIServerSupport bool, saName string) []corev1.Container {
+	if !enableAPIServerSupport {
+		return containers
+	}
+
 	for i := range containers {
 		containers[i].Env = RemoteContainerEnvVariables(containers[i].Env, saName)
 	}
@@ -223,9 +228,13 @@ func RemoteContainerEnvVariables(envs []corev1.EnvVar, saName string) []corev1.E
 
 // RemoteHostAliases forges the host aliases to override the IP address associated with the kubernetes.default service
 // to enable offloaded containers to contact back the local API server, instead of the remote one.
-func RemoteHostAliases(aliases []corev1.HostAlias, kubernetesServiceIPRetriever KubernetesServiceIPGetter) []corev1.HostAlias {
+func RemoteHostAliases(aliases []corev1.HostAlias, enableAPIServerSupport bool, retriever KubernetesServiceIPGetter) []corev1.HostAlias {
+	if !enableAPIServerSupport {
+		return aliases
+	}
+
 	return append(aliases, corev1.HostAlias{
-		IP: kubernetesServiceIPRetriever(), Hostnames: []string{kubernetesAPIService, kubernetesAPIService + ".svc"}})
+		IP: retriever(), Hostnames: []string{kubernetesAPIService, kubernetesAPIService + ".svc"}})
 }
 
 // RemoteTolerations forges the tolerations for a reflected pod.
@@ -244,18 +253,27 @@ func RemoteTolerations(inputTolerations []corev1.Toleration) []corev1.Toleration
 }
 
 // RemoteVolumes forges the volumes for a reflected pod, appropriately modifying the one related to the service account.
-func RemoteVolumes(volumes []corev1.Volume, saSecretRetriever func() string) []corev1.Volume {
+func RemoteVolumes(volumes []corev1.Volume, enableAPIServerSupport bool, saSecretRetriever func() string) []corev1.Volume {
 	for i := range volumes {
 		// Modify the projected volume which refers to the service account (if any),
 		// to make it target the underlying secret/configmap reflected to the remote cluster.
 		if volumes[i].Projected != nil && strings.HasPrefix(volumes[i].Name, ServiceAccountVolumeName) {
+			var offset int
 			for j := range volumes[i].Projected.Sources {
+				j -= offset // Acccount for the entry that might have been previously deleted.
 				source := &volumes[i].Projected.Sources[j]
 				if source.ConfigMap != nil {
 					// Replace the certification authority configmap with the remapped name.
 					source.ConfigMap.Name = RemoteConfigMapName(source.ConfigMap.Name)
 				} else if source.ServiceAccountToken != nil {
-					// Replace the ServiceAccountToken entry with the corresponding secret one.
+					if !enableAPIServerSupport {
+						// Remove the entry referring to the service account.
+						volumes[i].Projected.Sources = append(volumes[i].Projected.Sources[:j], volumes[i].Projected.Sources[j+1:]...)
+						offset++
+						continue
+					}
+
+					// Replace the ServiceAccountToken entry with the corresponding secret one, only in case it is enabled.
 					source.ServiceAccountToken = nil
 					source.Secret = &corev1.SecretProjection{
 						LocalObjectReference: corev1.LocalObjectReference{Name: saSecretRetriever()},