inject: allow automatic detection of native sidecar support

istio · Feb 26, 2024 · 75d38ed · 75d38ed
1 parent 774242c
commit 75d38ed
Show file tree

Hide file tree

Showing 34 changed files with 349 additions and 275 deletions.
diff --git a/manifests/charts/istio-control/istio-discovery/files/injection-template.yaml b/manifests/charts/istio-control/istio-discovery/files/injection-template.yaml
@@ -24,7 +24,6 @@
     {{- end }}
   {{- end }}
 {{- end }}
-{{ $nativeSidecar := (eq (env "ENABLE_NATIVE_SIDECARS" "false") "true") }}
 {{- $containers := list }}
 {{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
 metadata:
@@ -67,7 +66,7 @@ metadata:
 spec:
   {{- $holdProxy := and
       (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts)
-      (not $nativeSidecar) }}
+      (not .NativeSidecars) }}
   initContainers:
   {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }}
   {{ if or .Values.pilot.cni.enabled .Values.istio_cni.enabled -}}
@@ -187,7 +186,7 @@ spec:
       runAsNonRoot: false
       runAsUser: 0
   {{ end }}
-  {{ if not $nativeSidecar }}
+  {{ if not .NativeSidecars }}
   containers:
   {{ end }}
   - name: istio-proxy
@@ -196,7 +195,7 @@ spec:
   {{- else }}
     image: "{{ .ProxyImage }}"
   {{- end }}
-    {{ if $nativeSidecar }}restartPolicy: Always{{end}}
+    {{ if .NativeSidecars }}restartPolicy: Always{{end}}
     ports:
     - containerPort: 15090
       protocol: TCP
@@ -225,7 +224,7 @@ spec:
           command:
           - pilot-agent
           - wait
-  {{- else if $nativeSidecar }}
+  {{- else if .NativeSidecars }}
     {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}}
     lifecycle:
       preStop:

diff --git a/manifests/charts/istiod-remote/files/injection-template.yaml b/manifests/charts/istiod-remote/files/injection-template.yaml
@@ -24,7 +24,6 @@
     {{- end }}
   {{- end }}
 {{- end }}
-{{ $nativeSidecar := (eq (env "ENABLE_NATIVE_SIDECARS" "false") "true") }}
 {{- $containers := list }}
 {{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
 metadata:
@@ -67,7 +66,7 @@ metadata:
 spec:
   {{- $holdProxy := and
       (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts)
-      (not $nativeSidecar) }}
+      (not .NativeSidecars) }}
   initContainers:
   {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }}
   {{ if or .Values.pilot.cni.enabled .Values.istio_cni.enabled -}}
@@ -187,7 +186,7 @@ spec:
       runAsNonRoot: false
       runAsUser: 0
   {{ end }}
-  {{ if not $nativeSidecar }}
+  {{ if not .NativeSidecars }}
   containers:
   {{ end }}
   - name: istio-proxy
@@ -196,7 +195,7 @@ spec:
   {{- else }}
     image: "{{ .ProxyImage }}"
   {{- end }}
-    {{ if $nativeSidecar }}restartPolicy: Always{{end}}
+    {{ if .NativeSidecars }}restartPolicy: Always{{end}}
     ports:
     - containerPort: 15090
       protocol: TCP
@@ -225,7 +224,7 @@ spec:
           command:
           - pilot-agent
           - wait
-  {{- else if $nativeSidecar }}
+  {{- else if .NativeSidecars }}
     {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}}
     lifecycle:
       preStop:

diff --git a/...tor/cmd/mesh/testdata/manifest-generate/output/all_on.golden-show-in-gh-pull-request.yaml b/...tor/cmd/mesh/testdata/manifest-generate/output/all_on.golden-show-in-gh-pull-request.yaml
@@ -8951,7 +8951,7 @@ data:
             {{- end }}
           {{- end }}
         {{- end }}
-        {{ $nativeSidecar := (eq (env "ENABLE_NATIVE_SIDECARS" "false") "true") }}
+        {{ .NativeSidecars := (eq (env "ENABLE_NATIVE_SIDECARS" "false") "true") }}
         {{- $containers := list }}
         {{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
         metadata:
@@ -8994,7 +8994,7 @@ data:
         spec:
           {{- $holdProxy := and
               (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts)
-              (not $nativeSidecar) }}
+              (not .NativeSidecars) }}
           initContainers:
           {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }}
           {{ if .Values.istio_cni.enabled -}}
@@ -9114,7 +9114,7 @@ data:
               runAsNonRoot: false
               runAsUser: 0
           {{ end }}
-          {{ if not $nativeSidecar }}
+          {{ if not .NativeSidecars }}
           containers:
           {{ end }}
           - name: istio-proxy
@@ -9123,7 +9123,7 @@ data:
           {{- else }}
             image: "{{ .ProxyImage }}"
           {{- end }}
-            {{ if $nativeSidecar }}restartPolicy: Always{{end}}
+            {{ if .NativeSidecars }}restartPolicy: Always{{end}}
             ports:
             - containerPort: 15090
               protocol: TCP
@@ -9152,7 +9152,7 @@ data:
                   command:
                   - pilot-agent
                   - wait
-          {{- else if $nativeSidecar }}
+          {{- else if .NativeSidecars }}
             {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}}
             lifecycle:
               preStop:

diff --git a/operator/cmd/mesh/testdata/manifest-generate/output/pilot_default.golden.yaml b/operator/cmd/mesh/testdata/manifest-generate/output/pilot_default.golden.yaml
@@ -504,7 +504,7 @@ data:
             {{- end }}
           {{- end }}
         {{- end }}
-        {{ $nativeSidecar := (eq (env "ENABLE_NATIVE_SIDECARS" "false") "true") }}
+        {{ .NativeSidecars := (eq (env "ENABLE_NATIVE_SIDECARS" "false") "true") }}
         {{- $containers := list }}
         {{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
         metadata:
@@ -547,7 +547,7 @@ data:
         spec:
           {{- $holdProxy := and
               (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts)
-              (not $nativeSidecar) }}
+              (not .NativeSidecars) }}
           initContainers:
           {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }}
           {{ if .Values.istio_cni.enabled -}}
@@ -667,7 +667,7 @@ data:
               runAsNonRoot: false
               runAsUser: 0
           {{ end }}
-          {{ if not $nativeSidecar }}
+          {{ if not .NativeSidecars }}
           containers:
           {{ end }}
           - name: istio-proxy
@@ -676,7 +676,7 @@ data:
           {{- else }}
             image: "{{ .ProxyImage }}"
           {{- end }}
-            {{ if $nativeSidecar }}restartPolicy: Always{{end}}
+            {{ if .NativeSidecars }}restartPolicy: Always{{end}}
             ports:
             - containerPort: 15090
               protocol: TCP
@@ -705,7 +705,7 @@ data:
                   command:
                   - pilot-agent
                   - wait
-          {{- else if $nativeSidecar }}
+          {{- else if .NativeSidecars }}
             {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}}
             lifecycle:
               preStop:

diff --git a/operator/cmd/mesh/testdata/manifest-generate/output/sidecar_template.golden.yaml b/operator/cmd/mesh/testdata/manifest-generate/output/sidecar_template.golden.yaml
@@ -40,7 +40,7 @@ data:
             {{- end }}
           {{- end }}
         {{- end }}
-        {{ $nativeSidecar := (eq (env "ENABLE_NATIVE_SIDECARS" "false") "true") }}
+        {{ .NativeSidecars := (eq (env "ENABLE_NATIVE_SIDECARS" "false") "true") }}
         {{- $containers := list }}
         {{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
         metadata:
@@ -83,7 +83,7 @@ data:
         spec:
           {{- $holdProxy := and
               (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts)
-              (not $nativeSidecar) }}
+              (not .NativeSidecars) }}
           initContainers:
           {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }}
           {{ if .Values.istio_cni.enabled -}}
@@ -203,7 +203,7 @@ data:
               runAsNonRoot: false
               runAsUser: 0
           {{ end }}
-          {{ if not $nativeSidecar }}
+          {{ if not .NativeSidecars }}
           containers:
           {{ end }}
           - name: istio-proxy
@@ -212,7 +212,7 @@ data:
           {{- else }}
             image: "{{ .ProxyImage }}"
           {{- end }}
-            {{ if $nativeSidecar }}restartPolicy: Always{{end}}
+            {{ if .NativeSidecars }}restartPolicy: Always{{end}}
             ports:
             - containerPort: 15090
               protocol: TCP
@@ -241,7 +241,7 @@ data:
                   command:
                   - pilot-agent
                   - wait
-          {{- else if $nativeSidecar }}
+          {{- else if .NativeSidecars }}
             {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}}
             lifecycle:
               preStop:

diff --git a/pilot/pkg/config/kube/gateway/deploymentcontroller.go b/pilot/pkg/config/kube/gateway/deploymentcontroller.go
@@ -40,6 +40,7 @@ import (
 	"istio.io/istio/pkg/config/protocol"
 	"istio.io/istio/pkg/config/schema/gvk"
 	"istio.io/istio/pkg/config/schema/gvr"
+	common_features "istio.io/istio/pkg/features"
 	"istio.io/istio/pkg/kube"
 	"istio.io/istio/pkg/kube/controllers"
 	"istio.io/istio/pkg/kube/inject"
@@ -387,6 +388,7 @@ func (d *DeploymentController) configureIstioGateway(log *istiolog.Scope, gw gat
 		ServiceType:               serviceType,
 		ProxyUID:                  proxyUID,
 		ProxyGID:                  proxyGID,
+		CompliancePolicy:          common_features.CompliancePolicy,
 		InfrastructureLabels:      gw.GetLabels(),
 		InfrastructureAnnotations: gw.GetAnnotations(),
 	}
@@ -644,6 +646,7 @@ type TemplateInput struct {
 	Revision                  string
 	ProxyUID                  int64
 	ProxyGID                  int64
+	CompliancePolicy          string
 	InfrastructureLabels      map[string]string
 	InfrastructureAnnotations map[string]string
 	GatewayNameLabel          string

diff --git a/pilot/pkg/features/experimental.go b/pilot/pkg/features/experimental.go
@@ -217,9 +217,25 @@ var (
 	EnableOptimizedServicePush = env.RegisterBoolVar("ISTIO_ENABLE_OPTIMIZED_SERVICE_PUSH", true,
 		"If enabled, Istiod will not push changes on arbitrary annotation change.").Get()
 
-	// This is used in injection templates, it is not unused.
-	EnableNativeSidecars = env.Register("ENABLE_NATIVE_SIDECARS", false,
-		"If set, used Kubernetes native Sidecar container support. Requires SidecarContainer feature flag.")
+	EnableNativeSidecars = func() NativeSidecarMode {
+		v := env.Register("ENABLE_NATIVE_SIDECARS", "false",
+			"If set, used Kubernetes native Sidecar container support. Requires SidecarContainer feature flag."+
+				" Set to true or false to unconditionally enable. Set to auto-beta or auto-stable to automatically enable"+
+				" if support is detected (at the beta or stable level).").Get()
+		switch v {
+		case "never", "false":
+			return NativeSidecarModeNever
+		case "always", "true":
+			return NativeSidecarModeAlways
+		case "auto-beta":
+			return NativeSidecarModeAutoBeta
+		case "auto-stable":
+			return NativeSidecarModeAutoStable
+		default:
+			log.Warnf("unknown ENABLE_NATIVE_SIDECARS value %q", v)
+			return NativeSidecarModeNever
+		}
+	}()
 
 	OptimizedConfigRebuild = env.Register("ENABLE_OPTIMIZED_CONFIG_REBUILD", true,
 		"If enabled, pilot will only rebuild config for resources that have changed").Get()
@@ -228,3 +244,16 @@ var (
 		"If enabled, istiod will persist the oldest first heuristic for subtly conflicting traffic policy selection"+
 			"(such as with overlapping wildcard hosts)").Get()
 )
+
+type NativeSidecarMode int
+
+const (
+	// Never use native sidecar
+	NativeSidecarModeNever NativeSidecarMode = iota
+	// Always use native sidecar
+	NativeSidecarModeAlways = iota
+	// AutoBeta will use native sidecars if its detected as supported at a beta+ stability level
+	NativeSidecarModeAutoBeta = iota
+	// AutoStable will use native sidecars if its detected as supported at a stable stability level
+	NativeSidecarModeAutoStable = iota
+)
diff --git a/pkg/kube/inject/inject.go b/pkg/kube/inject/inject.go
@@ -43,7 +43,6 @@ import (
 	meshconfig "istio.io/api/mesh/v1alpha1"
 	proxyConfig "istio.io/api/networking/v1beta1"
 	opconfig "istio.io/istio/operator/pkg/apis/istio/v1alpha1"
-	"istio.io/istio/pilot/pkg/features"
 	"istio.io/istio/pkg/config/mesh"
 	common_features "istio.io/istio/pkg/features"
 	"istio.io/istio/pkg/kube"
@@ -106,6 +105,7 @@ type SidecarTemplateData struct {
 	MeshConfig               *meshconfig.MeshConfig
 	Values                   map[string]any
 	Revision                 string
+	NativeSidecars           bool
 	ProxyImage               string
 	ProxyUID                 int64
 	ProxyGID                 int64
@@ -419,16 +419,19 @@ func RunTemplate(params InjectionParameters) (mergedPod *corev1.Pod, templatePod
 
 	proxyUID, proxyGID := GetProxyIDs(params.namespace)
 
+	// When changing this, make sure to change TemplateInput in deploymentcontroller.go
 	data := SidecarTemplateData{
-		TypeMeta:                 params.typeMeta,
-		DeploymentMeta:           params.deployMeta,
-		ObjectMeta:               strippedPod.ObjectMeta,
-		Spec:                     strippedPod.Spec,
-		ProxyConfig:              params.proxyConfig,
-		MeshConfig:               meshConfig,
-		Values:                   params.valuesConfig.asMap,
-		Revision:                 params.revision,
-		ProxyImage:               ProxyImage(params.valuesConfig.asStruct, params.proxyConfig.Image, strippedPod.Annotations),
+		TypeMeta:       params.typeMeta,
+		DeploymentMeta: params.deployMeta,
+		ObjectMeta:     strippedPod.ObjectMeta,
+		Spec:           strippedPod.Spec,
+		ProxyConfig:    params.proxyConfig,
+		MeshConfig:     meshConfig,
+		Values:         params.valuesConfig.asMap,
+		Revision:       params.revision,
+		ProxyImage:     ProxyImage(params.valuesConfig.asStruct, params.proxyConfig.Image, strippedPod.Annotations),
+
+		NativeSidecars:           params.nativeSidecar,
 		ProxyUID:                 proxyUID,
 		ProxyGID:                 proxyGID,
 		InboundTrafficPolicyMode: InboundTrafficPolicyMode(meshConfig),
@@ -457,7 +460,7 @@ func RunTemplate(params InjectionParameters) (mergedPod *corev1.Pod, templatePod
 		// these will be in the `containers` field.
 		// So if we see the proxy container in `containers` in the original pod, and in `initContainers` in the template pod,
 		// move the container.
-		if features.EnableNativeSidecars.Get() &&
+		if params.nativeSidecar &&
 			FindContainer(ProxyContainerName, templatePod.Spec.InitContainers) != nil &&
 			FindContainer(ProxyContainerName, mergedPod.Spec.Containers) != nil {
 			mergedPod = mergedPod.DeepCopy()

diff --git a/pkg/kube/inject/inject_test.go b/pkg/kube/inject/inject_test.go
@@ -283,7 +283,7 @@ func TestInjection(t *testing.T) {
 			in:   "proxy-override-args.yaml",
 			want: "proxy-override-args-native.yaml.injected",
 			setup: func(t test.Failer) {
-				test.SetEnvForTest(t, features.EnableNativeSidecars.Name, "true")
+				test.SetForTest(t, &features.EnableNativeSidecars, features.NativeSidecarModeAlways)
 			},
 		},
 		{
@@ -294,14 +294,14 @@ func TestInjection(t *testing.T) {
 			in:   "gateway.yaml",
 			want: "gateway.yaml.injected",
 			setup: func(t test.Failer) {
-				test.SetEnvForTest(t, features.EnableNativeSidecars.Name, "true")
+				test.SetForTest(t, &features.EnableNativeSidecars, features.NativeSidecarModeAlways)
 			},
 		},
 		{
 			in:   "native-sidecar.yaml",
 			want: "native-sidecar.yaml.injected",
 			setup: func(t test.Failer) {
-				test.SetEnvForTest(t, features.EnableNativeSidecars.Name, "true")
+				test.SetForTest(t, &features.EnableNativeSidecars, features.NativeSidecarModeAlways)
 			},
 		},
 		{