Embrace non-sidecar mode on Kubernetes (#386)

Signed-off-by: David Grove <groved@us.ibm.com>

ci/testInCluster.sh

@@ -53,15 +53,15 @@ else
 fi
 
 
-echo "*** Running in-cluster no-sidecar actors-ykt ***"
+echo "*** Running in-cluster sidecar actors-ykt ***"
 
-helm install ykt-sc $ROOTDIR/examples/actors-ykt/deploy/chart --set image=localhost:5000/kar/kar-examples-js-actors-ykt --set noSidecar=true
+helm install ykt-sc $ROOTDIR/examples/actors-ykt/deploy/chart --set image=localhost:5000/kar/kar-examples-js-actors-ykt --set enableSidecar=true
 
 if helm test ykt-sc; then
-    echo "PASSED! In cluster no-sidecar actors-ykt passed."
+    echo "PASSED! In cluster sidecar actors-ykt passed."
     helm delete ykt-sc
 else
-    echo "FAILED: In cluster no-sidecar actors-ykt failed."
+    echo "FAILED: In cluster sidecar actors-ykt failed."
     kubectl logs ykt-client -c client
     kubectl logs ykt-client -c kar
     kubectl delete pod ykt-client

core/internal/config/config.go

@@ -87,8 +87,8 @@ var (
 	// RuntimePort is the HTTP port the runtime will be listening on
 	RuntimePort int
 
-	// KubernetesMode is true when this process is running in a sidecar container in a Kubernetes Pod
-	KubernetesMode bool
+	// SidecarMode is true when this process is running in a sidecar container in a Kubernetes Pod
+	SidecarMode bool
 
 	// KafkaConfig contains the configuration information to connect with Kafka
 	KafkaConfig rpc.Config
@@ -240,7 +240,7 @@ Available commands:
 		flag.DurationVar(&ActorReminderAcceptableDelay, "actor_reminder_acceptable_delay", 3*time.Second, "Threshold at which reminders are logged as being late")
 		flag.IntVar(&AppPort, "app_port", 8080, "The port used by KAR to connect to the application")
 		flag.IntVar(&RuntimePort, "runtime_port", 0, "The port used by the application to connect to KAR")
-		flag.BoolVar(&KubernetesMode, "kubernetes_mode", false, "Running as a sidecar container in a Kubernetes Pod")
+		flag.BoolVar(&SidecarMode, "sidecar_mode", false, "Running as a sidecar container in a Kubernetes Pod")
 		flag.BoolVar(&H2C, "h2c", false, "Use h2c to communicate with service")
 		flag.StringVar(&Hostname, "hostname", "localhost", "Hostname")
 		flag.DurationVar(&KafkaConfig.SessionBusyTimeout, "actor_busy_timeout", 2*time.Minute, "Time to wait on a busy actor before timing out (0 is infinite)")

core/internal/runtime/runtime-main.go

@@ -151,7 +151,7 @@ func Main() {
 	}()
 
 	var listenHost string
-	if config.KubernetesMode {
+	if config.SidecarMode {
 		listenHost = fmt.Sprintf(":%d", config.RuntimePort)
 	} else {
 		listenHost = fmt.Sprintf("127.0.0.1:%d", config.RuntimePort)

core/internal/sidecar/injection.go

@@ -40,6 +40,7 @@ var (
 const (
 	actorAnnotation       = "kar.ibm.com/actors"
 	appNameAnnotation     = "kar.ibm.com/app"
+	sidecarAnnotation     = "kar.ibm.com/sidecarContainer"
 	serviceNameAnnotation = "kar.ibm.com/service"
 	appPortAnnotation     = "kar.ibm.com/appPort"
 	runtimePortAnnotation = "kar.ibm.com/runtimePort"
@@ -48,8 +49,9 @@ const (
 
 	extraArgsSeparator = ","
 
-	defaultAppPort     = "8080"
-	defaultRuntimePort = "3500"
+	defaultAppPort          = "8080"
+	defaultRuntimePort      = "3500"
+	defaultSidecarContainer = "true"
 
 	sidecarName        = "kar"
 	karImagePullSecret = "kar.ibm.com.image-pull"
@@ -112,6 +114,7 @@ func possiblyInjectSidecar(ar v1.AdmissionReview) *v1.AdmissionResponse {
 	annotations := pod.GetObjectMeta().GetAnnotations()
 	if appName, ok := annotations[appNameAnnotation]; ok {
 		logger.Info("Pod %v has appName %v", pod.Name, appName)
+		patches := []patchOperation{}
 		containers := pod.Spec.Containers
 
 		for _, container := range containers {
@@ -121,7 +124,7 @@ func possiblyInjectSidecar(ar v1.AdmissionReview) *v1.AdmissionResponse {
 			}
 		}
 
-		cmdLine, appEnv, runtimePortStr := processAnnotations(pod)
+		extraArgs, appEnv, runtimePortStr, enableSidecar := processAnnotations(pod)
 		runtimePort, err := strconv.Atoi(runtimePortStr)
 
 		if len(appEnv) > 0 {
@@ -130,56 +133,63 @@ func possiblyInjectSidecar(ar v1.AdmissionReview) *v1.AdmissionResponse {
 			}
 		}
 
-		sidecar := []corev1.Container{{
-			Name:          sidecarName,
-			Image:         fmt.Sprintf("%s:%s", sidecarImage, sidecarImageTag),
-			Command:       []string{"/kar/bin/kar"},
-			Args:          cmdLine,
-			Env:           []corev1.EnvVar{{Name: "KAR_POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}}}},
-			Ports:         []corev1.ContainerPort{{ContainerPort: int32(runtimePort), Protocol: corev1.ProtocolTCP, Name: "kar"}},
-			LivenessProbe: &corev1.Probe{Handler: corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "kar/v1/system/health", Port: intstr.FromInt(runtimePort)}}},
-			Lifecycle:     &corev1.Lifecycle{PreStop: &corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "kar/v1/system/shutdown", Port: intstr.FromInt(runtimePort)}}},
-			VolumeMounts:  []corev1.VolumeMount{{Name: "kar-ibm-com-config", MountPath: karRTConfigMount, ReadOnly: true}},
-		}}
-		containers = append(sidecar, containers...)
-		updateContainersPatch := patchOperation{
+		if enableSidecar {
+			sidecar := []corev1.Container{{
+				Name:          sidecarName,
+				Image:         fmt.Sprintf("%s:%s", sidecarImage, sidecarImageTag),
+				Command:       []string{"/kar/bin/kar"},
+				Args:          append([]string{"run", "-app", appName}, extraArgs...),
+				Env:           []corev1.EnvVar{{Name: "KAR_POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}}}},
+				Ports:         []corev1.ContainerPort{{ContainerPort: int32(runtimePort), Protocol: corev1.ProtocolTCP, Name: "kar"}},
+				LivenessProbe: &corev1.Probe{Handler: corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "kar/v1/system/health", Port: intstr.FromInt(runtimePort)}}},
+				Lifecycle:     &corev1.Lifecycle{PreStop: &corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "kar/v1/system/shutdown", Port: intstr.FromInt(runtimePort)}}},
+				VolumeMounts:  []corev1.VolumeMount{{Name: "kar-ibm-com-config", MountPath: karRTConfigMount, ReadOnly: true}},
+			}}
+			containers = append(sidecar, containers...)
+		} else {
+			for index, container := range containers {
+				containers[index].VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: "kar-ibm-com-config", MountPath: karRTConfigMount, ReadOnly: true})
+				containers[index].Env = append(container.Env, corev1.EnvVar{Name: "KAR_EXTRA_ARGS", Value: strings.Join(extraArgs, " ")})
+			}
+		}
+		patches = append(patches, patchOperation{
 			Op:    "replace",
 			Path:  "/spec/containers",
 			Value: containers,
-		}
+		})
 
 		configVolume := corev1.Volume{
 			Name:         "kar-ibm-com-config",
 			VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: karRTConfigSecret}},
 		}
-		var addVolumePatch patchOperation
 		if pod.Spec.Volumes == nil {
-			addVolumePatch = patchOperation{
+			patches = append(patches, patchOperation{
 				Op:    "replace",
 				Path:  "/spec/volumes",
 				Value: []corev1.Volume{configVolume},
-			}
+			})
 		} else {
-			addVolumePatch = patchOperation{
+			patches = append(patches, patchOperation{
 				Op:    "add",
 				Path:  "/spec/volumes/-",
 				Value: configVolume,
-			}
+			})
 		}
 
-		imagePull := corev1.LocalObjectReference{Name: karImagePullSecret}
-		var pullSecretPatch patchOperation
-		if pod.Spec.ImagePullSecrets == nil {
-			pullSecretPatch = patchOperation{
-				Op:    "add",
-				Path:  "/spec/imagePullSecrets",
-				Value: []corev1.LocalObjectReference{imagePull},
-			}
-		} else {
-			pullSecretPatch = patchOperation{
-				Op:    "add",
-				Path:  "/spec/imagePullSecrets/-",
-				Value: imagePull,
+		if enableSidecar {
+			imagePull := corev1.LocalObjectReference{Name: karImagePullSecret}
+			if pod.Spec.ImagePullSecrets == nil {
+				patches = append(patches, patchOperation{
+					Op:    "add",
+					Path:  "/spec/imagePullSecrets",
+					Value: []corev1.LocalObjectReference{imagePull},
+				})
+			} else {
+				patches = append(patches, patchOperation{
+					Op:    "add",
+					Path:  "/spec/imagePullSecrets/-",
+					Value: imagePull,
+				})
 			}
 		}
 
@@ -198,13 +208,11 @@ func possiblyInjectSidecar(ar v1.AdmissionReview) *v1.AdmissionResponse {
 		if serviceName, ok := annotations[serviceNameAnnotation]; ok {
 			labels[serviceNameAnnotation] = serviceName
 		}
-		patchPodLabels := patchOperation{
+		patches = append(patches, patchOperation{
 			Op:    op,
 			Path:  "/metadata/labels",
 			Value: labels,
-		}
-
-		patches := []patchOperation{updateContainersPatch, addVolumePatch, pullSecretPatch, patchPodLabels}
+		})
 
 		patchBytes, err := json.Marshal(patches)
 		if err != nil {
@@ -216,50 +224,60 @@ func possiblyInjectSidecar(ar v1.AdmissionReview) *v1.AdmissionResponse {
 		pt := v1.PatchTypeJSONPatch
 		reviewResponse.PatchType = &pt
 	} else {
-		logger.Info("Pod %v lacks 'kar.ibm.com/app' annotation; no sidecar injected", pod.Name)
+		logger.Info("Pod %v lacks 'kar.ibm.com/app' annotation; no sidecar injected/enabled", pod.Name)
 	}
 
 	return &reviewResponse
 }
 
-func processAnnotations(pod corev1.Pod) ([]string, []corev1.EnvVar, string) {
+func processAnnotations(pod corev1.Pod) ([]string, []corev1.EnvVar, string, bool) {
 	annotations := pod.GetObjectMeta().GetAnnotations()
 	appName := annotations[appNameAnnotation]
-	cmd := []string{"run", "-kubernetes_mode", "-config_dir", karRTConfigMount, "-app", appName}
-	appEnv := []corev1.EnvVar{}
+	extraArgs := []string{"-config_dir", karRTConfigMount}
+	appEnv := []corev1.EnvVar{{Name: "KAR_APP", Value: appName}}
+
+	var sidecarContainer = defaultSidecarContainer
+	if sc, ok := annotations[sidecarAnnotation]; ok {
+		sidecarContainer = sc
+	}
+	if sidecarContainer == "true" {
+		extraArgs = append(extraArgs, "-sidecar_mode")
+	} else {
+		appEnv = append(appEnv, corev1.EnvVar{Name: "KAR_SIDECAR_IN_CONTAINER", Value: "true"})
+	}
 
 	if serviceName, ok := annotations[serviceNameAnnotation]; ok {
-		cmd = append(cmd, "-service", serviceName)
+		extraArgs = append(extraArgs, "-service", serviceName)
 	}
 
 	if actors, ok := annotations[actorAnnotation]; ok {
-		cmd = append(cmd, "-actors", actors)
+		extraArgs = append(extraArgs, "-actors", actors)
 	}
 
 	var appPort = defaultAppPort
 	if p, ok := annotations[appPortAnnotation]; ok {
 		appPort = p
 	}
-	cmd = append(cmd, "-app_port", appPort)
+	extraArgs = append(extraArgs, "-app_port", appPort)
 	appEnv = append(appEnv, corev1.EnvVar{Name: "KAR_APP_PORT", Value: appPort})
 
 	var runtimePort = defaultRuntimePort
 	if p, ok := annotations[runtimePortAnnotation]; ok {
 		runtimePort = p
 	}
-	cmd = append(cmd, "-runtime_port", runtimePort)
+	extraArgs = append(extraArgs, "-runtime_port", runtimePort)
 	appEnv = append(appEnv, corev1.EnvVar{Name: "KAR_RUNTIME_PORT", Value: runtimePort})
 
 	if verbose, ok := annotations[verboseAnnotation]; ok {
-		cmd = append(cmd, "-v", verbose)
+		extraArgs = append(extraArgs, "-v", verbose)
 	}
 
 	if moreArgs, ok := annotations[extraArgsAnnotation]; ok {
 		theArgs := strings.Split(moreArgs, extraArgsSeparator)
-		cmd = append(cmd, theArgs...)
+		extraArgs = append(extraArgs, theArgs...)
 	}
 
-	return cmd, appEnv, runtimePort
+	return extraArgs, appEnv, runtimePort, sidecarContainer == "true"
 }
 
 func toV1AdmissionResponse(err error) *v1.AdmissionResponse {

docs/kar-deployments.md

@@ -50,7 +50,7 @@ configurations to support high availability and increased scalability.
 
 When deployed on a Kubernetes of OpenShift cluster, the KAR runtime
 system also includes a mutating web hook that supports
-injecting a "sidecar" container into Pods that are annotated as
+injecting KAR runtime configuration data into Pods that are annotated as
 containing KAR application components.  This significantly simplifies
 the configuration of these components by automating the injection of
 the credentials needed to connect to the Redis and Kafka instances
@@ -139,18 +139,27 @@ any in-cluster deployment of KAR.
 
 For its in-cluster configurations, the KAR runtime system is deployed
 in the `kar-system` namespace and includes a mutating webhook whose
-job is to inject a "sidecar" container containing the `kar` executable
-into every Pod that is annotated with `kar.ibm.com/app`. This
-machinery enables existing Helm charts and Kubernetes YAML to be
-adapted for KAR with minimal changes.  The mutating webhook process
-the following annotations:
+job is to inject additional configuration information to enable
+application Pods to be joined to the KAR application mesh. Application
+Pods that should be mutated are indicated by annotating them with
+`kar.ibm.com/app`.  We support two modes of operation.  In the
+recommended mode, the application container already contains the `kar`
+executable and the webhook only needs to inject volume mounts and
+environment variables to configure it. If the container does not
+contain the `kar` executable then the webhook can inject and configure
+an additional "sidecar" container containing the `kar` executable.
+This sidecar mode enables unmodified containers to be joined to a KAR
+application mesh, but is less resilient to failures due to incomplete
+support by Kubernetes for sidecar container lifecycle operations.
+The mutating webhook process the following annotations:
    + kar.ibm.com/app - sets the `-app` argument of `kar run`
    + kar.ibm.com/actors: sets the `-actors` argument of `kar run`
    + kar.ibm.com/service: sets the `-service` argument of `kar run`
    + kar.ibm.com/verbose: sets the `-verbose` argument of `kar run`
    + kar.ibm.com/appPort: sets the `-app_port` argument of `kar run`
    + kar.ibm.com/runtimePort: sets the `-runtime_port` argument of `kar run`
    + kar.ibm.com/extraArgs: additional command line arguments for `kar run`
+   + kar.ibm.com/sidecarContainer - "true" to enable injection of a sidecar container
 
 If you are using a release version of the `kar` cli then, by default,
 the matching KAR runtime images will be pulled from our public quay.io

examples/actors-dp-java-reactive/deploy/server-quay.yaml

@@ -32,6 +32,7 @@ spec:
         kar.ibm.com/app: dp
         kar.ibm.com/actors: Cafe,Fork,Philosopher,Table
         kar.ibm.com/appPort: "8080"
+        kar.ibm.com/sidecarContainer: "false"
     spec:
       containers:
       - name: actors

examples/actors-dp-java-reactive/deploy/server.yaml

@@ -32,6 +32,7 @@ spec:
         kar.ibm.com/app: dp
         kar.ibm.com/actors: Cafe,Fork,Philosopher,Table
         kar.ibm.com/appPort: "8080"
+        kar.ibm.com/sidecarContainer: "false"
     spec:
       containers:
       - name: actors

examples/actors-dp-java/deploy/server-quay.yaml

@@ -32,6 +32,7 @@ spec:
         kar.ibm.com/app: dp
         kar.ibm.com/actors: Cafe,Fork,Philosopher,Table
         kar.ibm.com/appPort: "8080"
+        kar.ibm.com/sidecarContainer: "false"
     spec:
       containers:
       - name: actors

examples/actors-dp-java/deploy/server.yaml

@@ -32,6 +32,7 @@ spec:
         kar.ibm.com/app: dp
         kar.ibm.com/actors: Cafe,Fork,Philosopher,Table
         kar.ibm.com/appPort: "8080"
+        kar.ibm.com/sidecarContainer: "false"
     spec:
       containers:
       - name: actors

examples/actors-dp-js/dp-js-debug.yaml

@@ -26,6 +26,7 @@ metadata:
     kar.ibm.com/actors: Cafe,Table,Fork,Philosopher
     kar.ibm.com/extraArgs: '-actor_busy_timeout=0'
     kar.ibm.com/verbose: info
+    kar.ibm.com/sidecarContainer: "false"
 spec:
   restartPolicy: Never
   containers:

examples/actors-ykt/deploy/chart/templates/server.yaml

@@ -30,32 +30,11 @@ spec:
       labels:
         name: ykt-server
       annotations:
-{{ if not .Values.noSidecar }}
         kar.ibm.com/app: ykt
+        kar.ibm.com/sidecarContainer: "{{ .Values.enableSidecar }}"
         kar.ibm.com/actors: Company,Site,Office,Researcher
         kar.ibm.com/verbose: {{ .Values.verbose }}
-{{ end }}
     spec:
-{{ if .Values.noSidecar }}
-      volumes:
-      - name: kar-ibm-com-config
-        secret:
-          secretName: kar.ibm.com.runtime-config
-{{ end }}
       containers:
       - name: server
         image: {{ .Values.image }}
-
-        env:
-{{ if .Values.noSidecar }}
-        - name: KAR_SIDECAR_IN_CONTAINER
-          value: "true"
-        - name: KAR_APP
-          value: ykt
-        - name: KAR_EXTRA_ARGS
-          value: -config_dir /var/run/secrets/kar.ibm.com -v {{ .Values.verbose }} -actors Company,Site,Office,Researcher
-        volumeMounts:
-        - mountPath: /var/run/secrets/kar.ibm.com
-          name: kar-ibm-com-config
-          readOnly: true
-{{ end }}