Executor can access the k8s apiserver with a out-of-cluster config fi…

…le (#1134)

Executor can access the k8s apiserver with a out-of-cluster config file

docs/workflow-controller-configmap.yaml

@@ -22,6 +22,19 @@ data:
     # (available since Argo v2.3)
     parallelism: 10
 
+    # uncomment flowing lines if workflow controller runs in a different k8s cluster with the 
+    # workflow workloads, or needs to communicate with the k8s apiserver using an out-of-cluster
+    # kubeconfig secret
+    # kubeConfig:
+    #   # name of the kubeconfig secret, may not be empty when kubeConfig specified
+    #   secretName: kubeconfig-secret
+    #   # key of the kubeconfig secret, may not be empty when kubeConfig specified
+    #   secretKey: kubeconfig
+    #   # mounting path of the kubeconfig secret, default to /kube/config
+    #   mountPath: /kubeconfig/mount/path
+    #   # volume name when mounting the secret, default to kubeconfig
+    #   volumeName: kube-config-volume
+
     # artifactRepository defines the default location to be used as the artifact repository for
     # container artifacts.
     artifactRepository:

workflow/common/common.go

@@ -110,6 +110,9 @@ const (
 	GlobalVarWorkflowCreationTimestamp = "workflow.creationTimestamp"
 	// LocalVarPodName is a step level variable that references the name of the pod
 	LocalVarPodName = "pod.name"
+
+	KubeConfigDefaultMountPath  = "/kube/config"
+	KubeConfigDefaultVolumeName = "kubeconfig"
 )
 
 // ExecutionControl contains execution control parameters for executor to decide how to execute the container

workflow/controller/config.go

@@ -30,6 +30,9 @@ type WorkflowControllerConfig struct {
 	// ExecutorResources specifies the resource requirements that will be used for the executor sidecar
 	ExecutorResources *apiv1.ResourceRequirements `json:"executorResources,omitempty"`
 
+	// KubeConfig specifies a kube config file for the wait & init containers
+	KubeConfig *KubeConfig `json:"kubeConfig,omitempty"`
+
 	// ContainerRuntimeExecutor specifies the container runtime interface to use, default is docker
 	ContainerRuntimeExecutor string `json:"containerRuntimeExecutor,omitempty"`
 
@@ -62,6 +65,21 @@ type WorkflowControllerConfig struct {
 	Parallelism int `json:"parallelism,omitempty"`
 }
 
+// KubeConfig is used for wait & init sidecar containers to communicate with a k8s apiserver by a outofcluster method,
+// it is used when the workflow controller is in a different cluster with the workflow workloads
+type KubeConfig struct {
+	// SecretName of the kubeconfig secret
+	// may not be empty if kuebConfig specified
+	SecretName string `json:"secretName"`
+	// SecretKey of the kubeconfig in the secret
+	// may not be empty if kubeConfig specified
+	SecretKey string `json:"secretKey"`
+	// VolumeName of kubeconfig, default to 'kubeconfig'
+	VolumeName string `json:"volumeName,omitempty"`
+	// MountPath of the kubeconfig secret, default to '/kube/config'
+	MountPath string `json:"mountPath,omitempty"`
+}
+
 // ArtifactRepository represents a artifact repository in which a controller will store its artifacts
 type ArtifactRepository struct {
 	// ArchiveLogs enables log archiving

workflow/controller/workflowpod.go

@@ -247,20 +247,14 @@ func substituteGlobals(pod *apiv1.Pod, globalParams map[string]string) (*apiv1.P
 }
 
 func (woc *wfOperationCtx) newInitContainer(tmpl *wfv1.Template) apiv1.Container {
-	ctr := woc.newExecContainer(common.InitContainerName, false)
-	ctr.Command = []string{"argoexec"}
-	ctr.Args = []string{"init"}
-	ctr.VolumeMounts = []apiv1.VolumeMount{
-		volumeMountPodMetadata,
-	}
+	ctr := woc.newExecContainer(common.InitContainerName, false, "init")
+	ctr.VolumeMounts = append([]apiv1.VolumeMount{volumeMountPodMetadata}, ctr.VolumeMounts...)
 	return *ctr
 }
 
 func (woc *wfOperationCtx) newWaitContainer(tmpl *wfv1.Template) (*apiv1.Container, error) {
-	ctr := woc.newExecContainer(common.WaitContainerName, false)
-	ctr.Command = []string{"argoexec"}
-	ctr.Args = []string{"wait"}
-	ctr.VolumeMounts = woc.createVolumeMounts()
+	ctr := woc.newExecContainer(common.WaitContainerName, false, "wait")
+	ctr.VolumeMounts = append(woc.createVolumeMounts(), ctr.VolumeMounts...)
 	return ctr, nil
 }
 
@@ -317,6 +311,20 @@ func (woc *wfOperationCtx) createVolumes() []apiv1.Volume {
 	volumes := []apiv1.Volume{
 		volumePodMetadata,
 	}
+	if woc.controller.Config.KubeConfig != nil {
+		name := woc.controller.Config.KubeConfig.VolumeName
+		if name == "" {
+			name = common.KubeConfigDefaultVolumeName
+		}
+		volumes = append(volumes, apiv1.Volume{
+			Name: name,
+			VolumeSource: apiv1.VolumeSource{
+				Secret: &apiv1.SecretVolumeSource{
+					SecretName: woc.controller.Config.KubeConfig.SecretName,
+				},
+			},
+		})
+	}
 	switch woc.controller.Config.ContainerRuntimeExecutor {
 	case common.ContainerRuntimeExecutorKubelet, common.ContainerRuntimeExecutorK8sAPI:
 		return volumes
@@ -325,7 +333,7 @@ func (woc *wfOperationCtx) createVolumes() []apiv1.Volume {
 	}
 }
 
-func (woc *wfOperationCtx) newExecContainer(name string, privileged bool) *apiv1.Container {
+func (woc *wfOperationCtx) newExecContainer(name string, privileged bool, subCommand string) *apiv1.Container {
 	exec := apiv1.Container{
 		Name:            name,
 		Image:           woc.controller.executorImage(),
@@ -334,10 +342,29 @@ func (woc *wfOperationCtx) newExecContainer(name string, privileged bool) *apiv1
 		SecurityContext: &apiv1.SecurityContext{
 			Privileged: &privileged,
 		},
+		Command: []string{"argoexec"},
+		Args:    []string{subCommand},
 	}
 	if woc.controller.Config.ExecutorResources != nil {
 		exec.Resources = *woc.controller.Config.ExecutorResources
 	}
+	if woc.controller.Config.KubeConfig != nil {
+		path := woc.controller.Config.KubeConfig.MountPath
+		if path == "" {
+			path = common.KubeConfigDefaultMountPath
+		}
+		name := woc.controller.Config.KubeConfig.VolumeName
+		if name == "" {
+			name = common.KubeConfigDefaultVolumeName
+		}
+		exec.VolumeMounts = []apiv1.VolumeMount{{
+			Name:      name,
+			MountPath: path,
+			ReadOnly:  true,
+			SubPath:   woc.controller.Config.KubeConfig.SecretKey,
+		}}
+		exec.Args = append(exec.Args, "--kubeconfig="+path)
+	}
 	return &exec
 }
 

workflow/controller/workflowpod_test.go

@@ -266,3 +266,53 @@ func TestVolumeAndVolumeMounts(t *testing.T) {
 		assert.Equal(t, "volume-name", pod.Spec.Containers[0].VolumeMounts[0].Name)
 	}
 }
+
+func TestOutOfCluster(t *testing.T) {
+	// default mount path & volume name
+	{
+		woc := newWoc()
+		woc.controller.Config.KubeConfig = &KubeConfig{
+			SecretName: "foo",
+			SecretKey:  "bar",
+		}
+
+		woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "")
+		podName := getPodName(woc.wf)
+		pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{})
+
+		assert.Nil(t, err)
+		assert.Equal(t, "kubeconfig", pod.Spec.Volumes[1].Name)
+		assert.Equal(t, "foo", pod.Spec.Volumes[1].VolumeSource.Secret.SecretName)
+
+		// kubeconfig volume is the last one
+		idx := len(pod.Spec.Containers[1].VolumeMounts) - 1
+		assert.Equal(t, "kubeconfig", pod.Spec.Containers[1].VolumeMounts[idx].Name)
+		assert.Equal(t, "/kube/config", pod.Spec.Containers[1].VolumeMounts[idx].MountPath)
+		assert.Equal(t, "--kubeconfig=/kube/config", pod.Spec.Containers[1].Args[1])
+	}
+
+	// custom mount path & volume name, in case name collision
+	{
+		woc := newWoc()
+		woc.controller.Config.KubeConfig = &KubeConfig{
+			SecretName: "foo",
+			SecretKey:  "bar",
+			MountPath:  "/some/path/config",
+			VolumeName: "kube-config-secret",
+		}
+
+		woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "")
+		podName := getPodName(woc.wf)
+		pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{})
+
+		assert.Nil(t, err)
+		assert.Equal(t, "kube-config-secret", pod.Spec.Volumes[1].Name)
+		assert.Equal(t, "foo", pod.Spec.Volumes[1].VolumeSource.Secret.SecretName)
+
+		// kubeconfig volume is the last one
+		idx := len(pod.Spec.Containers[1].VolumeMounts) - 1
+		assert.Equal(t, "kube-config-secret", pod.Spec.Containers[1].VolumeMounts[idx].Name)
+		assert.Equal(t, "/some/path/config", pod.Spec.Containers[1].VolumeMounts[idx].MountPath)
+		assert.Equal(t, "--kubeconfig=/some/path/config", pod.Spec.Containers[1].Args[1])
+	}
+}