diff --git a/docs/workflow-controller-configmap.yaml b/docs/workflow-controller-configmap.yaml index b5d9c7b31471..e57d83c70afa 100644 --- a/docs/workflow-controller-configmap.yaml +++ b/docs/workflow-controller-configmap.yaml @@ -22,6 +22,19 @@ data: # (available since Argo v2.3) parallelism: 10 + # uncomment flowing lines if workflow controller runs in a different k8s cluster with the + # workflow workloads, or needs to communicate with the k8s apiserver using an out-of-cluster + # kubeconfig secret + # kubeConfig: + # # name of the kubeconfig secret, may not be empty when kubeConfig specified + # secretName: kubeconfig-secret + # # key of the kubeconfig secret, may not be empty when kubeConfig specified + # secretKey: kubeconfig + # # mounting path of the kubeconfig secret, default to /kube/config + # mountPath: /kubeconfig/mount/path + # # volume name when mounting the secret, default to kubeconfig + # volumeName: kube-config-volume + # artifactRepository defines the default location to be used as the artifact repository for # container artifacts. artifactRepository: diff --git a/workflow/common/common.go b/workflow/common/common.go index 7b326186464e..d228d9c0f835 100644 --- a/workflow/common/common.go +++ b/workflow/common/common.go @@ -110,6 +110,9 @@ const ( GlobalVarWorkflowCreationTimestamp = "workflow.creationTimestamp" // LocalVarPodName is a step level variable that references the name of the pod LocalVarPodName = "pod.name" + + KubeConfigDefaultMountPath = "/kube/config" + KubeConfigDefaultVolumeName = "kubeconfig" ) // ExecutionControl contains execution control parameters for executor to decide how to execute the container diff --git a/workflow/controller/config.go b/workflow/controller/config.go index 31874163b581..7da737d145bf 100644 --- a/workflow/controller/config.go +++ b/workflow/controller/config.go @@ -30,6 +30,9 @@ type WorkflowControllerConfig struct { // ExecutorResources specifies the resource requirements that will be used for the executor sidecar ExecutorResources *apiv1.ResourceRequirements `json:"executorResources,omitempty"` + // KubeConfig specifies a kube config file for the wait & init containers + KubeConfig *KubeConfig `json:"kubeConfig,omitempty"` + // ContainerRuntimeExecutor specifies the container runtime interface to use, default is docker ContainerRuntimeExecutor string `json:"containerRuntimeExecutor,omitempty"` @@ -62,6 +65,21 @@ type WorkflowControllerConfig struct { Parallelism int `json:"parallelism,omitempty"` } +// KubeConfig is used for wait & init sidecar containers to communicate with a k8s apiserver by a outofcluster method, +// it is used when the workflow controller is in a different cluster with the workflow workloads +type KubeConfig struct { + // SecretName of the kubeconfig secret + // may not be empty if kuebConfig specified + SecretName string `json:"secretName"` + // SecretKey of the kubeconfig in the secret + // may not be empty if kubeConfig specified + SecretKey string `json:"secretKey"` + // VolumeName of kubeconfig, default to 'kubeconfig' + VolumeName string `json:"volumeName,omitempty"` + // MountPath of the kubeconfig secret, default to '/kube/config' + MountPath string `json:"mountPath,omitempty"` +} + // ArtifactRepository represents a artifact repository in which a controller will store its artifacts type ArtifactRepository struct { // ArchiveLogs enables log archiving diff --git a/workflow/controller/workflowpod.go b/workflow/controller/workflowpod.go index 5b416fa42fb0..9f7081e4daf2 100644 --- a/workflow/controller/workflowpod.go +++ b/workflow/controller/workflowpod.go @@ -247,20 +247,14 @@ func substituteGlobals(pod *apiv1.Pod, globalParams map[string]string) (*apiv1.P } func (woc *wfOperationCtx) newInitContainer(tmpl *wfv1.Template) apiv1.Container { - ctr := woc.newExecContainer(common.InitContainerName, false) - ctr.Command = []string{"argoexec"} - ctr.Args = []string{"init"} - ctr.VolumeMounts = []apiv1.VolumeMount{ - volumeMountPodMetadata, - } + ctr := woc.newExecContainer(common.InitContainerName, false, "init") + ctr.VolumeMounts = append([]apiv1.VolumeMount{volumeMountPodMetadata}, ctr.VolumeMounts...) return *ctr } func (woc *wfOperationCtx) newWaitContainer(tmpl *wfv1.Template) (*apiv1.Container, error) { - ctr := woc.newExecContainer(common.WaitContainerName, false) - ctr.Command = []string{"argoexec"} - ctr.Args = []string{"wait"} - ctr.VolumeMounts = woc.createVolumeMounts() + ctr := woc.newExecContainer(common.WaitContainerName, false, "wait") + ctr.VolumeMounts = append(woc.createVolumeMounts(), ctr.VolumeMounts...) return ctr, nil } @@ -317,6 +311,20 @@ func (woc *wfOperationCtx) createVolumes() []apiv1.Volume { volumes := []apiv1.Volume{ volumePodMetadata, } + if woc.controller.Config.KubeConfig != nil { + name := woc.controller.Config.KubeConfig.VolumeName + if name == "" { + name = common.KubeConfigDefaultVolumeName + } + volumes = append(volumes, apiv1.Volume{ + Name: name, + VolumeSource: apiv1.VolumeSource{ + Secret: &apiv1.SecretVolumeSource{ + SecretName: woc.controller.Config.KubeConfig.SecretName, + }, + }, + }) + } switch woc.controller.Config.ContainerRuntimeExecutor { case common.ContainerRuntimeExecutorKubelet, common.ContainerRuntimeExecutorK8sAPI: return volumes @@ -325,7 +333,7 @@ func (woc *wfOperationCtx) createVolumes() []apiv1.Volume { } } -func (woc *wfOperationCtx) newExecContainer(name string, privileged bool) *apiv1.Container { +func (woc *wfOperationCtx) newExecContainer(name string, privileged bool, subCommand string) *apiv1.Container { exec := apiv1.Container{ Name: name, Image: woc.controller.executorImage(), @@ -334,10 +342,29 @@ func (woc *wfOperationCtx) newExecContainer(name string, privileged bool) *apiv1 SecurityContext: &apiv1.SecurityContext{ Privileged: &privileged, }, + Command: []string{"argoexec"}, + Args: []string{subCommand}, } if woc.controller.Config.ExecutorResources != nil { exec.Resources = *woc.controller.Config.ExecutorResources } + if woc.controller.Config.KubeConfig != nil { + path := woc.controller.Config.KubeConfig.MountPath + if path == "" { + path = common.KubeConfigDefaultMountPath + } + name := woc.controller.Config.KubeConfig.VolumeName + if name == "" { + name = common.KubeConfigDefaultVolumeName + } + exec.VolumeMounts = []apiv1.VolumeMount{{ + Name: name, + MountPath: path, + ReadOnly: true, + SubPath: woc.controller.Config.KubeConfig.SecretKey, + }} + exec.Args = append(exec.Args, "--kubeconfig="+path) + } return &exec } diff --git a/workflow/controller/workflowpod_test.go b/workflow/controller/workflowpod_test.go index c5dfe4b8f121..cc7928bccc1b 100644 --- a/workflow/controller/workflowpod_test.go +++ b/workflow/controller/workflowpod_test.go @@ -266,3 +266,53 @@ func TestVolumeAndVolumeMounts(t *testing.T) { assert.Equal(t, "volume-name", pod.Spec.Containers[0].VolumeMounts[0].Name) } } + +func TestOutOfCluster(t *testing.T) { + // default mount path & volume name + { + woc := newWoc() + woc.controller.Config.KubeConfig = &KubeConfig{ + SecretName: "foo", + SecretKey: "bar", + } + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + + assert.Nil(t, err) + assert.Equal(t, "kubeconfig", pod.Spec.Volumes[1].Name) + assert.Equal(t, "foo", pod.Spec.Volumes[1].VolumeSource.Secret.SecretName) + + // kubeconfig volume is the last one + idx := len(pod.Spec.Containers[1].VolumeMounts) - 1 + assert.Equal(t, "kubeconfig", pod.Spec.Containers[1].VolumeMounts[idx].Name) + assert.Equal(t, "/kube/config", pod.Spec.Containers[1].VolumeMounts[idx].MountPath) + assert.Equal(t, "--kubeconfig=/kube/config", pod.Spec.Containers[1].Args[1]) + } + + // custom mount path & volume name, in case name collision + { + woc := newWoc() + woc.controller.Config.KubeConfig = &KubeConfig{ + SecretName: "foo", + SecretKey: "bar", + MountPath: "/some/path/config", + VolumeName: "kube-config-secret", + } + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + + assert.Nil(t, err) + assert.Equal(t, "kube-config-secret", pod.Spec.Volumes[1].Name) + assert.Equal(t, "foo", pod.Spec.Volumes[1].VolumeSource.Secret.SecretName) + + // kubeconfig volume is the last one + idx := len(pod.Spec.Containers[1].VolumeMounts) - 1 + assert.Equal(t, "kube-config-secret", pod.Spec.Containers[1].VolumeMounts[idx].Name) + assert.Equal(t, "/some/path/config", pod.Spec.Containers[1].VolumeMounts[idx].MountPath) + assert.Equal(t, "--kubeconfig=/some/path/config", pod.Spec.Containers[1].Args[1]) + } +}