Skip to content

Commit

Permalink
feat(controller): Use different container runtime executors for each …
Browse files Browse the repository at this point in the history
…workflow. Close #4254 (#4998)

Signed-off-by: Alex Collins <alex_collins@intuit.com>
  • Loading branch information
alexec committed Feb 19, 2021
1 parent 590df1d commit 687479f
Show file tree
Hide file tree
Showing 16 changed files with 496 additions and 257 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ install: $(MANIFESTS) $(E2E_MANIFESTS) dist/kustomize
kubectl get ns $(KUBE_NAMESPACE) || kubectl create ns $(KUBE_NAMESPACE)
kubectl config set-context --current --namespace=$(KUBE_NAMESPACE)
@echo "installing PROFILE=$(PROFILE) VERSION=$(VERSION), E2E_EXECUTOR=$(E2E_EXECUTOR)"
dist/kustomize build --load_restrictor=none test/e2e/manifests/$(PROFILE) | sed 's/argoproj\//$(IMAGE_NAMESPACE)\//' | sed 's/:latest/:$(VERSION)/' | sed 's/pns/$(E2E_EXECUTOR)/' | kubectl -n $(KUBE_NAMESPACE) apply -f -
dist/kustomize build --load_restrictor=none test/e2e/manifests/$(PROFILE) | sed 's/argoproj\//$(IMAGE_NAMESPACE)\//' | sed 's/:latest/:$(VERSION)/' | sed 's/containerRuntimeExecutor: docker/containerRuntimeExecutor: $(E2E_EXECUTOR)/' | kubectl -n $(KUBE_NAMESPACE) apply --prune -l app.kubernetes.io/part-of=argo -f -
kubectl -n $(KUBE_NAMESPACE) apply -f test/stress/massive-workflow.yaml
kubectl -n $(KUBE_NAMESPACE) rollout restart deploy workflow-controller
kubectl -n $(KUBE_NAMESPACE) rollout restart deploy argo-server
Expand Down
13 changes: 13 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ type Config struct {
// ContainerRuntimeExecutor specifies the container runtime interface to use, default is docker
ContainerRuntimeExecutor string `json:"containerRuntimeExecutor,omitempty"`

ContainerRuntimeExecutors ContainerRuntimeExecutors `json:"containerRuntimeExecutors,omitempty"`

// KubeletPort is needed when using the kubelet containerRuntimeExecutor, default to 10250
KubeletPort int `json:"kubeletPort,omitempty"`

Expand Down Expand Up @@ -105,6 +107,17 @@ type Config struct {
InitialDelay metav1.Duration `json:"initialDelay,omitempty"`
}

func (c Config) GetContainerRuntimeExecutor(labels labels.Labels) (string, error) {
name, err := c.ContainerRuntimeExecutors.Select(labels)
if err != nil {
return "", err
}
if name != "" {
return name, nil
}
return c.ContainerRuntimeExecutor, nil
}

// PodSpecLogStrategy contains the configuration for logging the pod spec in controller log for debugging purpose
type PodSpecLogStrategy struct {
FailedPod bool `json:"failedPod,omitempty"`
Expand Down
30 changes: 30 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"testing"

"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/utils/pointer"
)

Expand Down Expand Up @@ -71,3 +73,31 @@ func TestDatabaseConfig(t *testing.T) {
assert.Equal(t, "my-host", DatabaseConfig{Host: "my-host"}.GetHostname())
assert.Equal(t, "my-host:1234", DatabaseConfig{Host: "my-host", Port: 1234}.GetHostname())
}

func TestContainerRuntimeExecutor(t *testing.T) {
t.Run("Default", func(t *testing.T) {
c := Config{ContainerRuntimeExecutor: "foo"}
executor, err := c.GetContainerRuntimeExecutor(labels.Set{})
assert.NoError(t, err)
assert.Equal(t, "foo", executor)
})
t.Run("Error", func(t *testing.T) {
c := Config{ContainerRuntimeExecutor: "foo", ContainerRuntimeExecutors: ContainerRuntimeExecutors{
{Name: "bar", Selector: metav1.LabelSelector{
MatchLabels: map[string]string{"!": "!"},
}},
}}
_, err := c.GetContainerRuntimeExecutor(labels.Set{})
assert.Error(t, err)
})
t.Run("NoError", func(t *testing.T) {
c := Config{ContainerRuntimeExecutor: "foo", ContainerRuntimeExecutors: ContainerRuntimeExecutors{
{Name: "bar", Selector: metav1.LabelSelector{
MatchLabels: map[string]string{"baz": "qux"},
}},
}}
executor, err := c.GetContainerRuntimeExecutor(labels.Set(map[string]string{"baz": "qux"}))
assert.NoError(t, err)
assert.Equal(t, "bar", executor)
})
}
36 changes: 36 additions & 0 deletions config/container_runtime_executors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package config

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)

type ContainerRuntimeExecutors []ContainerRuntimeExecutor

// select the correct executor to use
// this may return an empty string of there is not executor found
func (e ContainerRuntimeExecutors) Select(labels labels.Labels) (string, error) {
for _, c := range e {
ok, err := c.Matches(labels)
if err != nil {
return "", err
}
if ok {
return c.Name, nil
}
}
return "", nil
}

type ContainerRuntimeExecutor struct {
Name string `json:"name"`
Selector metav1.LabelSelector `json:"selector"`
}

func (e ContainerRuntimeExecutor) Matches(labels labels.Labels) (bool, error) {
x, err := metav1.LabelSelectorAsSelector(&e.Selector)
if err != nil {
return false, err
}
return x.Matches(labels), nil
}
43 changes: 43 additions & 0 deletions config/container_runtime_executors_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package config

import (
"testing"

"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)

func TestContainerRuntimeExecutors(t *testing.T) {
t.Run("Empty", func(t *testing.T) {
x := ContainerRuntimeExecutors{}
e, err := x.Select(labels.Set{})
assert.NoError(t, err)
assert.Empty(t, e)
})
t.Run("Select", func(t *testing.T) {
x := ContainerRuntimeExecutors{
{
Name: "foo",
Selector: metav1.LabelSelector{
MatchLabels: map[string]string{"bar": ""},
},
},
}
e, err := x.Select(labels.Set(map[string]string{"bar": ""}))
assert.NoError(t, err)
assert.Equal(t, "foo", e)
})
t.Run("Error", func(t *testing.T) {
x := ContainerRuntimeExecutors{
{
Name: "foo",
Selector: metav1.LabelSelector{
MatchLabels: map[string]string{"!": "!"},
},
},
}
_, err := x.Select(labels.Set(map[string]string{"bar": ""}))
assert.Error(t, err)
})
}
33 changes: 33 additions & 0 deletions docs/fields.md
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,8 @@ Workflow is the definition of a workflow resource

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`sidecar-dind.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-dind.yaml)

- [`sidecar-nginx.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-nginx.yaml)
Expand Down Expand Up @@ -571,6 +573,8 @@ WorkflowSpec is the specification of a Workflow.

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`sidecar-dind.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-dind.yaml)

- [`sidecar-nginx.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-nginx.yaml)
Expand Down Expand Up @@ -932,6 +936,8 @@ CronWorkflowSpec is the specification of a CronWorkflow

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`sidecar-dind.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-dind.yaml)

- [`sidecar-nginx.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-nginx.yaml)
Expand Down Expand Up @@ -1251,6 +1257,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate.

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`sidecar-dind.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-dind.yaml)

- [`sidecar-nginx.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-nginx.yaml)
Expand Down Expand Up @@ -1546,6 +1554,13 @@ _No description available_

ExecutorConfig holds configurations of an executor container.

<details>
<summary>Examples with this field (click to open)</summary>
<br>

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)
</details>

### Fields
| Field Name | Field Type | Description |
|:----------:|:----------:|---------------|
Expand Down Expand Up @@ -1888,6 +1903,8 @@ Template is a reusable and composable unit of execution in a workflow

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`sidecar-dind.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-dind.yaml)

- [`sidecar-nginx.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-nginx.yaml)
Expand Down Expand Up @@ -2157,6 +2174,8 @@ Outputs hold parameters, artifacts, and results from a step

- [`pod-spec-from-previous-step.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-spec-from-previous-step.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`suspend-template-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/suspend-template-outputs.yaml)

- [`work-avoidance.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/work-avoidance.yaml)
Expand Down Expand Up @@ -2404,6 +2423,8 @@ Parameter indicate a passed string parameter to a service template with an optio

- [`scripts-python.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/scripts-python.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`step-level-timeout.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/step-level-timeout.yaml)

- [`steps.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/steps.yaml)
Expand Down Expand Up @@ -3463,6 +3484,8 @@ ValueFrom describes a location in which to obtain the value to a parameter

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`suspend-template-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/suspend-template-outputs.yaml)

- [`event-consumer-workfloweventbinding.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml)
Expand Down Expand Up @@ -3550,6 +3573,8 @@ MetricLabel is a single label for a prometheus metric
- [`pod-metadata.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-metadata.yaml)

- [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/resource-delete-with-flags.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)
</details>

### Fields
Expand Down Expand Up @@ -4073,6 +4098,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`sidecar-dind.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-dind.yaml)

- [`sidecar-nginx.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-nginx.yaml)
Expand Down Expand Up @@ -4598,6 +4625,8 @@ A single application container that you want to run within a pod.

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`sidecar-dind.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-dind.yaml)

- [`sidecar-nginx.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-nginx.yaml)
Expand Down Expand Up @@ -5267,6 +5296,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`sidecar-dind.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-dind.yaml)

- [`sidecar-nginx.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/sidecar-nginx.yaml)
Expand Down Expand Up @@ -5763,6 +5794,8 @@ EnvVarSource represents a source for the value of an EnvVar.

- [`secrets.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml)

- [`selected-executor-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/selected-executor-workflow.yaml)

- [`suspend-template-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/suspend-template-outputs.yaml)

- [`event-consumer-workfloweventbinding.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml)
Expand Down
Loading

0 comments on commit 687479f

Please sign in to comment.