ci: Run all executors on CI (#4957)

Signed-off-by: Alex Collins <alex_collins@intuit.com>

.github/workflows/ci-build.yaml

@@ -42,31 +42,38 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # kubelet is not included because it'd take ages to get it working methinks
-        test: [ "smoke", "test-e2e", "test-cli", "test-e2e-cron" ]
-        containerRuntimeExecutor: [ "docker", "k8sapi", "pns" ]
-        alwaysOffloadNodeStatus: ["true", "false"]
-        exclude:
-          # only use alwaysOffloadNodeStatus=true for test-e2e
+        # this list can be used to run certain interesting suites of tests
+        test:
+          - smoke
+          - test-cli
+          - test-e2e
+        # `docker` is omitted from this list because we want to run the most thorough set of tests on it,
+        # but always offloading, all persistence related tests, and cron tests
+        containerRuntimeExecutor:
+          - k8sapi
+          - kubelet
+          - pns
+        profile: [ minimal ]
+        alwaysOffloadNodeStatus: [ "false" ]
+        include:
+          # run all test with docker - this is the most complete set of tests
           - test: smoke
+            profile: mysql
+            containerRuntimeExecutor: docker
             alwaysOffloadNodeStatus: "true"
           - test: test-cli
+            profile: mysql
+            containerRuntimeExecutor: docker
             alwaysOffloadNodeStatus: "true"
-          - test: test-e2e-cron
-            alwaysOffloadNodeStatus: "true"
-          # only containerRuntimeExecutor=docker for smoke and test-e2e
           - test: test-e2e
-            containerRuntimeExecutor: k8sapi
-          - test: test-e2e
-            containerRuntimeExecutor: pns
-          - test: test-cli
-            containerRuntimeExecutor: k8sapi
-          - test: test-cli
-            containerRuntimeExecutor: pns
-          - test: test-e2e-cron
-            containerRuntimeExecutor: k8sapi
+            profile: mysql
+            containerRuntimeExecutor: docker
+            alwaysOffloadNodeStatus: "true"
+          # test cron workflows
           - test: test-e2e-cron
-            containerRuntimeExecutor: pns
+            profile: minimal
+            containerRuntimeExecutor: docker
+            alwaysOffloadNodeStatus: "false"
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -100,19 +107,15 @@ jobs:
       - name: Start Argo
         env:
           GOPATH: /home/runner/go
-          PROFILE: mysql
         run: |
           echo '127.0.0.1 dex'      | sudo tee -a /etc/hosts
           echo '127.0.0.1 minio'    | sudo tee -a /etc/hosts
           echo '127.0.0.1 postgres' | sudo tee -a /etc/hosts
           echo '127.0.0.1 mysql'    | sudo tee -a /etc/hosts
           mkdir -p /tmp/log/argo-e2e
           git fetch --tags
-          if [ ${{matrix.test}} == smoke ] || [ ${{matrix.test}} == test-e2e-cron ] ; then
-            PROFILE=minimal
-          fi
-          KUBECONFIG=~/.kube/config make install PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=${{matrix.alwaysOffloadNodeStatus}} DEV_IMAGE=true STATIC_FILES=false
-          KUBECONFIG=~/.kube/config make start   PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=${{matrix.alwaysOffloadNodeStatus}} DEV_IMAGE=true STATIC_FILES=false 2>&1 > /tmp/log/argo-e2e/argo.log &
+          KUBECONFIG=~/.kube/config make install PROFILE=${{matrix.profile}} E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=${{matrix.alwaysOffloadNodeStatus}} DEV_IMAGE=true STATIC_FILES=false
+          KUBECONFIG=~/.kube/config make start   PROFILE=${{matrix.profile}} E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=${{matrix.alwaysOffloadNodeStatus}} DEV_IMAGE=true STATIC_FILES=false 2>&1 > /tmp/log/argo-e2e/argo.log &
       - name: Install gotestsum
         run: go install gotest.tools/gotestsum
       - name: Wait for Argo Server to be ready

Makefile

@@ -501,7 +501,9 @@ test-e2e:
 
 .PHONY: test-cli
 test-cli:
-	$(GOTEST) -timeout 15m -count 1 --tags cli -p 1 ./test/e2e
+	E2E_MODE=GRPC  $(GOTEST) -timeout 15m -count 1 --tags cli -p 1 ./test/e2e
+	E2E_MODE=HTTP1 $(GOTEST) -timeout 15m -count 1 --tags cli -p 1 ./test/e2e
+	E2E_MODE=KUBE  $(GOTEST) -timeout 15m -count 1 --tags cli -p 1 ./test/e2e
 
 .PHONY: test-e2e-cron
 test-e2e-cron:

manifests/quick-start-minimal.yaml

@@ -464,6 +464,18 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubelet-executor
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes/proxy
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: argo-binding
@@ -538,6 +550,19 @@ subjects:
   name: argo-server
   namespace: argo
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubelet-executor-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubelet-executor
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: argo
+---
 apiVersion: v1
 data:
   default-v1: |

manifests/quick-start-mysql.yaml

@@ -464,6 +464,18 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubelet-executor
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes/proxy
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: argo-binding
@@ -538,6 +550,19 @@ subjects:
   name: argo-server
   namespace: argo
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubelet-executor-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubelet-executor
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: argo
+---
 apiVersion: v1
 data:
   default-v1: |

manifests/quick-start-postgres.yaml

@@ -464,6 +464,18 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubelet-executor
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes/proxy
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: argo-binding
@@ -538,6 +550,19 @@ subjects:
   name: argo-server
   namespace: argo
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubelet-executor-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubelet-executor
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: argo
+---
 apiVersion: v1
 data:
   default-v1: |

manifests/quick-start/base/kubelet-executor-clusterrole.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubelet-executor
+rules:
+  # This allows the kubelet executor.
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/proxy
+    verbs:
+      - get

manifests/quick-start/base/kubelet-executor-default-clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubelet-executor-default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubelet-executor
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: argo

manifests/quick-start/base/kustomization.yaml

@@ -7,6 +7,8 @@ resources:
   - webhooks
   - argo-server-sso-secret.yaml
   - workflow-role.yaml
+  - kubelet-executor-clusterrole.yaml
+  - kubelet-executor-default-clusterrolebinding.yaml
   - workflow-default-rolebinding.yaml
   - cluster-workflow-template-rbac.yaml
   - artifact-repositories-configmap.yaml

test/e2e/argo_server_test.go

@@ -93,6 +93,7 @@ func (s *ArgoServerSuite) TestVersion() {
 }
 
 func (s *ArgoServerSuite) TestMetrics() {
+	s.Need(fixtures.CI)
 	s.e().GET("/metrics").
 		Expect().
 		Status(200).
@@ -332,6 +333,7 @@ func (s *ArgoServerSuite) TestOauth() {
 }
 
 func (s *ArgoServerSuite) TestUnauthorized() {
+	s.Need(fixtures.RBAC)
 	token := s.bearerToken
 	defer func() { s.bearerToken = token }()
 	s.bearerToken = "test-token"
@@ -490,6 +492,7 @@ func (s *ArgoServerSuite) TestPermission() {
 	// Test creating workflow with bad token
 	s.bearerToken = badToken
 	s.Run("CreateWFBadToken", func() {
+		s.Need(fixtures.RBAC)
 		s.e().POST("/api/v1/workflows/" + nsName).
 			WithBytes([]byte(`{
   "workflow": {
@@ -522,6 +525,7 @@ func (s *ArgoServerSuite) TestPermission() {
 	// Test list workflows with bad token
 	s.bearerToken = badToken
 	s.Run("ListWFsBadToken", func() {
+		s.Need(fixtures.RBAC)
 		s.e().GET("/api/v1/workflows/" + nsName).
 			Expect().
 			Status(403)
@@ -538,6 +542,7 @@ func (s *ArgoServerSuite) TestPermission() {
 		// Test delete workflow with bad token
 		s.bearerToken = badToken
 		s.Run("DeleteWFWithBadToken", func() {
+			s.Need(fixtures.RBAC)
 			s.e().DELETE("/api/v1/workflows/" + nsName + "/test-wf-good").
 				Expect().
 				Status(403)
@@ -546,6 +551,7 @@ func (s *ArgoServerSuite) TestPermission() {
 		// Test delete workflow with good token
 		s.bearerToken = goodToken
 		s.Run("DeleteWFWithGoodToken", func() {
+			s.Need(fixtures.RBAC)
 			s.e().DELETE("/api/v1/workflows/" + nsName + "/test-wf-good").
 				Expect().
 				Status(200)
@@ -570,6 +576,7 @@ func (s *ArgoServerSuite) TestPermission() {
 
 			s.bearerToken = badToken
 			s.Run("ListArchivedWFsBadToken", func() {
+				s.Need(fixtures.RBAC)
 				s.e().GET("/api/v1/archived-workflows").
 					WithQuery("listOptions.labelSelector", "argo-e2e").
 					WithQuery("listOptions.fieldSelector", "metadata.namespace="+nsName).
@@ -589,6 +596,7 @@ func (s *ArgoServerSuite) TestPermission() {
 			// Test get archived wf with bad token
 			s.bearerToken = badToken
 			s.Run("GetArchivedWFsBadToken", func() {
+				s.Need(fixtures.RBAC)
 				s.e().GET("/api/v1/archived-workflows/" + uid).
 					Expect().
 					Status(403)
@@ -597,13 +605,15 @@ func (s *ArgoServerSuite) TestPermission() {
 			// Test deleting archived wf with bad token
 			s.bearerToken = badToken
 			s.Run("DeleteArchivedWFsBadToken", func() {
+				s.Need(fixtures.RBAC)
 				s.e().DELETE("/api/v1/archived-workflows/" + uid).
 					Expect().
 					Status(403)
 			})
 			// Test deleting archived wf with good token
 			s.bearerToken = goodToken
 			s.Run("DeleteArchivedWFsGoodToken", func() {
+				s.Need(fixtures.RBAC)
 				s.e().DELETE("/api/v1/archived-workflows/" + uid).
 					Expect().
 					Status(200)
@@ -849,6 +859,7 @@ func (s *ArgoServerSuite) TestWorkflowService() {
 	})
 
 	s.Run("Terminate", func() {
+		s.Need(fixtures.None(fixtures.K8SAPI, fixtures.Kubelet))
 		s.e().PUT("/api/v1/workflows/argo/" + name + "/terminate").
 			Expect().
 			Status(200)
@@ -867,6 +878,7 @@ func (s *ArgoServerSuite) TestWorkflowService() {
 	})
 
 	s.Run("Resubmit", func() {
+		s.Need(fixtures.BaseLayerArtifacts)
 		s.e().PUT("/api/v1/workflows/argo/" + name + "/resubmit").
 			WithBytes([]byte(`{"memoized": true}`)).
 			Expect().
@@ -1038,9 +1050,7 @@ spec:
 
 // make sure we can download an artifact
 func (s *ArgoServerSuite) TestArtifactServer() {
-	if !s.Persistence.IsEnabled() {
-		s.T().SkipNow()
-	}
+	s.Need(fixtures.WorkflowArchive)
 	var uid types.UID
 	var name string
 	s.Given().
@@ -1176,9 +1186,7 @@ func (s *ArgoServerSuite) TestWorkflowServiceStream() {
 }
 
 func (s *ArgoServerSuite) TestArchivedWorkflowService() {
-	if !s.Persistence.IsEnabled() {
-		s.T().SkipNow()
-	}
+	s.Need(fixtures.WorkflowArchive)
 	var uid types.UID
 	s.Given().
 		Workflow(`