test: Support running DatapathConfiguration tests on single node

Recent refactoring of the manifest deployments has broken the ability
to run these tests on a single note (such as microk8s / minikube).
Restore this ability by adding a new `-cilium.multinode` flag which can
be disabled, and autodisabling it for these two integrations.

Signed-off-by: Joe Stringer <joe@cilium.io>

test/config/config.go

@@ -40,6 +40,9 @@ type CiliumTestConfigType struct {
 	Kubeconfig          string
 	Registry            string
 	Benchmarks          bool
+	// Multinode enables the running of tests that involve more than one
+	// node. If false, some tests will silently skip multinode checks.
+	Multinode bool
 }
 
 // CiliumTestConfig holds the global configuration of commandline flags
@@ -77,4 +80,6 @@ func (c *CiliumTestConfigType) ParseFlags() {
 	flag.StringVar(&c.Registry, "cilium.registry", "", "docker registry hostname for Cilium image")
 	flag.BoolVar(&c.Benchmarks, "cilium.benchmarks", false,
 		"Specifies benchmark tests should be run which may increase test time")
+	flag.BoolVar(&c.Multinode, "cilium.multinode", true,
+		"Enable tests across multiple nodes. If disabled, such tests may silently pass")
 }

test/helpers/manifest.go

@@ -20,6 +20,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cilium/cilium/test/config"
 	ginkgoext "github.com/cilium/cilium/test/ginkgo-ext"
 )
 
@@ -38,6 +39,12 @@ type Manifest struct {
 	// a file that contains any number of Deployments, DaemonSets, ...
 	Filename string
 
+	// Alternate is an alternative file (not path) of the manifest that
+	// takes the place of 'Filename' for single-node testing. It is
+	// otherwise equivalent and must point to a file containing resources
+	// to deploy.
+	Alternate string
+
 	// DaemonSetNames is the list of all daemonset names in the manifest
 	DaemonSetNames []string
 
@@ -61,9 +68,13 @@ type Manifest struct {
 	Singleton bool
 }
 
-// GetFilename resolves the filename for the manifest.
+// GetFilename resolves the filename for the manifest depending on whether the
+// alternate filename is used (ie, single node testing YAMLs)
 func (m Manifest) GetFilename() string {
-	return m.Filename
+	if config.CiliumTestConfig.Multinode {
+		return m.Filename
+	}
+	return m.Alternate
 }
 
 // Deploy deploys the manifest. It will call ginkgoext.Fail() if any aspect of

test/k8sT/DatapathConfiguration.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cilium/cilium/test/config"
 	. "github.com/cilium/cilium/test/ginkgo-ext"
 	"github.com/cilium/cilium/test/helpers"
 
@@ -594,8 +595,13 @@ func fetchPodsWithOffset(kubectl *helpers.Kubectl, namespace, name, filter, host
 	pods, err := kubectl.GetPodNames(namespace, filter)
 	ExpectWithOffset(callOffset, err).Should(BeNil(), "Failure while retrieving pod name for %s", filter)
 	if requireMultiNode {
-		ExpectWithOffset(callOffset, len(pods)).Should(BeNumerically(">", 1),
-			fmt.Sprintf("This test requires at least two %s instances, but only one was found", name))
+		if config.CiliumTestConfig.Multinode {
+			ExpectWithOffset(callOffset, len(pods)).Should(BeNumerically(">", 1),
+				fmt.Sprintf("This test requires at least two %s instances, but only one was found", name))
+		} else {
+			By("Ignoring the requirement for clients on multiple nodes")
+			requireMultiNode = false
+		}
 	}
 
 	// Fetch the json description of one of the pods
@@ -659,6 +665,10 @@ func testPodConnectivityAndReturnIP(kubectl *helpers.Kubectl, requireMultiNode b
 }
 
 func testPodHTTPAcrossNodes(kubectl *helpers.Kubectl, namespace string) bool {
+	if !config.CiliumTestConfig.Multinode {
+		By("Skipping multinode HTTP check")
+		return true
+	}
 	result, _ := testPodHTTP(kubectl, namespace, true, 1)
 	return result
 }
@@ -772,16 +782,17 @@ func testPodNetperf(kubectl *helpers.Kubectl, namespace string, requireMultiNode
 }
 
 func monitorConnectivityAcrossNodes(kubectl *helpers.Kubectl) (monitorRes *helpers.CmdRes, monitorCancel func(), targetIP string) {
-	// For local single-node testing, configure requireMultiNode to "false"
-	// and add the labels "cilium.io/ci-node: k8s1" to the node.
-	requireMultiNode := true
+	requireMultinode := config.CiliumTestConfig.Multinode
+	if !config.CiliumTestConfig.Multinode {
+		By("Performing multinode connectivity check within a single node")
+	}
 
 	ciliumPodK8s1, err := kubectl.GetCiliumPodOnNodeWithLabel(helpers.CiliumNamespace, helpers.K8s1)
 	ExpectWithOffset(1, err).Should(BeNil(), "Cannot get cilium pod on k8s1")
 
 	By(fmt.Sprintf("Launching cilium monitor on %q", ciliumPodK8s1))
 	monitorRes, monitorCancel = kubectl.MonitorStart(helpers.CiliumNamespace, ciliumPodK8s1)
-	result, targetIP := testPodConnectivityAndReturnIP(kubectl, requireMultiNode, 2)
+	result, targetIP := testPodConnectivityAndReturnIP(kubectl, requireMultinode, 2)
 	ExpectWithOffset(1, result).Should(BeTrue(), "Connectivity test between nodes failed")
 
 	return monitorRes, monitorCancel, targetIP

test/k8sT/manifests.go

@@ -23,6 +23,7 @@ var (
 
 	DemoDaemonSet = helpers.Manifest{
 		Filename:        "demo_ds.yaml",
+		Alternate:       "demo_ds_local.yaml",
 		DaemonSetNames:  []string{"testds", "testclient"},
 		DeploymentNames: []string{"test-k8s2"},
 		NumPods:         1,

test/k8sT/manifests/demo_ds_local.yaml

@@ -0,0 +1,233 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: testds
+spec:
+  selector:
+    matchLabels:
+      zgroup: testDS
+  template:
+    metadata:
+      labels:
+        zgroup: testDS
+    spec:
+      containers:
+      - name: web
+        image: docker.io/cilium/echoserver:1.10.1
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 80
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 80
+      - name: udp
+        image: docker.io/cilium/echoserver-udp:v2020.01.30
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 69
+          protocol: UDP
+      terminationGracePeriodSeconds: 0
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: testclient
+spec:
+  selector:
+    matchLabels:
+      zgroup: testDSClient
+  template:
+    metadata:
+      labels:
+        zgroup: testDSClient
+    spec:
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: web
+        image: docker.io/cilium/demo-client:latest
+        imagePullPolicy: IfNotPresent
+        command: [ "sleep" ]
+        args:
+          - "1000h"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-k8s2
+spec:
+  selector:
+    matchLabels:
+      zgroup: test-k8s2
+  template:
+    metadata:
+      labels:
+        zgroup: test-k8s2
+    spec:
+      containers:
+      - name: web
+        image: docker.io/cilium/echoserver:1.10.1
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 80
+          hostPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 80
+      - name: udp
+        image: docker.io/cilium/echoserver-udp:v2020.01.30
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 69
+          hostPort: 6969
+          protocol: UDP
+      terminationGracePeriodSeconds: 0
+      nodeSelector:
+        "cilium.io/ci-node": k8s1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: testds-service
+spec:
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP
+  - name: tftp
+    port: 69
+    targetPort: 69
+    protocol: UDP
+  selector:
+    zgroup: testDS
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-nodeport
+spec:
+  type: NodePort
+  ports:
+  - port: 10080
+    targetPort: 80
+    protocol: TCP
+    name: http
+  - port: 10069
+    targetPort: 69
+    protocol: UDP
+    name: tftp
+  selector:
+    zgroup: testDS
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-affinity
+spec:
+  type: NodePort
+  ports:
+  - port: 10080
+    targetPort: 80
+    protocol: TCP
+    name: http
+  - port: 10069
+    targetPort: 69
+    protocol: UDP
+    name: tftp
+  sessionAffinity: ClientIP
+  selector:
+    zgroup: testDS
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-nodeport-local
+spec:
+  type: NodePort
+  externalTrafficPolicy: Local
+  ports:
+  - port: 10080
+    targetPort: 80
+    protocol: TCP
+    name: http
+  - port: 10069
+    targetPort: 69
+    protocol: UDP
+    name: tftp
+  selector:
+    zgroup: testDS
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-nodeport-local-k8s2
+spec:
+  type: NodePort
+  externalTrafficPolicy: Local
+  ports:
+  - port: 10080
+    targetPort: 80
+    protocol: TCP
+    name: http
+  - port: 10069
+    targetPort: 69
+    protocol: UDP
+    name: tftp
+  selector:
+    zgroup: test-k8s2
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-nodeport-k8s2
+spec:
+  type: NodePort
+  ports:
+  - port: 10080
+    targetPort: 80
+    protocol: TCP
+    name: http
+  - port: 10069
+    targetPort: 69
+    protocol: UDP
+    name: tftp
+  selector:
+    zgroup: test-k8s2
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-lb
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 80
+    targetPort: 80
+    protocol: TCP
+    name: http
+  selector:
+    zgroup: testDS
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-lb-local-k8s2
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  ports:
+  - port: 80
+    targetPort: 80
+    protocol: TCP
+    name: http
+  selector:
+    zgroup: test-k8s2

test/test_suite_test.go

@@ -88,6 +88,15 @@ func TestTest(t *testing.T) {
 	}
 	if integration := helpers.GetCurrentIntegration(); integration != "" {
 		fmt.Printf("Using CNI_INTEGRATION=%q\n", integration)
+
+		switch integration {
+		case helpers.CIIntegrationMicrok8s:
+			fallthrough
+		case helpers.CIIntegrationMinikube:
+			fmt.Printf("Disabling multinode testing")
+			config.CiliumTestConfig.Multinode = false
+		default:
+		}
 	}
 
 	configLogsOutput()