From 36888fa74f123d3cbc85739e44225d0c51e95482 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Fri, 15 May 2020 14:13:04 +0200 Subject: [PATCH 1/6] test: Get node names and ip addrs only once in K8sServices They don't change during test runs. Signed-off-by: Martynas Pumputis --- test/k8sT/Services.go | 192 ++++++++++++++++++++---------------------- 1 file changed, 92 insertions(+), 100 deletions(-) diff --git a/test/k8sT/Services.go b/test/k8sT/Services.go index b64751cf6bed..69cc15d5f7ec 100644 --- a/test/k8sT/Services.go +++ b/test/k8sT/Services.go @@ -31,20 +31,29 @@ import ( ) var _ = Describe("K8sServicesTest", func() { + const ( + serviceName = "app1-service" + testDSClient = "zgroup=testDSClient" + testDS = "zgroup=testDS" + testDSK8s2 = "zgroup=test-k8s2" + echoServiceName = "echo" + echoPodLabel = "name=echo" + ) + var ( - kubectl *helpers.Kubectl + kubectl *helpers.Kubectl + ciliumFilename string - ciliumFilename string - serviceName = "app1-service" backgroundCancel context.CancelFunc = func() {} backgroundError error enableBackgroundReport = true - ciliumPodK8s1 string - testDSClient = "zgroup=testDSClient" - testDS = "zgroup=testDS" - testDSK8s2 = "zgroup=test-k8s2" - echoServiceName = "echo" - echoPodLabel = "name=echo" + + k8s1NodeName string + k8s2NodeName string + outsideNodeName string + k8s1IP string + k8s2IP string + outsideIP string ) applyPolicy := func(path string) { @@ -54,15 +63,16 @@ var _ = Describe("K8sServicesTest", func() { } BeforeAll(func() { - var err error - kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger) + k8s1NodeName, k8s1IP = kubectl.GetNodeInfo(helpers.K8s1) + k8s2NodeName, k8s2IP = kubectl.GetNodeInfo(helpers.K8s2) + if helpers.ExistNodeWithoutCilium() { + outsideNodeName, outsideIP = kubectl.GetNodeInfo(helpers.GetNodeWithoutCilium()) + } + ciliumFilename = helpers.TimestampFilename("cilium.yaml") DeployCiliumAndDNS(kubectl, ciliumFilename) - - ciliumPodK8s1, err = kubectl.GetCiliumPodOnNodeWithLabel(helpers.CiliumNamespace, helpers.K8s1) - Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s1") }) AfterFailed(func() { @@ -202,6 +212,8 @@ var _ = Describe("K8sServicesTest", func() { Expect(govalidator.IsIP(clusterIP)).Should(BeTrue(), "ClusterIP is not an IP") By("testing connectivity via cluster IP %s", clusterIP) + ciliumPodK8s1, err := kubectl.GetCiliumPodOnNodeWithLabel(helpers.CiliumNamespace, helpers.K8s1) + Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s1") monitorRes, monitorCancel := kubectl.MonitorStart(helpers.CiliumNamespace, ciliumPodK8s1) defer func() { monitorCancel() @@ -211,13 +223,12 @@ var _ = Describe("K8sServicesTest", func() { httpSVCURL := fmt.Sprintf("http://%s/", clusterIP) tftpSVCURL := fmt.Sprintf("tftp://%s/hello", clusterIP) - k8s1Name, _ := kubectl.GetNodeInfo(helpers.K8s1) - status, err := kubectl.ExecInHostNetNS(context.TODO(), k8s1Name, + status, err := kubectl.ExecInHostNetNS(context.TODO(), k8s1NodeName, helpers.CurlFail(httpSVCURL)) Expect(err).To(BeNil(), "Cannot run curl in host netns") status.ExpectSuccess("cannot curl to service IP from host") - status, err = kubectl.ExecInHostNetNS(context.TODO(), k8s1Name, + status, err = kubectl.ExecInHostNetNS(context.TODO(), k8s1NodeName, helpers.CurlFail(tftpSVCURL)) Expect(err).To(BeNil(), "Cannot run curl in host netns") status.ExpectSuccess("cannot curl to service IP from host") @@ -274,13 +285,12 @@ var _ = Describe("K8sServicesTest", func() { }) It("Checks service on same node", func() { - k8s1Name, _ := kubectl.GetNodeInfo(helpers.K8s1) - status, err := kubectl.ExecInHostNetNS(context.TODO(), k8s1Name, + status, err := kubectl.ExecInHostNetNS(context.TODO(), k8s1NodeName, helpers.CurlFail(`"http://[%s]/"`, demoClusterIPv6)) Expect(err).To(BeNil(), "Cannot run curl in host netns") status.ExpectSuccess("cannot curl to service IP from host") - status, err = kubectl.ExecInHostNetNS(context.TODO(), k8s1Name, + status, err = kubectl.ExecInHostNetNS(context.TODO(), k8s1NodeName, helpers.CurlFail(`"tftp://[%s]/hello"`, demoClusterIPv6)) Expect(err).To(BeNil(), "Cannot run curl in host netns") status.ExpectSuccess("cannot curl to service IP from host") @@ -422,16 +432,15 @@ var _ = Describe("K8sServicesTest", func() { if checkSourceIP { cmd += " | grep client_address=" } - clientNodeName, clientIP := kubectl.GetNodeInfo(helpers.GetNodeWithoutCilium()) - res, err := kubectl.ExecInHostNetNS(context.TODO(), clientNodeName, cmd) + res, err := kubectl.ExecInHostNetNS(context.TODO(), outsideNodeName, cmd) Expect(err).Should(BeNil(), "Cannot exec in k8s3 host netns") ExpectWithOffset(1, res).Should(helpers.CMDSuccess(), "Can not connect to service %q from outside cluster", url) if checkSourceIP { // Parse the IPs to avoid issues with 4-in-6 formats sourceIP := net.ParseIP(strings.TrimSpace(strings.Split(res.GetStdOut(), "=")[1])) - clientIP := net.ParseIP(clientIP) - Expect(sourceIP).To(Equal(clientIP)) + outsideIP := net.ParseIP(outsideIP) + Expect(sourceIP).To(Equal(outsideIP)) } } } @@ -575,12 +584,9 @@ var _ = Describe("K8sServicesTest", func() { secondaryK8s1IPv4, secondaryK8s2IPv4 string ) - k8s1Name, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1) - k8s2Name, k8s2IP := kubectl.GetNodeInfo(helpers.K8s2) - if testSecondaryNodePortIP { - secondaryK8s1IPv4, _ = getIPv4Andv6AddrForIface(k8s1Name, helpers.SecondaryIface) - secondaryK8s2IPv4, _ = getIPv4Andv6AddrForIface(k8s2Name, helpers.SecondaryIface) + secondaryK8s1IPv4, _ = getIPv4Andv6AddrForIface(k8s1NodeName, helpers.SecondaryIface) + secondaryK8s2IPv4, _ = getIPv4Andv6AddrForIface(k8s2NodeName, helpers.SecondaryIface) } err := kubectl.Get(helpers.DefaultNamespace, "service test-nodeport").Unmarshal(&data) @@ -595,18 +601,18 @@ var _ = Describe("K8sServicesTest", func() { count := 10 httpURL = getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) httpURL = getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) httpURL = getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) if testFromOutside { doRequestsFromThirdHost(httpURL, count, false) doRequestsFromThirdHost(tftpURL, count, false) @@ -614,13 +620,13 @@ var _ = Describe("K8sServicesTest", func() { httpURL = getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) httpURL = getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) if testFromOutside { doRequestsFromThirdHost(httpURL, count, false) doRequestsFromThirdHost(tftpURL, count, false) @@ -628,8 +634,8 @@ var _ = Describe("K8sServicesTest", func() { httpURL = getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) // From pod via node IPs httpURL = getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort) @@ -654,31 +660,31 @@ var _ = Describe("K8sServicesTest", func() { if bpfNodePort { // From host via local cilium_host - localCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s1Name) + localCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s1NodeName) Expect(err).Should(BeNil(), "Cannot retrieve local cilium_host ipv4") httpURL = getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) httpURL = getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) // From host via remote cilium_host - remoteCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s2Name) + remoteCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s2NodeName) Expect(err).Should(BeNil(), "Cannot retrieve remote cilium_host ipv4") httpURL = getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) httpURL = getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) // From pod via loopback (host reachable services) httpURL = getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort) @@ -716,8 +722,8 @@ var _ = Describe("K8sServicesTest", func() { if testSecondaryNodePortIP { httpURL = getHTTPLink(secondaryK8s1IPv4, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink(secondaryK8s1IPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) if testFromOutside { doRequestsFromThirdHost(httpURL, count, false) doRequestsFromThirdHost(tftpURL, count, false) @@ -725,8 +731,8 @@ var _ = Describe("K8sServicesTest", func() { httpURL = getHTTPLink(secondaryK8s2IPv4, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink(secondaryK8s2IPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s2Name) - doRequests(tftpURL, count, k8s2Name) + doRequests(httpURL, count, k8s2NodeName) + doRequests(tftpURL, count, k8s2NodeName) if testFromOutside { doRequestsFromThirdHost(httpURL, count, false) doRequestsFromThirdHost(tftpURL, count, false) @@ -734,17 +740,17 @@ var _ = Describe("K8sServicesTest", func() { } // Ensure the NodePort cannot be bound from any redirected address - failBind(localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1Name) - failBind(localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1Name) - failBind("127.0.0.1", data.Spec.Ports[0].NodePort, "tcp", k8s1Name) - failBind("127.0.0.1", data.Spec.Ports[1].NodePort, "udp", k8s1Name) - failBind("", data.Spec.Ports[0].NodePort, "tcp", k8s1Name) - failBind("", data.Spec.Ports[1].NodePort, "udp", k8s1Name) - - failBind("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort, "tcp", k8s1Name) - failBind("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort, "udp", k8s1Name) - failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1Name) - failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1Name) + failBind(localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind(localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + failBind("127.0.0.1", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind("127.0.0.1", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + failBind("", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind("", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + + failBind("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) } } @@ -753,7 +759,6 @@ var _ = Describe("K8sServicesTest", func() { err := kubectl.Get(helpers.DefaultNamespace, "service test-nodeport").Unmarshal(&data) Expect(err).Should(BeNil(), "Cannot retrieve service") - _, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1) httpURL := getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort) tftpURL := getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort) @@ -794,13 +799,12 @@ var _ = Describe("K8sServicesTest", func() { err = kubectl.Get(helpers.DefaultNamespace, "service test-affinity").Unmarshal(&data) Expect(err).Should(BeNil(), "Cannot retrieve service") - k8s1NodeName, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1) httpURL := getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort) cmd := helpers.CurlFail(httpURL) + " | grep 'Hostname:' " // pod name is in the hostname if fromOutside { - from, _ = kubectl.GetNodeInfo(helpers.GetNodeWithoutCilium()) + from = outsideNodeName } else { pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, testDSClient) ExpectWithOffset(1, err).Should(BeNil(), "cannot retrieve pod names by filter %q", testDSClient) @@ -885,9 +889,6 @@ var _ = Describe("K8sServicesTest", func() { tftpURL string ) - k8s1Name, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1) - k8s2Name, k8s2IP := kubectl.GetNodeInfo(helpers.K8s2) - // Checks requests are not SNATed when externalTrafficPolicy=Local err := kubectl.Get(helpers.DefaultNamespace, "service test-nodeport-local").Unmarshal(&data) Expect(err).Should(BeNil(), "Can not retrieve service") @@ -909,17 +910,17 @@ var _ = Describe("K8sServicesTest", func() { httpURL = getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1Name) - doRequests(httpURL, count, k8s2Name) - doRequests(tftpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s2Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(httpURL, count, k8s2NodeName) + doRequests(tftpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s2NodeName) httpURL = getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort) tftpURL = getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort) - failRequests(httpURL, count, k8s1Name) - failRequests(httpURL, count, k8s2Name) - failRequests(tftpURL, count, k8s1Name) - failRequests(tftpURL, count, k8s2Name) + failRequests(httpURL, count, k8s1NodeName) + failRequests(httpURL, count, k8s2NodeName) + failRequests(tftpURL, count, k8s1NodeName) + failRequests(tftpURL, count, k8s2NodeName) } testHostPort := func() { @@ -928,9 +929,6 @@ var _ = Describe("K8sServicesTest", func() { tftpURL string ) - k8s1Name, _ := kubectl.GetNodeInfo(helpers.K8s1) - k8s2Name, k8s2IP := kubectl.GetNodeInfo(helpers.K8s2) - httpHostPort := int32(8080) tftpHostPort := int32(6969) @@ -953,18 +951,16 @@ var _ = Describe("K8sServicesTest", func() { tftpURL = getTFTPLink(k8s2IP, tftpHostPort) // ... from same node - doRequests(httpURL, count, k8s2Name) - doRequests(tftpURL, count, k8s2Name) + doRequests(httpURL, count, k8s2NodeName) + doRequests(tftpURL, count, k8s2NodeName) // ... from different node - doRequests(httpURL, count, k8s1Name) - doRequests(tftpURL, count, k8s1Name) + doRequests(httpURL, count, k8s1NodeName) + doRequests(tftpURL, count, k8s1NodeName) } testHealthCheckNodePort := func() { var data v1.Service - k8s1Name, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1) - k8s2Name, k8s2IP := kubectl.GetNodeInfo(helpers.K8s2) // Service with HealthCheckNodePort that only has backends on k8s2 err := kubectl.Get(helpers.DefaultNamespace, "service test-lb-local-k8s2").Unmarshal(&data) @@ -974,13 +970,13 @@ var _ = Describe("K8sServicesTest", func() { // Checks that requests to k8s2 return 200 url := getHTTPLink(k8s2IP, data.Spec.HealthCheckNodePort) - doRequestsExpectingHTTPCode(url, count, "200", k8s1Name) - doRequestsExpectingHTTPCode(url, count, "200", k8s2Name) + doRequestsExpectingHTTPCode(url, count, "200", k8s1NodeName) + doRequestsExpectingHTTPCode(url, count, "200", k8s2NodeName) // Checks that requests to k8s1 return 503 Service Unavailable url = getHTTPLink(k8s1IP, data.Spec.HealthCheckNodePort) - doRequestsExpectingHTTPCode(url, count, "503", k8s1Name) - doRequestsExpectingHTTPCode(url, count, "503", k8s2Name) + doRequestsExpectingHTTPCode(url, count, "503", k8s1NodeName) + doRequestsExpectingHTTPCode(url, count, "503", k8s2NodeName) } testIPv4FragmentSupport := func() { @@ -988,8 +984,6 @@ var _ = Describe("K8sServicesTest", func() { data v1.Service srcPort = 12345 ) - k8s1Name, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1) - k8s2Name, k8s2IP := kubectl.GetNodeInfo(helpers.K8s2) kubeProxy := !helpers.RunsWithoutKubeProxy() // Get testDSClient and testDS pods running on k8s1. @@ -1012,9 +1006,9 @@ var _ = Describe("K8sServicesTest", func() { doFragmentedRequest(clientPod, srcPort+4, serverPort, "::ffff:"+k8s2IP, nodePort, kubeProxy) if !kubeProxy { - localCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s1Name) + localCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s1NodeName) Expect(err).Should(BeNil(), "Cannot retrieve local cilium_host ipv4") - remoteCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s2Name) + remoteCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s2NodeName) Expect(err).Should(BeNil(), "Cannot retrieve remote cilium_host ipv4") // From pod via local cilium_host @@ -1175,7 +1169,6 @@ var _ = Describe("K8sServicesTest", func() { SkipItIf(helpers.DoesNotExistNodeWithoutCilium, "Tests GH#10983", func() { var data v1.Service - _, k8s2IP := kubectl.GetNodeInfo(helpers.K8s2) // We need two NodePort services with the same single endpoint, // so thus we choose the "test-nodeport{-local,}-k8s2" svc. @@ -1229,7 +1222,6 @@ var _ = Describe("K8sServicesTest", func() { var data v1.Service err := kubectl.Get(helpers.DefaultNamespace, "service test-nodeport").Unmarshal(&data) Expect(err).Should(BeNil(), "Cannot retrieve service") - _, k8s1IP := kubectl.GetNodeInfo(helpers.K8s1) url := getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort) doRequestsFromThirdHost(url, 10, true) @@ -1672,7 +1664,7 @@ var _ = Describe("K8sServicesTest", func() { By("Checking that policies were correctly imported into Cilium") - ciliumPodK8s1, err = kubectl.GetCiliumPodOnNodeWithLabel(helpers.CiliumNamespace, helpers.K8s1) + ciliumPodK8s1, err := kubectl.GetCiliumPodOnNodeWithLabel(helpers.CiliumNamespace, helpers.K8s1) Expect(err).Should(BeNil(), "Cannot get cilium pod on k8s1") res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPodK8s1, policyCmd) res.ExpectSuccess("Policy %s is not imported", policyCmd) From e2e9a5c0086c1e7cd53ca7b8201cb26ada9e3073 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Fri, 15 May 2020 16:02:13 +0200 Subject: [PATCH 2/6] test: Parallelize testNodePort Regroup the test cases and run them in parallel. On my machine the invocation of testNodePort() went from ~2:50 min to ~1:15 min. Signed-off-by: Martynas Pumputis --- test/k8sT/Services.go | 256 +++++++++++++++++++++--------------------- 1 file changed, 127 insertions(+), 129 deletions(-) diff --git a/test/k8sT/Services.go b/test/k8sT/Services.go index 69cc15d5f7ec..39a1dc045775 100644 --- a/test/k8sT/Services.go +++ b/test/k8sT/Services.go @@ -20,6 +20,7 @@ import ( "net" "strconv" "strings" + "sync" "time" . "github.com/cilium/cilium/test/ginkgo-ext" @@ -580,164 +581,159 @@ var _ = Describe("K8sServicesTest", func() { testNodePort := func(bpfNodePort, testSecondaryNodePortIP, testFromOutside bool) { var ( - data v1.Service - secondaryK8s1IPv4, secondaryK8s2IPv4 string + err error + data v1.Service + wg sync.WaitGroup + secondaryK8s1IPv4, secondaryK8s2IPv4 string + localCiliumHostIPv4, remoteCiliumHostIPv4 string ) - if testSecondaryNodePortIP { - secondaryK8s1IPv4, _ = getIPv4Andv6AddrForIface(k8s1NodeName, helpers.SecondaryIface) - secondaryK8s2IPv4, _ = getIPv4Andv6AddrForIface(k8s2NodeName, helpers.SecondaryIface) - } - - err := kubectl.Get(helpers.DefaultNamespace, "service test-nodeport").Unmarshal(&data) + err = kubectl.Get(helpers.DefaultNamespace, "service test-nodeport").Unmarshal(&data) Expect(err).Should(BeNil(), "Can not retrieve service") - httpURL := getHTTPLink(data.Spec.ClusterIP, data.Spec.Ports[0].Port) - tftpURL := getTFTPLink(data.Spec.ClusterIP, data.Spec.Ports[1].Port) - testCurlRequest(testDSClient, httpURL) - testCurlRequest(testDSClient, tftpURL) - // From host via localhost IP - // TODO: IPv6 - count := 10 - httpURL = getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) + // These are going to be tested from pods running in their own net namespaces + testURLsFromPods := []string{ + getHTTPLink(data.Spec.ClusterIP, data.Spec.Ports[0].Port), + getTFTPLink(data.Spec.ClusterIP, data.Spec.Ports[1].Port), - httpURL = getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) + getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort), + getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) - if testFromOutside { - doRequestsFromThirdHost(httpURL, count, false) - doRequestsFromThirdHost(tftpURL, count, false) - } + getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) + getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort), + getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) - if testFromOutside { - doRequestsFromThirdHost(httpURL, count, false) - doRequestsFromThirdHost(tftpURL, count, false) + getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort), } + if bpfNodePort { + localCiliumHostIPv4, err = kubectl.GetCiliumHostIPv4(context.TODO(), k8s1NodeName) + Expect(err).Should(BeNil(), "Cannot retrieve k8s1 cilium_host ipv4") + remoteCiliumHostIPv4, err = kubectl.GetCiliumHostIPv4(context.TODO(), k8s2NodeName) + Expect(err).Should(BeNil(), "Cannot retrieve k8s2 cilium_host ipv4") - httpURL = getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) + testURLsFromPods = append(testURLsFromPods, []string{ + getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort), + getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort), - // From pod via node IPs - httpURL = getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort) - testCurlRequest(testDSClient, tftpURL) - testCurlRequest(testDSClient, httpURL) + getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort) - testCurlRequest(testDSClient, tftpURL) - testCurlRequest(testDSClient, httpURL) + getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort), + getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort) - testCurlRequest(testDSClient, httpURL) - testCurlRequest(testDSClient, tftpURL) + getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort), + }...) - httpURL = getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort) - testCurlRequest(testDSClient, httpURL) - testCurlRequest(testDSClient, tftpURL) + } - if bpfNodePort { - // From host via local cilium_host - localCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s1NodeName) - Expect(err).Should(BeNil(), "Cannot retrieve local cilium_host ipv4") - httpURL = getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) + // There are tested from pods running in the host net namespace + testURLsFromHosts := []string{ + getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort), + getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) + getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort), - // From host via remote cilium_host - remoteCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s2NodeName) - Expect(err).Should(BeNil(), "Cannot retrieve remote cilium_host ipv4") + getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort), + getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) + getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) + getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort), + getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort), - // From pod via loopback (host reachable services) - httpURL = getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort) - testCurlRequestFail(testDSClient, httpURL) - testCurlRequestFail(testDSClient, tftpURL) + getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort), + } + if bpfNodePort { + testURLsFromHosts = append(testURLsFromHosts, []string{ + getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort), + getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort) - testCurlRequestFail(testDSClient, httpURL) - testCurlRequestFail(testDSClient, tftpURL) + getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort), - // From pod via local cilium_host - httpURL = getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort) - testCurlRequest(testDSClient, httpURL) - testCurlRequest(testDSClient, tftpURL) + getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort), + getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort) - testCurlRequest(testDSClient, httpURL) - testCurlRequest(testDSClient, tftpURL) + getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort), + getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort), + }...) + } + if testSecondaryNodePortIP { + secondaryK8s1IPv4, _ = getIPv4Andv6AddrForIface(k8s1NodeName, helpers.SecondaryIface) + secondaryK8s2IPv4, _ = getIPv4Andv6AddrForIface(k8s2NodeName, helpers.SecondaryIface) - // From pod via remote cilium_host - httpURL = getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort) - testCurlRequest(testDSClient, httpURL) - testCurlRequest(testDSClient, tftpURL) + testURLsFromHosts = append(testURLsFromHosts, []string{ + getHTTPLink(secondaryK8s1IPv4, data.Spec.Ports[0].NodePort), + getTFTPLink(secondaryK8s1IPv4, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort) - testCurlRequest(testDSClient, httpURL) - testCurlRequest(testDSClient, tftpURL) + getHTTPLink(secondaryK8s2IPv4, data.Spec.Ports[0].NodePort), + getTFTPLink(secondaryK8s2IPv4, data.Spec.Ports[1].NodePort), + }...) + } + testURLsFromOutside := []string{} + if testFromOutside { + // These are tested from external node which does not run + // cilium-agent (so it's not a subject to bpf_sock) + testURLsFromOutside = []string{ + getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort), + getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort), + + getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort), + getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort), + } if testSecondaryNodePortIP { - httpURL = getHTTPLink(secondaryK8s1IPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(secondaryK8s1IPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s1NodeName) - doRequests(tftpURL, count, k8s1NodeName) - if testFromOutside { - doRequestsFromThirdHost(httpURL, count, false) - doRequestsFromThirdHost(tftpURL, count, false) - } + testURLsFromOutside = append(testURLsFromOutside, []string{ + getHTTPLink(secondaryK8s1IPv4, data.Spec.Ports[0].NodePort), + getTFTPLink(secondaryK8s1IPv4, data.Spec.Ports[1].NodePort), - httpURL = getHTTPLink(secondaryK8s2IPv4, data.Spec.Ports[0].NodePort) - tftpURL = getTFTPLink(secondaryK8s2IPv4, data.Spec.Ports[1].NodePort) - doRequests(httpURL, count, k8s2NodeName) - doRequests(tftpURL, count, k8s2NodeName) - if testFromOutside { - doRequestsFromThirdHost(httpURL, count, false) - doRequestsFromThirdHost(tftpURL, count, false) - } + getHTTPLink(secondaryK8s2IPv4, data.Spec.Ports[0].NodePort), + getTFTPLink(secondaryK8s2IPv4, data.Spec.Ports[1].NodePort), + }...) } + } + + count := 10 + for _, url := range testURLsFromPods { + wg.Add(1) + go func(url string) { + defer wg.Done() + testCurlRequest(testDSClient, url) + }(url) + } + for _, url := range testURLsFromHosts { + wg.Add(1) + go func(url string) { + defer wg.Done() + doRequests(url, count, k8s1NodeName) + }(url) + } + for _, url := range testURLsFromOutside { + wg.Add(1) + go func(url string) { + defer wg.Done() + doRequestsFromThirdHost(url, count, false) + }(url) + } + // TODO: IPv6 + + if bpfNodePort { + httpURL := getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort) + tftpURL := getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort) + testCurlRequestFail(testDSClient, httpURL) + testCurlRequestFail(testDSClient, tftpURL) + + httpURL = getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort) + tftpURL = getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort) + testCurlRequestFail(testDSClient, httpURL) + testCurlRequestFail(testDSClient, tftpURL) // Ensure the NodePort cannot be bound from any redirected address failBind(localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) @@ -752,6 +748,8 @@ var _ = Describe("K8sServicesTest", func() { failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) } + + wg.Wait() } testNodePortExternal := func(checkTCP, checkUDP bool) { From 6dca4ce1a07c31dc30c4f7b17fc96d518c9673d3 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Fri, 15 May 2020 16:56:09 +0200 Subject: [PATCH 3/6] test: Unroll curl requests in K8sService suite To avoid excessive "kubectl exec", do curl requests in batches. Signed-off-by: Martynas Pumputis --- test/helpers/wrappers.go | 11 +++++++++++ test/k8sT/Services.go | 34 ++++++++++++++-------------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/test/helpers/wrappers.go b/test/helpers/wrappers.go index 55236c2541df..3ca8f1cbbfcd 100644 --- a/test/helpers/wrappers.go +++ b/test/helpers/wrappers.go @@ -73,6 +73,17 @@ func CurlFail(endpoint string, optionalValues ...interface{}) string { CurlConnectTimeout, CurlMaxTimeout, endpoint, statsInfo) } +// CurlFailNoStats does the same as CurlFail() except that it does not print +// the stats info. +func CurlFailNoStats(endpoint string, optionalValues ...interface{}) string { + if len(optionalValues) > 0 { + endpoint = fmt.Sprintf(endpoint, optionalValues...) + } + return fmt.Sprintf( + `curl --path-as-is -s -D /dev/stderr --fail --connect-timeout %[1]d --max-time %[2]d %[3]s`, + CurlConnectTimeout, CurlMaxTimeout, endpoint) +} + // CurlWithHTTPCode retunrs the string representation of the curl command which // only outputs the HTTP code returned by its execution against the specified // endpoint. It takes a variadic optinalValues argument. This is passed on to diff --git a/test/k8sT/Services.go b/test/k8sT/Services.go index 39a1dc045775..d6517b429608 100644 --- a/test/k8sT/Services.go +++ b/test/k8sT/Services.go @@ -136,23 +136,18 @@ var _ = Describe("K8sServicesTest", func() { } testCurlRequest := func(clientPodLabel, url string) { - pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, clientPodLabel) - ExpectWithOffset(1, err).Should(BeNil(), "cannot retrieve pod names by filter %q", testDSClient) // A DS with client is running in each node. So we try from each node // that can connect to the service. To make sure that the cross-node // service connectivity is correct we tried 10 times, so balance in the // two nodes + pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, clientPodLabel) + ExpectWithOffset(1, err).Should(BeNil(), "cannot retrieve pod names by filter %q", testDSClient) + count := 10 + cmd := fmt.Sprintf(`/bin/sh -c 'for i in $(seq 1 %d); do %s; done'`, count, helpers.CurlFailNoStats(url)) for _, pod := range pods { - tries := 10 - By("Making %d curl requests from %q to %q", tries, pod, url) - for i := 1; i <= tries; i++ { - res := kubectl.ExecPodCmd( - helpers.DefaultNamespace, pod, - helpers.CurlFail(url)) - ExpectWithOffset(1, res).Should(helpers.CMDSuccess(), - "Pod %q can not connect to service %q (failed in request %d/%d)", - pod, url, i, tries) - } + By("Making %d curl requests from %s pod to service %s", count, pod, url) + res := kubectl.ExecPodCmd(helpers.DefaultNamespace, pod, cmd) + ExpectWithOffset(1, res).Should(helpers.CMDSuccess(), "Request from %s pod to service %s failed", pod, url) } } @@ -378,14 +373,13 @@ var _ = Describe("K8sServicesTest", func() { } doRequests := func(url string, count int, fromPod string) { - By("Making %d curl requests from %s to %q", count, fromPod, url) - for i := 1; i <= count; i++ { - res, err := kubectl.ExecInHostNetNS(context.TODO(), fromPod, helpers.CurlFail(url)) - ExpectWithOffset(1, err).To(BeNil(), "Cannot run curl in host netns") - ExpectWithOffset(1, res).Should(helpers.CMDSuccess(), - "%s host can not connect to service %q (failed in request %d/%d)", - fromPod, url, i, count) - } + By("Making %d curl requests from pod (host netns) %s to %q", count, fromPod, url) + cmd := fmt.Sprintf(`/bin/sh -c 'for i in $(seq 1 %d); do %s; done'`, count, + helpers.CurlFailNoStats(url)) + res, err := kubectl.ExecInHostNetNS(context.TODO(), fromPod, cmd) + ExpectWithOffset(1, err).To(BeNil(), "Cannot run curl in host netns") + ExpectWithOffset(1, res).Should(helpers.CMDSuccess(), + "Request from %s to service %s failed", fromPod, url) } failRequests := func(url string, count int, fromPod string) { From ff6451f5e3b94fb9726d2131bb2acadb54638e0b Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Mon, 18 May 2020 10:02:14 +0200 Subject: [PATCH 4/6] test: Move failBind tests to separate test case It's enough to test it only once, thus the move. Signed-off-by: Martynas Pumputis --- test/k8sT/Services.go | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/test/k8sT/Services.go b/test/k8sT/Services.go index d6517b429608..de91302cfa47 100644 --- a/test/k8sT/Services.go +++ b/test/k8sT/Services.go @@ -729,23 +729,34 @@ var _ = Describe("K8sServicesTest", func() { testCurlRequestFail(testDSClient, httpURL) testCurlRequestFail(testDSClient, tftpURL) - // Ensure the NodePort cannot be bound from any redirected address - failBind(localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) - failBind(localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) - failBind("127.0.0.1", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) - failBind("127.0.0.1", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) - failBind("", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) - failBind("", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) - - failBind("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) - failBind("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) - failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) - failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) } wg.Wait() } + testFailBind := func() { + var data v1.Service + var localCiliumHostIPv4 string + + err := kubectl.Get(helpers.DefaultNamespace, "service test-nodeport").Unmarshal(&data) + Expect(err).Should(BeNil(), "Can not retrieve service") + localCiliumHostIPv4, err = kubectl.GetCiliumHostIPv4(context.TODO(), k8s1NodeName) + Expect(err).Should(BeNil(), "Cannot retrieve k8s1 cilium_host ipv4") + + // Ensure the NodePort cannot be bound from any redirected address + failBind(localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind(localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + failBind("127.0.0.1", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind("127.0.0.1", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + failBind("", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind("", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + + failBind("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName) + failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1NodeName) + } + testNodePortExternal := func(checkTCP, checkUDP bool) { var data v1.Service @@ -1101,6 +1112,10 @@ var _ = Describe("K8sServicesTest", func() { testHealthCheckNodePort() }) + It("Tests that binding to NodePort port fails", func() { + testFailBind() + }) + It("Tests HostPort", func() { testHostPort() }) @@ -1143,6 +1158,10 @@ var _ = Describe("K8sServicesTest", func() { testHealthCheckNodePort() }) + It("Tests that binding to NodePort port fails", func() { + testFailBind() + }) + It("Tests HostPort", func() { testHostPort() }) From 88c3d2a895a740a416651bed65c4b9742e1aebc7 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Mon, 18 May 2020 10:18:29 +0200 Subject: [PATCH 5/6] test: Skip duplicate ClusterIP tests Test ClusterIP from a pod netns and a hostnetns in testNodePort(), so that some ClusterIP test cases can be skipped. Signed-off-by: Martynas Pumputis --- test/k8sT/Services.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/k8sT/Services.go b/test/k8sT/Services.go index de91302cfa47..35ffa18467a8 100644 --- a/test/k8sT/Services.go +++ b/test/k8sT/Services.go @@ -202,7 +202,7 @@ var _ = Describe("K8sServicesTest", func() { _ = kubectl.Delete(echoSVCYAML) }) - It("Checks service on same node", func() { + SkipItIf(helpers.RunsWithoutKubeProxy, "Checks service on same node", func() { clusterIP, _, err := kubectl.GetServiceHostPort(helpers.DefaultNamespace, serviceName) Expect(err).Should(BeNil(), "Cannot get service %s", serviceName) Expect(govalidator.IsIP(clusterIP)).Should(BeTrue(), "ClusterIP is not an IP") @@ -240,7 +240,7 @@ var _ = Describe("K8sServicesTest", func() { testCurlRequest("id=app2", httpSVCURL) testCurlRequest("id=app2", tftpSVCURL) } - }, 300) + }) It("Checks service accessing itself (hairpin flow)", func() { clusterIP, _, err := kubectl.GetServiceHostPort(helpers.DefaultNamespace, echoServiceName) @@ -321,7 +321,7 @@ var _ = Describe("K8sServicesTest", func() { ExpectAllPodsTerminated(kubectl) }) - It("Checks ClusterIP Connectivity", func() { + SkipItIf(helpers.RunsWithoutKubeProxy, "Checks ClusterIP Connectivity", func() { service := "testds-service" clusterIP, _, err := kubectl.GetServiceHostPort(helpers.DefaultNamespace, service) @@ -626,6 +626,9 @@ var _ = Describe("K8sServicesTest", func() { // There are tested from pods running in the host net namespace testURLsFromHosts := []string{ + getHTTPLink(data.Spec.ClusterIP, data.Spec.Ports[0].Port), + getTFTPLink(data.Spec.ClusterIP, data.Spec.Ports[1].Port), + getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort), getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort), From 9c03240a81e690203de45acf2f8b42be1a4e647a Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Mon, 18 May 2020 14:25:12 +0200 Subject: [PATCH 6/6] test: Call GinkgoRecover() from testNodePort() goroutines Otherwise, in a case of a failure, ginkgo will panic leaving a cryptic message, from which it's harder to determine which test case has failed. Signed-off-by: Martynas Pumputis --- test/k8sT/Services.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/k8sT/Services.go b/test/k8sT/Services.go index 35ffa18467a8..02ca29296226 100644 --- a/test/k8sT/Services.go +++ b/test/k8sT/Services.go @@ -701,6 +701,7 @@ var _ = Describe("K8sServicesTest", func() { for _, url := range testURLsFromPods { wg.Add(1) go func(url string) { + defer GinkgoRecover() defer wg.Done() testCurlRequest(testDSClient, url) }(url) @@ -708,6 +709,7 @@ var _ = Describe("K8sServicesTest", func() { for _, url := range testURLsFromHosts { wg.Add(1) go func(url string) { + defer GinkgoRecover() defer wg.Done() doRequests(url, count, k8s1NodeName) }(url) @@ -715,6 +717,7 @@ var _ = Describe("K8sServicesTest", func() { for _, url := range testURLsFromOutside { wg.Add(1) go func(url string) { + defer GinkgoRecover() defer wg.Done() doRequestsFromThirdHost(url, count, false) }(url)