Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

test: Speed up K8sServicesTest #11550

Merged
merged 6 commits into from May 20, 2020
Merged

test: Speed up K8sServicesTest #11550

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
36888fa
test: Get node names and ip addrs only once in K8sServices
brb May 15, 2020
e2e9a5c
test: Parallelize testNodePort
brb May 15, 2020
6dca4ce
test: Unroll curl requests in K8sService suite
brb May 15, 2020
ff6451f
test: Move failBind tests to separate test case
brb May 18, 2020
88c3d2a
test: Skip duplicate ClusterIP tests
brb May 18, 2020
9c03240
test: Call GinkgoRecover() from testNodePort() goroutines
brb May 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
256 changes: 127 additions & 129 deletions test/k8sT/Services.go
View file
Open in desktop
Expand Up @@ -20,6 +20,7 @@ import (
"net"
"strconv"
"strings"
"sync"
"time"

. "github.com/cilium/cilium/test/ginkgo-ext"
Expand Down Expand Up @@ -580,164 +581,159 @@ var _ = Describe("K8sServicesTest", func() {

testNodePort := func(bpfNodePort, testSecondaryNodePortIP, testFromOutside bool) {
var (
data v1.Service
secondaryK8s1IPv4, secondaryK8s2IPv4 string
err error
data v1.Service
wg sync.WaitGroup
secondaryK8s1IPv4, secondaryK8s2IPv4 string
localCiliumHostIPv4, remoteCiliumHostIPv4 string
)

if testSecondaryNodePortIP {
secondaryK8s1IPv4, _ = getIPv4Andv6AddrForIface(k8s1NodeName, helpers.SecondaryIface)
secondaryK8s2IPv4, _ = getIPv4Andv6AddrForIface(k8s2NodeName, helpers.SecondaryIface)
}

err := kubectl.Get(helpers.DefaultNamespace, "service test-nodeport").Unmarshal(&data)
err = kubectl.Get(helpers.DefaultNamespace, "service test-nodeport").Unmarshal(&data)
Expect(err).Should(BeNil(), "Can not retrieve service")
httpURL := getHTTPLink(data.Spec.ClusterIP, data.Spec.Ports[0].Port)
tftpURL := getTFTPLink(data.Spec.ClusterIP, data.Spec.Ports[1].Port)
testCurlRequest(testDSClient, httpURL)
testCurlRequest(testDSClient, tftpURL)

// From host via localhost IP
// TODO: IPv6
count := 10
httpURL = getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
// These are going to be tested from pods running in their own net namespaces
testURLsFromPods := []string{
getHTTPLink(data.Spec.ClusterIP, data.Spec.Ports[0].Port),
getTFTPLink(data.Spec.ClusterIP, data.Spec.Ports[1].Port),

httpURL = getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort),
getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
if testFromOutside {
doRequestsFromThirdHost(httpURL, count, false)
doRequestsFromThirdHost(tftpURL, count, false)
}
getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort),
getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
if testFromOutside {
doRequestsFromThirdHost(httpURL, count, false)
doRequestsFromThirdHost(tftpURL, count, false)
getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort),
}
if bpfNodePort {
localCiliumHostIPv4, err = kubectl.GetCiliumHostIPv4(context.TODO(), k8s1NodeName)
Expect(err).Should(BeNil(), "Cannot retrieve k8s1 cilium_host ipv4")
remoteCiliumHostIPv4, err = kubectl.GetCiliumHostIPv4(context.TODO(), k8s2NodeName)
Expect(err).Should(BeNil(), "Cannot retrieve k8s2 cilium_host ipv4")

httpURL = getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
testURLsFromPods = append(testURLsFromPods, []string{
getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort),
getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort),

// From pod via node IPs
httpURL = getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort)
testCurlRequest(testDSClient, tftpURL)
testCurlRequest(testDSClient, httpURL)
getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort)
testCurlRequest(testDSClient, tftpURL)
testCurlRequest(testDSClient, httpURL)
getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort),
getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort)
testCurlRequest(testDSClient, httpURL)
testCurlRequest(testDSClient, tftpURL)
getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort),
}...)

httpURL = getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort)
testCurlRequest(testDSClient, httpURL)
testCurlRequest(testDSClient, tftpURL)
}

if bpfNodePort {
// From host via local cilium_host
localCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s1NodeName)
Expect(err).Should(BeNil(), "Cannot retrieve local cilium_host ipv4")
httpURL = getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
// There are tested from pods running in the host net namespace
testURLsFromHosts := []string{
getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort),
getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort),

// From host via remote cilium_host
remoteCiliumHostIPv4, err := kubectl.GetCiliumHostIPv4(context.TODO(), k8s2NodeName)
Expect(err).Should(BeNil(), "Cannot retrieve remote cilium_host ipv4")
getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort),
getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
getHTTPLink("::ffff:"+k8s1IP, data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:"+k8s1IP, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort),
getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort),

// From pod via loopback (host reachable services)
httpURL = getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort)
testCurlRequestFail(testDSClient, httpURL)
testCurlRequestFail(testDSClient, tftpURL)
getHTTPLink("::ffff:"+k8s2IP, data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:"+k8s2IP, data.Spec.Ports[1].NodePort),
}
if bpfNodePort {
testURLsFromHosts = append(testURLsFromHosts, []string{
getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort),
getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort)
testCurlRequestFail(testDSClient, httpURL)
testCurlRequestFail(testDSClient, tftpURL)
getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort),

// From pod via local cilium_host
httpURL = getHTTPLink(localCiliumHostIPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(localCiliumHostIPv4, data.Spec.Ports[1].NodePort)
testCurlRequest(testDSClient, httpURL)
testCurlRequest(testDSClient, tftpURL)
getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort),
getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort)
testCurlRequest(testDSClient, httpURL)
testCurlRequest(testDSClient, tftpURL)
getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort),
getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort),
}...)
}
if testSecondaryNodePortIP {
secondaryK8s1IPv4, _ = getIPv4Andv6AddrForIface(k8s1NodeName, helpers.SecondaryIface)
secondaryK8s2IPv4, _ = getIPv4Andv6AddrForIface(k8s2NodeName, helpers.SecondaryIface)

// From pod via remote cilium_host
httpURL = getHTTPLink(remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort)
testCurlRequest(testDSClient, httpURL)
testCurlRequest(testDSClient, tftpURL)
testURLsFromHosts = append(testURLsFromHosts, []string{
getHTTPLink(secondaryK8s1IPv4, data.Spec.Ports[0].NodePort),
getTFTPLink(secondaryK8s1IPv4, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:"+remoteCiliumHostIPv4, data.Spec.Ports[1].NodePort)
testCurlRequest(testDSClient, httpURL)
testCurlRequest(testDSClient, tftpURL)
getHTTPLink(secondaryK8s2IPv4, data.Spec.Ports[0].NodePort),
getTFTPLink(secondaryK8s2IPv4, data.Spec.Ports[1].NodePort),
}...)
}

testURLsFromOutside := []string{}
if testFromOutside {
// These are tested from external node which does not run
// cilium-agent (so it's not a subject to bpf_sock)
testURLsFromOutside = []string{
getHTTPLink(k8s1IP, data.Spec.Ports[0].NodePort),
getTFTPLink(k8s1IP, data.Spec.Ports[1].NodePort),

getHTTPLink(k8s2IP, data.Spec.Ports[0].NodePort),
getTFTPLink(k8s2IP, data.Spec.Ports[1].NodePort),
}
if testSecondaryNodePortIP {
httpURL = getHTTPLink(secondaryK8s1IPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(secondaryK8s1IPv4, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s1NodeName)
doRequests(tftpURL, count, k8s1NodeName)
if testFromOutside {
doRequestsFromThirdHost(httpURL, count, false)
doRequestsFromThirdHost(tftpURL, count, false)
}
testURLsFromOutside = append(testURLsFromOutside, []string{
getHTTPLink(secondaryK8s1IPv4, data.Spec.Ports[0].NodePort),
getTFTPLink(secondaryK8s1IPv4, data.Spec.Ports[1].NodePort),

httpURL = getHTTPLink(secondaryK8s2IPv4, data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink(secondaryK8s2IPv4, data.Spec.Ports[1].NodePort)
doRequests(httpURL, count, k8s2NodeName)
doRequests(tftpURL, count, k8s2NodeName)
if testFromOutside {
doRequestsFromThirdHost(httpURL, count, false)
doRequestsFromThirdHost(tftpURL, count, false)
}
getHTTPLink(secondaryK8s2IPv4, data.Spec.Ports[0].NodePort),
getTFTPLink(secondaryK8s2IPv4, data.Spec.Ports[1].NodePort),
}...)
}
}

count := 10
for _, url := range testURLsFromPods {
wg.Add(1)
go func(url string) {
defer wg.Done()
testCurlRequest(testDSClient, url)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you will need to increment ExpectWithOffset offset (first argument) in each helper test function called in goroutines so we get better context information about error if assertion fails.

}(url)
}
for _, url := range testURLsFromHosts {
wg.Add(1)
go func(url string) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above

defer wg.Done()
doRequests(url, count, k8s1NodeName)
}(url)
}
for _, url := range testURLsFromOutside {
wg.Add(1)
go func(url string) {
defer wg.Done()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

doRequestsFromThirdHost(url, count, false)
}(url)
}
// TODO: IPv6

if bpfNodePort {
httpURL := getHTTPLink("127.0.0.1", data.Spec.Ports[0].NodePort)
tftpURL := getTFTPLink("127.0.0.1", data.Spec.Ports[1].NodePort)
testCurlRequestFail(testDSClient, httpURL)
testCurlRequestFail(testDSClient, tftpURL)

httpURL = getHTTPLink("::ffff:127.0.0.1", data.Spec.Ports[0].NodePort)
tftpURL = getTFTPLink("::ffff:127.0.0.1", data.Spec.Ports[1].NodePort)
testCurlRequestFail(testDSClient, httpURL)
testCurlRequestFail(testDSClient, tftpURL)

// Ensure the NodePort cannot be bound from any redirected address
failBind(localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName)
Expand All @@ -752,6 +748,8 @@ var _ = Describe("K8sServicesTest", func() {
failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[0].NodePort, "tcp", k8s1NodeName)
failBind("::ffff:"+localCiliumHostIPv4, data.Spec.Ports[1].NodePort, "udp", k8s1NodeName)
}

wg.Wait()
}

testNodePortExternal := func(checkTCP, checkUDP bool) {
Expand Down