Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add e2e test for dual-stack secondary service IPs #96484

Merged
merged 2 commits into from Nov 18, 2020
Merged

add e2e test for dual-stack secondary service IPs #96484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ed694a1
add e2e test for dual-stack secondary service IPs
Nov 11, 2020
ad043f2
e2e dualstack test fixes
Nov 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions test/e2e/framework/network/BUILD
View file
Open in desktop
Expand Up @@ -24,6 +24,7 @@ go_library(
"//test/e2e/framework/ssh:go_default_library",
"//test/utils/image:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
)

Expand Down
54 changes: 45 additions & 9 deletions test/e2e/framework/network/utils.go
View file
Open in desktop
Expand Up @@ -46,6 +46,7 @@ import (
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
imageutils "k8s.io/kubernetes/test/utils/image"
netutils "k8s.io/utils/net"
)

const (
Expand Down Expand Up @@ -99,6 +100,11 @@ func EnableSCTP(config *NetworkingTestConfig) {
config.SCTPEnabled = true
}

// EnableDualStack create Dual Stack services
func EnableDualStack(config *NetworkingTestConfig) {
config.DualStackEnabled = true
}

// UseHostNetwork run the test container with HostNetwork=true.
func UseHostNetwork(config *NetworkingTestConfig) {
config.HostNetwork = true
Expand Down Expand Up @@ -161,6 +167,8 @@ type NetworkingTestConfig struct {
// if the test pods are listening on sctp port. We need this as sctp tests
// are marked as disruptive as they may load the sctp module.
SCTPEnabled bool
// DualStackEnabled enables dual stack on services
DualStackEnabled bool
// EndpointPods are the pods belonging to the Service created by this
// test config. Each invocation of `setup` creates a service with
// 1 pod per node running the netexecImage.
Expand All @@ -173,17 +181,21 @@ type NetworkingTestConfig struct {
// SessionAffinityService is a Service with SessionAffinity=ClientIP
// spanning over all endpointPods.
SessionAffinityService *v1.Service
// ExternalAddrs is a list of external IPs of nodes in the cluster.
ExternalAddr string
// Nodes is a list of nodes in the cluster.
Nodes []v1.Node
// MaxTries is the number of retries tolerated for tests run against
// endpoints and services created by this config.
MaxTries int
// The ClusterIP of the Service reated by this test config.
// The ClusterIP of the Service created by this test config.
ClusterIP string
// External ip of first node for use in nodePort testing.
// The SecondaryClusterIP of the Service created by this test config.
SecondaryClusterIP string
// NodeIP it's an ExternalIP if the node has one,
// or an InternalIP if not, for use in nodePort testing.
NodeIP string
// SecondaryNodeIP it's an ExternalIP of the secondary IP family if the node has one,
// or an InternalIP if not, for usein nodePort testing.
SecondaryNodeIP string
aojea marked this conversation as resolved.
Show resolved Hide resolved
// The http/udp/sctp nodePorts of the Service.
NodeHTTPPort int
NodeUDPPort int
Expand Down Expand Up @@ -649,6 +661,10 @@ func (config *NetworkingTestConfig) createNodePortServiceSpec(svcName string, se
if config.SCTPEnabled {
res.Spec.Ports = append(res.Spec.Ports, v1.ServicePort{Port: ClusterSCTPPort, Name: "sctp", Protocol: v1.ProtocolSCTP, TargetPort: intstr.FromInt(EndpointSCTPPort)})
}
if config.DualStackEnabled {
requireDual := v1.IPFamilyPolicyRequireDualStack
res.Spec.IPFamilyPolicy = &requireDual
}
return res
}

Expand Down Expand Up @@ -733,7 +749,6 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) {
framework.ExpectNoError(framework.WaitForAllNodesSchedulable(config.f.ClientSet, 10*time.Minute))
nodeList, err := e2enode.GetReadySchedulableNodes(config.f.ClientSet)
framework.ExpectNoError(err)
config.ExternalAddr = e2enode.FirstAddress(nodeList, v1.NodeExternalIP)

e2eskipper.SkipUnlessNodeCountIsAtLeast(2)
config.Nodes = nodeList.Items
Expand All @@ -754,11 +769,32 @@ func (config *NetworkingTestConfig) setup(selector map[string]string) {
continue
}
}

// obtain the ClusterIP
config.ClusterIP = config.NodePortService.Spec.ClusterIP
if config.ExternalAddr != "" {
config.NodeIP = config.ExternalAddr
} else {
config.NodeIP = e2enode.FirstAddress(nodeList, v1.NodeInternalIP)
if config.DualStackEnabled {
config.SecondaryClusterIP = config.NodePortService.Spec.ClusterIPs[1]
}

// Obtain the primary IP family of the Cluster based on the first ClusterIP
// TODO: Eventually we should just be getting these from Spec.IPFamilies
// but for now that would only if the feature gate is enabled.
family := v1.IPv4Protocol
secondaryFamily := v1.IPv6Protocol
aojea marked this conversation as resolved.
Show resolved Hide resolved
if netutils.IsIPv6String(config.ClusterIP) {
family = v1.IPv6Protocol
secondaryFamily = v1.IPv4Protocol
}
// Get Node IPs from the cluster, ExternalIPs take precedence
config.NodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, family)
if config.NodeIP == "" {
config.NodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, family)
}
if config.DualStackEnabled {
config.SecondaryNodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeExternalIP, secondaryFamily)
if config.SecondaryNodeIP == "" {
config.SecondaryNodeIP = e2enode.FirstAddressByTypeAndFamily(nodeList, v1.NodeInternalIP, secondaryFamily)
}
}

ginkgo.By("Waiting for NodePort service to expose endpoint")
Expand Down
13 changes: 12 additions & 1 deletion test/e2e/framework/node/resource.go
View file
Open in desktop
Expand Up @@ -20,7 +20,6 @@ import (
"context"
"encoding/json"
"fmt"
netutil "k8s.io/utils/net"
"net"
"strings"
"time"
Expand All @@ -42,6 +41,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
clientretry "k8s.io/client-go/util/retry"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
netutil "k8s.io/utils/net"
)

const (
Expand Down Expand Up @@ -249,6 +249,17 @@ func GetInternalIP(node *v1.Node) (string, error) {
return host, nil
}

// FirstAddressByTypeAndFamily returns the first address that matches the given type and family of the list of nodes
func FirstAddressByTypeAndFamily(nodelist *v1.NodeList, addrType v1.NodeAddressType, family v1.IPFamily) string {
for _, n := range nodelist.Items {
addresses := GetAddressesByTypeAndFamily(&n, addrType, family)
if len(addresses) > 0 {
return addresses[0]
}
}
return ""
}

// GetAddressesByTypeAndFamily returns a list of addresses of the given addressType for the given node
// and filtered by IPFamily
func GetAddressesByTypeAndFamily(node *v1.Node, addressType v1.NodeAddressType, family v1.IPFamily) (ips []string) {
Expand Down
143 changes: 143 additions & 0 deletions test/e2e/network/dual_stack.go
View file
Open in desktop
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"fmt"
"net"
"strings"
"time"

"github.com/onsi/ginkgo"
Expand All @@ -32,6 +33,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
imageutils "k8s.io/kubernetes/test/utils/image"
Expand Down Expand Up @@ -435,6 +437,147 @@ var _ = SIGDescribe("[Feature:IPv6DualStackAlphaFeature] [LinuxOnly]", func() {
})
// TODO (khenidak add slice validation logic, since endpoint controller only operates
// on primary ClusterIP

// Service Granular Checks as in k8s.io/kubernetes/test/e2e/network/networking.go
// but using the secondary IP, so we run the same tests for each ClusterIP family
ginkgo.Describe("Granular Checks: Services Secondary IP Family", func() {

ginkgo.It("should function for pod-Service: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())

ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeHTTPPort))
config.DialFromTestContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
})

ginkgo.It("should function for pod-Service: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())

ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeUDPPort))
config.DialFromTestContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
})

// [Disruptive] because it conflicts with tests that call CheckSCTPModuleLoadedOnNodes
ginkgo.It("should function for pod-Service: sctp [Feature:SCTPConnectivity][Disruptive]", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.EnableSCTP)
ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort))
config.DialFromTestContainer("sctp", config.SecondaryClusterIP, e2enetwork.ClusterSCTPPort, config.MaxTries, 0, config.EndpointHostnames())

ginkgo.By(fmt.Sprintf("dialing(sctp) %v --> %v:%v (nodeIP)", config.TestContainerPod.Name, config.SecondaryNodeIP, config.NodeSCTPPort))
config.DialFromTestContainer("sctp", config.SecondaryNodeIP, config.NodeSCTPPort, config.MaxTries, 0, config.EndpointHostnames())
})

ginkgo.It("should function for node-Service: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork)
ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromNode("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())

ginkgo.By(fmt.Sprintf("dialing(http) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeHTTPPort))
config.DialFromNode("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
})

ginkgo.It("should function for node-Service: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack, e2enetwork.UseHostNetwork)
ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (config.clusterIP)", config.SecondaryNodeIP, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromNode("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())

ginkgo.By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.SecondaryNodeIP, config.SecondaryNodeIP, config.NodeUDPPort))
config.DialFromNode("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
})

ginkgo.It("should function for endpoint-Service: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromEndpointContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())

ginkgo.By(fmt.Sprintf("dialing(http) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeHTTPPort))
config.DialFromEndpointContainer("http", config.SecondaryNodeIP, config.NodeHTTPPort, config.MaxTries, 0, config.EndpointHostnames())
})

ginkgo.It("should function for endpoint-Service: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (config.clusterIP)", config.EndpointPods[0].Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromEndpointContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())

ginkgo.By(fmt.Sprintf("dialing(udp) %v (endpoint) --> %v:%v (nodeIP)", config.EndpointPods[0].Name, config.SecondaryNodeIP, config.NodeUDPPort))
config.DialFromEndpointContainer("udp", config.SecondaryNodeIP, config.NodeUDPPort, config.MaxTries, 0, config.EndpointHostnames())
})

ginkgo.It("should update endpoints: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, config.EndpointHostnames())

config.DeleteNetProxyPod()

ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
config.DialFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames())
})

ginkgo.It("should update endpoints: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, config.EndpointHostnames())

config.DeleteNetProxyPod()

ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
config.DialFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, config.MaxTries, config.EndpointHostnames())
})

// [LinuxOnly]: Windows does not support session affinity.
ginkgo.It("should function for client IP based session affinity: http [LinuxOnly]", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort))

// Check if number of endpoints returned are exactly one.
eps, err := config.GetEndpointsFromTestContainer("http", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterHTTPPort, e2enetwork.SessionAffinityChecks)
if err != nil {
framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err)
}
if len(eps) == 0 {
framework.Failf("Unexpected no endpoints return")
}
if len(eps) > 1 {
framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps)
}
})

// [LinuxOnly]: Windows does not support session affinity.
ginkgo.It("should function for client IP based session affinity: udp [LinuxOnly]", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v", config.TestContainerPod.Name, config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort))

// Check if number of endpoints returned are exactly one.
eps, err := config.GetEndpointsFromTestContainer("udp", config.SessionAffinityService.Spec.ClusterIPs[1], e2enetwork.ClusterUDPPort, e2enetwork.SessionAffinityChecks)
if err != nil {
framework.Failf("ginkgo.Failed to get endpoints from test container, error: %v", err)
}
if len(eps) == 0 {
framework.Failf("Unexpected no endpoints return")
}
if len(eps) > 1 {
framework.Failf("Unexpected endpoints return: %v, expect 1 endpoints", eps)
}
})

ginkgo.It("should be able to handle large requests: http", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort))
message := strings.Repeat("42", 1000)
config.DialEchoFromTestContainer("http", config.SecondaryClusterIP, e2enetwork.ClusterHTTPPort, config.MaxTries, 0, message)
})

ginkgo.It("should be able to handle large requests: udp", func() {
config := e2enetwork.NewNetworkingTestConfig(f, e2enetwork.EnableDualStack)
ginkgo.By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.SecondaryClusterIP, e2enetwork.ClusterUDPPort))
message := "n" + strings.Repeat("o", 1999)
config.DialEchoFromTestContainer("udp", config.SecondaryClusterIP, e2enetwork.ClusterUDPPort, config.MaxTries, 0, message)
})
})
})

func validateNumOfServicePorts(svc *v1.Service, expectedNumOfPorts int) {
Expand Down