Skip to content

Commit

Permalink
[e2e] Support dual-stack - utils
Browse files Browse the repository at this point in the history 
* Support dual-stack in e2e utils functions and methods
* Adjust related e2e test code
* Define some dual-stack Service annotations

Signed-off-by: Zhecheng Li <zhechengli@microsoft.com>
  • Loading branch information
@lzhecheng
lzhecheng committed Jan 31, 2023
1 parent 325a40e commit e97a042
Show file tree
Hide file tree
Showing 12 changed files with 482 additions and 211 deletions.
12 changes: 10 additions & 2 deletions pkg/consts/consts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,12 +201,18 @@ var (
false: "service.beta.kubernetes.io/azure-load-balancer-ipv4",
true: "service.beta.kubernetes.io/azure-load-balancer-ipv6",
}
ServiceAnnotationPIPNameDualStack = map[bool]string{
false: "service.beta.kubernetes.io/azure-pip-name-ipv4",
true: "service.beta.kubernetes.io/azure-pip-name-ipv6",
}
ServiceAnnotationPIPPrefixIDDualStack = map[bool]string{
false: "service.beta.kubernetes.io/azure-pip-prefix-id-ipv4",
true: "service.beta.kubernetes.io/azure-pip-prefix-id-ipv6",
}
)

// load balancer
const (
// PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured
PreConfiguredBackendPoolLoadBalancerTypesNone = ""
// PreConfiguredBackendPoolLoadBalancerTypesInternal means that the `internal` load balancers are pre-configured
PreConfiguredBackendPoolLoadBalancerTypesInternal = "internal"
// PreConfiguredBackendPoolLoadBalancerTypesExternal means that the `external` load balancers are pre-configured
Expand Down Expand Up @@ -352,6 +358,8 @@ const (
FrontendIPConfigNameMaxLength = 80
// LoadBalancerRuleNameMaxLength is the max length of the load balancing rule
LoadBalancerRuleNameMaxLength = 80
// IPFamilySuffixLength is the length of suffix length of IP family ("-IPv4", "-IPv6")
IPFamilySuffixLength = 5

// LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration is the lb backend pool config type node IP configuration
LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration = "nodeIPConfiguration"
Expand Down
153 changes: 82 additions & 71 deletions tests/e2e/network/ensureloadbalancer.go
View file

Large diffs are not rendered by default.

27 changes: 16 additions & 11 deletions tests/e2e/network/network_security_group.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,14 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(

It("should add the rule when expose a service", func() {
By("Creating a service and expose it")
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, map[string]string{}, ports)
defer func() {
By("Cleaning up")
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]

By("Validating ip exists in Security Group")
port := fmt.Sprintf("%d", serverPort)
Expand Down Expand Up @@ -140,15 +142,15 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(
annotation := map[string]string{
consts.ServiceAnnotationSharedSecurityRule: "true",
}
ip1 := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips1 := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)

defer func() {
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()

serviceName2 := serviceName + "-share"
ip2 := createAndExposeDefaultServiceWithAnnotation(cs, serviceName2, ns.Name, labels, annotation, ports)
ips2 := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName2, ns.Name, labels, annotation, ports)
defer func() {
By("Cleaning up")
err := utils.DeleteService(cs, ns.Name, serviceName2)
Expand All @@ -160,12 +162,12 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(
nsgs, err := tc.GetClusterSecurityGroups()
Expect(err).NotTo(HaveOccurred())

ipList := []string{ip1, ip2}
ipList := append(ips1, ips2...)
Expect(validateSharedSecurityRuleExists(nsgs, ipList, port)).To(BeTrue(), "Security rule for service %s not exists", serviceName)

By("Validate automatically adjust or delete the rule, when service is deleted")
Expect(utils.DeleteService(cs, ns.Name, serviceName)).NotTo(HaveOccurred())
ipList = []string{ip2}
ipList = ips2
Expect(validateSharedSecurityRuleExists(nsgs, ipList, port)).To(BeTrue(), "Security rule should be modified to only contain service %s", serviceName2)

Expect(utils.DeleteService(cs, ns.Name, serviceName2)).NotTo(HaveOccurred())
Expand Down Expand Up @@ -195,7 +197,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(
utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name)

By("Waiting for the service to be exposed")
_, err = utils.WaitServiceExposure(cs, ns.Name, serviceName, "")
_, err = utils.WaitServiceExposure(cs, ns.Name, serviceName, []string{})
Expect(err).NotTo(HaveOccurred())

By("Validating if the corresponding IP prefix existing in nsg")
Expand Down Expand Up @@ -232,8 +234,10 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(
Expect(err).NotTo(HaveOccurred())

By("Waiting for the service to expose")
internalIP, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "")
internalIPs, err := utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, []string{})
Expect(err).NotTo(HaveOccurred())
Expect(len(internalIPs)).NotTo(BeZero())
internalIP := internalIPs[0]

By("Checking if there is a deny_all rule")
nsgs, err := tc.GetClusterSecurityGroups()
Expand Down Expand Up @@ -270,7 +274,9 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(

// Check Service connectivity with the deny-all-except-lb-range ExecAgnhostPod
By("Waiting for the service to expose")
internalIP, err = utils.WaitServiceExposureAndGetIP(cs, ns.Name, serviceName)
internalIPs, err = utils.WaitServiceExposureAndGetIPs(cs, ns.Name, serviceName)
Expect(len(internalIPs)).NotTo(BeZero())
internalIP = internalIPs[0]
for _, port := range service.Spec.Ports {
utils.Logf("checking the connectivity of addr %s:%d with protocol %v", internalIP, int(port.Port), port.Protocol)
err := utils.ValidateServiceConnectivity(ns.Name, agnhostPod, internalIP, int(port.Port), port.Protocol)
Expand Down Expand Up @@ -302,12 +308,11 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(
consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true",
}
service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports)
service = updateServiceLBIP(service, false, targetIP)
service = updateServiceLBIPs(service, false, []string{targetIP})
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "")
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, []string{targetIP})
Expect(err).NotTo(HaveOccurred())
Expect(ip).To(Equal(targetIP))

defer func() {
By("cleaning up")
Expand Down
4 changes: 3 additions & 1 deletion tests/e2e/network/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,9 @@ var _ = Describe("Azure nodes", func() {
Expect(ok).To(BeTrue())
Expect(nodeRG).NotTo(Equal(rgMaster))

publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, map[string]string{}, ports)
publicIPs := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, map[string]string{}, ports)
Expect(len(publicIPs)).NotTo(BeZero())
publicIP := publicIPs[0]
lb := getAzureLoadBalancerFromPIP(tc, publicIP, rgMaster, rgMaster)

utils.Logf("finding NIC of the node %s, assuming it's in the same rg as master", nodeNotInRGMaster.Name)
Expand Down
60 changes: 42 additions & 18 deletions tests/e2e/network/private_link_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,14 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

// get pls from azure client
Expand All @@ -126,12 +128,14 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

// get pls from azure client
Expand Down Expand Up @@ -161,12 +165,14 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

// get pls from azure client
Expand All @@ -185,12 +191,14 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

// get pls from azure client
Expand All @@ -206,18 +214,22 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

selectedip, err := utils.SelectAvailablePrivateIP(tc)
selectedIPs, err := utils.SelectAvailablePrivateIPs(tc)
Expect(err).NotTo(HaveOccurred())
annotation[consts.ServiceAnnotationPLSIpConfigurationIPAddress] = selectedip
utils.Logf("Now update private link service's static ip to %s", selectedip)
Expect(len(selectedIPs)).NotTo(BeZero())
selectedIP := selectedIPs[0]
annotation[consts.ServiceAnnotationPLSIpConfigurationIPAddress] = selectedIP
utils.Logf("Now update private link service's static ip to %s", selectedIP)

service, err := cs.CoreV1().Services(ns.Name).Get(context.TODO(), serviceName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expand All @@ -226,16 +238,18 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
Expect(err).NotTo(HaveOccurred())

ip, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "")
ips, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, serviceName, []string{})
Expect(err).NotTo(HaveOccurred())
Expect(len(ips)).NotTo(BeZero())
ip = ips[0]

// wait and check pls is updated also
err = wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) {
pls := getPrivateLinkServiceFromIP(tc, ip, "", "", "")
return pls.IPConfigurations != nil &&
len(*pls.IPConfigurations) == 1 &&
(*pls.IPConfigurations)[0].PrivateIPAllocationMethod == network.Static &&
*(*pls.IPConfigurations)[0].PrivateIPAddress == selectedip, nil
*(*pls.IPConfigurations)[0].PrivateIPAddress == selectedIP, nil
})
Expect(err).NotTo(HaveOccurred())
})
Expand All @@ -253,12 +267,14 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

// get pls from azure client
Expand All @@ -279,12 +295,14 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

// get pls from azure client
Expand All @@ -306,12 +324,14 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

// get pls from azure client
Expand Down Expand Up @@ -339,12 +359,14 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
}

// create service with given annotation and wait it to expose
ip := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, serviceName, ns.Name, labels, annotation, ports)
defer func() {
utils.Logf("cleaning up test service %s", serviceName)
err := utils.DeleteService(cs, ns.Name, serviceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Get Internal IP: %s", ip)

// get pls from azure client
Expand All @@ -366,11 +388,13 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
consts.ServiceAnnotationPLSIpConfigurationIPAddressCount: strconv.Itoa(ipAddrCount),
}
svc1 := "service1"
ip := createAndExposeDefaultServiceWithAnnotation(cs, svc1, ns.Name, labels, annotation, ports)
ips := createAndExposeDefaultServiceWithAnnotation(cs, tc.IPFamily, svc1, ns.Name, labels, annotation, ports)
defer func() {
err := utils.DeleteService(cs, ns.Name, svc1)
Expect(err).NotTo(HaveOccurred())
}()
Expect(len(ips)).NotTo(BeZero())
ip := ips[0]
utils.Logf("Successfully created %s in namespace %s with IP %s", svc1, ns.Name, ip)

deployName0 := "pls-deploy0"
Expand Down Expand Up @@ -398,12 +422,12 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
err = utils.DeleteService(cs, ns.Name, svc2)
Expect(err).NotTo(HaveOccurred())
}()
service2 = updateServiceLBIP(service2, true, ip)
service2 = updateServiceLBIPs(service2, true, ips)
_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, svc2, ip)
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, tc.IPFamily, ns.Name, svc2, ips)
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created %s in namespace %s with IP %s", svc2, ns.Name, ip)
utils.Logf("Successfully created %s in namespace %s with IPs %q", svc2, ns.Name, ips)

// get pls from azure client
pls := getPrivateLinkServiceFromIP(tc, ip, "", "", "")
Expand Down
Loading

0 comments on commit e97a042

Please sign in to comment.