Skip to content

Commit

Permalink
Merge pull request #2254 from k8s-infra-cherrypick-robot/cherry-pick-…
Browse files Browse the repository at this point in the history 
…2240-to-release-1.1

[release-1.1] Improve flaky e2e tests
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Sep 3, 2022
2 parents 6e4a39e + d0e191a commit 5feb0df
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 125 deletions.
108 changes: 14 additions & 94 deletions tests/e2e/network/ensureloadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,98 +308,6 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
Expect(ip).To(Equal(targetIP))
})

// internal w/o IP -> public w/o IP
// This test is to replace an upstream k/k one because there's a bug:
// https://github.com/kubernetes/kubernetes/blob/373c08e0c7873a76cecde1d6d714cc2ff7af0c9a/test/e2e/network/loadbalancer.go#L574
// https://github.com/kubernetes/kubernetes/pull/109413
It("should support updating an internal Service to a public one", func() {
By("Creating an internal Service")
service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports)
_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
defer func() {
By("Cleaning up")
err = utils.DeleteService(cs, ns.Name, testServiceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service %s in namespace %s", testServiceName, ns.Name)

By("Waiting for exposure of the internal Service")
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
Expect(err).NotTo(HaveOccurred())
list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{})
Expect(errList).NotTo(HaveOccurred())
utils.Logf("Events list:")
for i, event := range list.Items {
utils.Logf("%d. %v", i, event)
}

By("Updating the Service to public")
service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{})
service = updateServiceBalanceIP(service, false, "")

_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
Expect(err).NotTo(HaveOccurred())

By("Expect the Service IP to be a public one")
var targetIP string
err = wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) {
svc, err := cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{})
if err != nil {
return false, err
}
targetIP = svc.Status.LoadBalancer.Ingress[0].IP
if utils.IsInternalEndpoint(targetIP) {
utils.Logf("expected IP is public, current IP is internal, retry in 10 seconds")
return false, nil
}
return true, nil
})
Expect(err).NotTo(HaveOccurred())

_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, targetIP)
Expect(err).NotTo(HaveOccurred())
})

// public w/o IP -> internal w/ IP
// This test is to replace an upstream k/k one because there's a bug:
// https://github.com/kubernetes/kubernetes/blob/373c08e0c7873a76cecde1d6d714cc2ff7af0c9a/test/e2e/network/loadbalancer.go#L574
// https://github.com/kubernetes/kubernetes/pull/109413
It("should support updating a public Service to an internal one with specific IP", func() {
service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports)
_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
defer func() {
By("Cleaning up")
err = utils.DeleteService(cs, ns.Name, testServiceName)
Expect(err).NotTo(HaveOccurred())
}()
Expect(err).NotTo(HaveOccurred())
utils.Logf("Successfully created LoadBalancer service %s in namespace %s", testServiceName, ns.Name)

By("Waiting for exposure of a public Service")
_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
Expect(err).NotTo(HaveOccurred())
list, errList := cs.CoreV1().Events(ns.Name).List(context.TODO(), metav1.ListOptions{})
Expect(errList).NotTo(HaveOccurred())
utils.Logf("Events list:")
for i, event := range list.Items {
utils.Logf("%d. %v", i, event)
}

internalIP, err := utils.SelectAvailablePrivateIP(tc)
Expect(err).NotTo(HaveOccurred())

By("Updating the public service to an internal one with an IP")
utils.Logf("will update IP to %s", internalIP)
service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{})
service = updateServiceBalanceIP(service, true, internalIP)
_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
Expect(err).NotTo(HaveOccurred())

_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, internalIP)
Expect(err).NotTo(HaveOccurred())
})

It("should have no operation since no change in service when update", Label(utils.TestSuiteLabelSlow), func() {
suffix := string(uuid.NewUUID())[0:4]
ipName := basename + "-public-remain" + suffix
Expand Down Expand Up @@ -640,12 +548,24 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
}()

By("testing if floating IP disabled in load balancer rule")
pipFrontendConfigID := getPIPFrontendConfigurationID(tc, ip, tc.GetResourceGroup(), "")
pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/")
Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0))

lb := getAzureLoadBalancerFromPIP(tc, ip, tc.GetResourceGroup(), "")
lbRules := lb.LoadBalancingRules
Expect(len(*lbRules)).To(Equal(len(ports)))
found := false
for _, lbRule := range *lbRules {
Expect(to.Bool(lbRule.EnableFloatingIP)).To(BeFalse())
utils.Logf("Checking LB rule %q, may not be the corresponding rule of the Service", *lbRule.Name)
lbRuleSplit := strings.Split(*lbRule.Name, "-")
Expect(len(lbRuleSplit)).NotTo(Equal(0))
if pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] == lbRuleSplit[0] {
Expect(to.Bool(lbRule.EnableFloatingIP)).To(BeFalse())
found = true
break
}
}
Expect(found).To(Equal(true))
})
})

Expand Down
90 changes: 59 additions & 31 deletions tests/e2e/network/service_annotations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,26 +238,6 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
Expect(*idleTimeout).To(Equal(int32(5)))
})

// It("should support service annotation 'ServiceAnnotationLoadBalancerMixedProtocols'", func() {
// annotation := map[string]string{
// azureprovider.ServiceAnnotationLoadBalancerMixedProtocols: "true",
// }

// // create service with given annotation and wait it to expose
// publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)

// // get lb from azure client
// lb := getAzureLoadBalancer(publicIP)

// existingProtocols := make(map[network.TransportProtocol]int)
// for _, rule := range *lb.LoadBalancingRules {
// if _, ok := existingProtocols[rule.Protocol]; !ok {
// existingProtocols[rule.Protocol]++
// }
// }
// Expect(len(existingProtocols)).To(Equal(2))
// })

It("should support service annotation 'service.beta.kubernetes.io/azure-load-balancer-resource-group'", func() {
By("creating a test resource group")
rg, cleanup := utils.CreateTestResourceGroup(tc)
Expand Down Expand Up @@ -521,18 +501,33 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
err = utils.DeletePIPWithRetry(tc, publicIP, tc.GetResourceGroup())
Expect(err).NotTo(HaveOccurred())
}()
pipFrontendConfigID := getPIPFrontendConfigurationID(tc, publicIP, tc.GetResourceGroup(), "")
pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/")
Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0))

var lb *network.LoadBalancer
var targetProbes []*network.Probe
//wait for backend update
err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
lb = getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "")
return len(*lb.LoadBalancerPropertiesFormat.Probes) == 1, nil
targetProbes = []*network.Probe{}
for i := range *lb.LoadBalancerPropertiesFormat.Probes {
probe := (*lb.LoadBalancerPropertiesFormat.Probes)[i]
utils.Logf("One probe of LB is %q", *probe.Name)
probeSplit := strings.Split(*probe.Name, "-")
Expect(len(probeSplit)).NotTo(Equal(0))
if pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] == probeSplit[0] {
targetProbes = append(targetProbes, &probe)
}
}

return len(targetProbes) == 1, nil
})
Expect(err).NotTo(HaveOccurred())

By("Validating health probe configs")
var numberOfProbes *int32
for _, probe := range *lb.Probes {
for _, probe := range targetProbes {
if probe.NumberOfProbes != nil {
numberOfProbes = probe.NumberOfProbes
}
Expand All @@ -555,19 +550,32 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
err = utils.DeletePIPWithRetry(tc, publicIP, tc.GetResourceGroup())
Expect(err).NotTo(HaveOccurred())
}()
pipFrontendConfigID := getPIPFrontendConfigurationID(tc, publicIP, tc.GetResourceGroup(), "")
pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/")
Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0))

var lb *network.LoadBalancer
var targetProbes []*network.Probe
//wait for backend update
err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
lb = getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "")
return len(*lb.LoadBalancerPropertiesFormat.Probes) == 1, nil
targetProbes = []*network.Probe{}
for i := range *lb.LoadBalancerPropertiesFormat.Probes {
probe := (*lb.LoadBalancerPropertiesFormat.Probes)[i]
utils.Logf("One probe of LB is %q", *probe.Name)
probeSplit := strings.Split(*probe.Name, "-")
Expect(len(probeSplit)).NotTo(Equal(0))
if pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] == probeSplit[0] {
targetProbes = append(targetProbes, &probe)
}
}
return len(targetProbes) == 1, nil
})
Expect(err).NotTo(HaveOccurred())
// get lb from azure client
By("Validating health probe configs")
probes := *lb.Probes
Expect((len(probes))).To(Equal(1))
Expect(probes[0].Protocol).To(Equal(network.ProbeProtocolHTTP))
Expect((len(targetProbes))).To(Equal(1))
Expect(targetProbes[0].Protocol).To(Equal(network.ProbeProtocolHTTP))
})
})

Expand Down Expand Up @@ -791,12 +799,27 @@ var _ = Describe("Multi-ports service", Label(utils.TestSuiteLabelMultiPorts), f
Expect(retryErr).NotTo(HaveOccurred())
utils.Logf("Successfully updated LoadBalancer service " + serviceName + " in namespace " + ns.Name)

pipFrontendConfigID := getPIPFrontendConfigurationID(tc, publicIP, tc.GetResourceGroup(), "")
pipFrontendConfigIDSplit := strings.Split(pipFrontendConfigID, "/")
Expect(len(pipFrontendConfigIDSplit)).NotTo(Equal(0))

//wait for backend update
var targetProbes []*network.Probe
err = wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) {
lb := getAzureLoadBalancerFromPIP(tc, publicIP, tc.GetResourceGroup(), "")
return len(*lb.LoadBalancerPropertiesFormat.Probes) == 2 &&
*(*lb.LoadBalancerPropertiesFormat.Probes)[0].Port != nodeHealthCheckPort &&
*(*lb.LoadBalancerPropertiesFormat.Probes)[1].Port != nodeHealthCheckPort, nil
targetProbes = []*network.Probe{}
for i := range *lb.LoadBalancerPropertiesFormat.Probes {
probe := (*lb.LoadBalancerPropertiesFormat.Probes)[i]
utils.Logf("One probe of LB is %q", *probe.Name)
probeSplit := strings.Split(*probe.Name, "-")
Expect(len(probeSplit)).NotTo(Equal(0))
if pipFrontendConfigIDSplit[len(pipFrontendConfigIDSplit)-1] == probeSplit[0] {
targetProbes = append(targetProbes, &probe)
}
}
return len(targetProbes) == 2 &&
*(targetProbes)[0].Port != nodeHealthCheckPort &&
*(targetProbes)[1].Port != nodeHealthCheckPort, nil
})
Expect(err).NotTo(HaveOccurred())
})
Expand Down Expand Up @@ -832,7 +855,7 @@ func waitComparePIPTags(tc *utils.AzureTestClient, expectedTags map[string]*stri
return err
}

func getAzureLoadBalancerFromPIP(tc *utils.AzureTestClient, pip, pipResourceGroup, lbResourceGroup string) *network.LoadBalancer {
func getPIPFrontendConfigurationID(tc *utils.AzureTestClient, pip, pipResourceGroup, lbResourceGroup string) string {
utils.Logf("Getting public IPs in the resourceGroup " + pipResourceGroup)
pipList, err := tc.ListPublicIPs(pipResourceGroup)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -849,8 +872,13 @@ func getAzureLoadBalancerFromPIP(tc *utils.AzureTestClient, pip, pipResourceGrou
break
}
}
Expect(pipFrontendConfigurationID).NotTo(Equal(""))
utils.Logf("Successfully obtained PIP front config id: %v", pipFrontendConfigurationID)
return pipFrontendConfigurationID
}

func getAzureLoadBalancerFromPIP(tc *utils.AzureTestClient, pip, pipResourceGroup, lbResourceGroup string) *network.LoadBalancer {
pipFrontendConfigurationID := getPIPFrontendConfigurationID(tc, pip, pipResourceGroup, lbResourceGroup)
Expect(pipFrontendConfigurationID).NotTo(Equal(""))

utils.Logf("Getting loadBalancer name from pipFrontendConfigurationID")
match := lbNameRE.FindStringSubmatch(pipFrontendConfigurationID)
Expand Down

0 comments on commit 5feb0df

Please sign in to comment.