Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add more e2e tests for DaemonSet templateGeneration and pod adoption #42211

Merged
merged 1 commit into from
Mar 9, 2017
+165 −7
Merged

Add more e2e tests for DaemonSet templateGeneration and pod adoption #42211

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
172 changes: 165 additions & 7 deletions test/e2e/daemon_set.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package e2e
import (
"fmt"
"reflect"
"sort"
"strings"
"time"

Expand Down Expand Up @@ -235,28 +236,38 @@ var _ = framework.KubeDescribe("Daemon set [Serial]", func() {
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to revive")
})

It("Should not update pod when spec was updated and update strategy is on delete", func() {
It("Should not update pod when spec was updated and update strategy is OnDelete", func() {
label := map[string]string{daemonsetNameLabel: dsName}

framework.Logf("Creating simple daemon set %s", dsName)
ds, err := c.Extensions().DaemonSets(ns).Create(newDaemonSet(dsName, image, label))
Expect(err).NotTo(HaveOccurred())
Expect(ds.Spec.TemplateGeneration).To(Equal(int64(1)))

By("Check that daemon pods launch on every node of the cluster.")
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, label, ds))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to start")

By("Make sure all daemon pods have correct template generation 1")
err = checkDaemonPodsTemplateGeneration(c, ns, label, "1")
Expect(err).NotTo(HaveOccurred())

By("Update daemon pods image.")
ds, err = c.Extensions().DaemonSets(ns).Get(dsName, metav1.GetOptions{})
ds.Spec.Template.Spec.Containers[0].Image = redisImage
_, err = c.Extensions().DaemonSets(ns).Update(ds)
ds, err = c.Extensions().DaemonSets(ns).Update(ds)
Expect(err).NotTo(HaveOccurred())
Expect(ds.Spec.TemplateGeneration).To(Equal(int64(2)))

By("Check that demon pods have not set updated image.")
By("Check that daemon pods images aren't updated.")
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkDaemonPodsImage(c, ns, label, image))
Expect(err).NotTo(HaveOccurred())

By("Make sure all daemon pods have correct template generation 1")
err = checkDaemonPodsTemplateGeneration(c, ns, label, "1")
Expect(err).NotTo(HaveOccurred())

By("Check that daemon pods are still running on every node of the cluster.")
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, label, ds))
Expand All @@ -266,32 +277,118 @@ var _ = framework.KubeDescribe("Daemon set [Serial]", func() {
It("Should update pod when spec was updated and update strategy is RollingUpdate", func() {
label := map[string]string{daemonsetNameLabel: dsName}

framework.Logf("Creating simple daemon set %s", dsName)
ds, err := c.Extensions().DaemonSets(ns).Create(newDaemonSet(dsName, image, label))
templateGeneration := int64(999)
framework.Logf("Creating simple daemon set %s with templateGeneration %d", dsName, templateGeneration)
ds := newDaemonSet(dsName, image, label)
ds.Spec.TemplateGeneration = templateGeneration
ds, err := c.Extensions().DaemonSets(ns).Create(ds)
Expect(err).NotTo(HaveOccurred())
Expect(ds.Spec.TemplateGeneration).To(Equal(templateGeneration))

By("Check that daemon pods launch on every node of the cluster.")
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, label, ds))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to start")

By(fmt.Sprintf("Make sure all daemon pods have correct template generation %d", templateGeneration))
err = checkDaemonPodsTemplateGeneration(c, ns, label, fmt.Sprint(templateGeneration))
Expect(err).NotTo(HaveOccurred())

By("Update daemon pods image.")
ds, err = c.Extensions().DaemonSets(ns).Get(dsName, metav1.GetOptions{})
ds.Spec.Template.Spec.Containers[0].Image = redisImage
ds.Spec.UpdateStrategy = extensions.DaemonSetUpdateStrategy{Type: extensions.RollingUpdateDaemonSetStrategyType}
_, err = c.Extensions().DaemonSets(ns).Update(ds)
ds, err = c.Extensions().DaemonSets(ns).Update(ds)
Expect(err).NotTo(HaveOccurred())
Expect(ds.Spec.TemplateGeneration).To(Equal(templateGeneration + 1))

By("Check that demon pods have not set updated image.")
By("Check that daemon pods images are updated.")
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkDaemonPodsImage(c, ns, label, redisImage))
Expect(err).NotTo(HaveOccurred())

By(fmt.Sprintf("Make sure all daemon pods have correct template generation %d", templateGeneration+1))
err = checkDaemonPodsTemplateGeneration(c, ns, label, fmt.Sprint(templateGeneration+1))
Expect(err).NotTo(HaveOccurred())

By("Check that daemon pods are still running on every node of the cluster.")
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, label, ds))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to start")
})

It("Should adopt or recreate existing pods when creating a RollingUpdate DaemonSet with matching or mismatching templateGeneration", func() {
label := map[string]string{daemonsetNameLabel: dsName}

templateGeneration := int64(999)
framework.Logf("Creating simple daemon set %s with templateGeneration %d", dsName, templateGeneration)
ds := newDaemonSet(dsName, image, label)
ds.Spec.TemplateGeneration = templateGeneration
ds.Spec.UpdateStrategy = extensions.DaemonSetUpdateStrategy{Type: extensions.RollingUpdateDaemonSetStrategyType}
ds, err := c.Extensions().DaemonSets(ns).Create(ds)
Expect(err).NotTo(HaveOccurred())
Expect(ds.Spec.TemplateGeneration).To(Equal(templateGeneration))

By("Check that daemon pods launch on every node of the cluster.")
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, label, ds))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to start")

By(fmt.Sprintf("Make sure all daemon pods have correct template generation %d", templateGeneration))
err = checkDaemonPodsTemplateGeneration(c, ns, label, fmt.Sprint(templateGeneration))
Expect(err).NotTo(HaveOccurred())

dsPodsLastCreationTime := getDaemonPodsLastCreationTime(c, ns, label)

By(fmt.Sprintf("Deleting DaemonSet %s and orphaning its pods", dsName))
trueVar := true
deleteOptions := &metav1.DeleteOptions{OrphanDependents: &trueVar}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't this depend on owner references?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes it depends on #42173

deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(ds.UID))
err = c.Extensions().DaemonSets(ns).Delete(ds.Name, deleteOptions)
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkDaemonSetDeleted(f, ns, ds.Name))
Expect(err).NotTo(HaveOccurred(), "error waiting for DaemonSet to be deleted")

newDSName := dsName + "-new-adopt"
By(fmt.Sprintf("Creating a new RollingUpdate DaemonSet %s to adopt pods", newDSName))
newDS := newDaemonSet(newDSName, image, label)
newDS.Spec.TemplateGeneration = templateGeneration
newDS.Spec.UpdateStrategy = extensions.DaemonSetUpdateStrategy{Type: extensions.RollingUpdateDaemonSetStrategyType}
newDS, err = c.Extensions().DaemonSets(ns).Create(newDS)
Expect(err).NotTo(HaveOccurred())
Expect(newDS.Spec.TemplateGeneration).To(Equal(templateGeneration))

By(fmt.Sprintf("Make sure no daemon pod updated its template generation %d", templateGeneration))
err = checkDaemonPodsTemplateGeneration(c, ns, label, fmt.Sprint(templateGeneration))
Expect(err).NotTo(HaveOccurred())

By("Make sure no pods are recreated")
newDSPodsFirstCreationTime := getDaemonPodsFirstCreationTime(c, ns, label)
Expect(newDSPodsFirstCreationTime.Before(dsPodsLastCreationTime) ||
newDSPodsFirstCreationTime.Equal(dsPodsLastCreationTime)).To(BeTrue())

By(fmt.Sprintf("Deleting DaemonSet %s and orphaning its pods", newDSName))
orphanDependents := true
err = c.Extensions().DaemonSets(ns).Delete(newDSName, &metav1.DeleteOptions{OrphanDependents: &orphanDependents})
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkDaemonSetDeleted(f, ns, newDSName))
Expect(err).NotTo(HaveOccurred(), "error waiting for DaemonSet to be deleted")

newRestartDSName := dsName + "-new-restart"
By(fmt.Sprintf("Creating a new RollingUpdate DaemonSet %s to restart adopted pods", newRestartDSName))
newRestartDS := newDaemonSet(newRestartDSName, image, label)
newRestartDS.Spec.UpdateStrategy = extensions.DaemonSetUpdateStrategy{Type: extensions.RollingUpdateDaemonSetStrategyType}
newRestartDS, err = c.Extensions().DaemonSets(ns).Create(newRestartDS)
Expect(err).NotTo(HaveOccurred())
Expect(newRestartDS.Spec.TemplateGeneration).To(Equal(int64(1)))

By("Wait for all DaemonSet pods template Generation to be updated to 1")
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, templateGenerationMatch(c, ns, label, "1"))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod template generation to be 1")

By("Make sure pods are recreated")
newRestartDSPodsFirstCreationTime := getDaemonPodsFirstCreationTime(c, ns, label)
Expect(dsPodsLastCreationTime.Before(newRestartDSPodsFirstCreationTime)).To(BeTrue())
})
})

func newDaemonSet(dsName, image string, label map[string]string) *extensions.DaemonSet {
Expand Down Expand Up @@ -488,3 +585,64 @@ func checkDaemonPodsImage(c clientset.Interface, ns string, selector map[string]
return true, nil
}
}

func templateGenerationMatch(c clientset.Interface, ns string, selector map[string]string, templateGeneration string) func() (bool, error) {
return func() (bool, error) {
err := checkDaemonPodsTemplateGeneration(c, ns, selector, templateGeneration)
match := err == nil
return match, nil
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can also return err here

Copy link
Member Author

@janetkuo janetkuo Feb 28, 2017 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the condition func for wait.Poll. If we return an err that's not nil, we'll immediately exit the wait loop instead of retrying again.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, right

}
}

func checkDaemonPodsTemplateGeneration(c clientset.Interface, ns string, label map[string]string, templateGeneration string) error {
pods := listDaemonPods(c, ns, label)
for _, pod := range pods.Items {
podTemplateGeneration := pod.Labels[extensions.DaemonSetTemplateGenerationKey]
if podTemplateGeneration != templateGeneration {
return fmt.Errorf("Expected pod %s/%s template generation %s, but got %s", pod.Namespace, pod.Name, templateGeneration, podTemplateGeneration)
}
}
return nil
}

func checkDaemonSetDeleted(f *framework.Framework, ns, name string) func() (bool, error) {
return func() (bool, error) {
_, err := f.ClientSet.Extensions().DaemonSets(ns).Get(name, metav1.GetOptions{})
if !apierrs.IsNotFound(err) {
return false, err
}
return true, nil
}
}

func getDaemonPodsLastCreationTime(c clientset.Interface, ns string, label map[string]string) metav1.Time {
sortedPods := getDaemonPodsSortedByCreationTime(c, ns, label)
return sortedPods[len(sortedPods)-1].ObjectMeta.CreationTimestamp
}

func getDaemonPodsFirstCreationTime(c clientset.Interface, ns string, label map[string]string) metav1.Time {
sortedPods := getDaemonPodsSortedByCreationTime(c, ns, label)
return sortedPods[0].ObjectMeta.CreationTimestamp
}

func getDaemonPodsSortedByCreationTime(c clientset.Interface, ns string, label map[string]string) []v1.Pod {
podList := listDaemonPods(c, ns, label)
pods := podList.Items
if len(pods) > 1 {
sort.Sort(podByCreationTimestamp(pods))
}
return pods
}

// podByCreationTimestamp sorts a list of DaemonSet pods by creation timestamp, using their names as a tie breaker.
type podByCreationTimestamp []v1.Pod
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't we have this already somewhere else?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have a podByCreationTimestamp in daemonset controller but it's []*v1.Pod not []v1.Pod


func (o podByCreationTimestamp) Len() int { return len(o) }
func (o podByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] }

func (o podByCreationTimestamp) Less(i, j int) bool {
if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) {
return o[i].Name < o[j].Name
}
return o[i].CreationTimestamp.Before(o[j].CreationTimestamp)
}