-
Notifications
You must be signed in to change notification settings - Fork 1
/
daemonsets.go
181 lines (150 loc) · 5.69 KB
/
daemonsets.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrades
import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/api/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
)
// DaemonSetUpgradeTest tests that a DaemonSet is running before and after
// a cluster upgrade.
type DaemonSetUpgradeTest struct {
daemonSet *extensions.DaemonSet
}
func (DaemonSetUpgradeTest) Name() string { return "daemonset-upgrade" }
// Setup creates a DaemonSet and verifies that it's running
func (t *DaemonSetUpgradeTest) Setup(f *framework.Framework) {
daemonSetName := "ds1"
labelSet := map[string]string{"ds-name": daemonSetName}
image := "gcr.io/google_containers/serve_hostname:v1.4"
ns := f.Namespace
t.daemonSet = &extensions.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: ns.Name,
Name: daemonSetName,
},
Spec: extensions.DaemonSetSpec{
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labelSet,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: daemonSetName,
Image: image,
Ports: []v1.ContainerPort{{ContainerPort: 9376}},
},
},
},
},
},
}
By("Creating a DaemonSet")
var err error
if t.daemonSet, err = f.ClientSet.Extensions().DaemonSets(ns.Name).Create(t.daemonSet); err != nil {
framework.Failf("unable to create test DaemonSet %s: %v", t.daemonSet.Name, err)
}
By("Waiting for DaemonSet pods to become ready")
err = wait.Poll(framework.Poll, framework.PodStartTimeout, func() (bool, error) {
return checkRunningOnAllNodes(f, t.daemonSet.Namespace, t.daemonSet.Labels)
})
framework.ExpectNoError(err)
By("Validating the DaemonSet after creation")
t.validateRunningDaemonSet(f)
}
// Test waits until the upgrade has completed and then verifies that the DaemonSet
// is still running
func (t *DaemonSetUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
By("Waiting for upgradet to complete before re-validating DaemonSet")
<-done
By("validating the DaemonSet is still running after upgrade")
t.validateRunningDaemonSet(f)
}
// Teardown cleans up any remaining resources.
func (t *DaemonSetUpgradeTest) Teardown(f *framework.Framework) {
// rely on the namespace deletion to clean up everything
}
func (t *DaemonSetUpgradeTest) validateRunningDaemonSet(f *framework.Framework) {
By("confirming the DaemonSet pods are running on all expected nodes")
res, err := checkRunningOnAllNodes(f, t.daemonSet.Namespace, t.daemonSet.Labels)
framework.ExpectNoError(err)
if !res {
framework.Failf("expected DaemonSet pod to be running on all nodes, it was not")
}
// DaemonSet resource itself should be good
By("confirming the DaemonSet resource is in a good state")
res, err = checkDaemonStatus(f, t.daemonSet.Namespace, t.daemonSet.Name)
framework.ExpectNoError(err)
if !res {
framework.Failf("expected DaemonSet to be in a good state, it was not")
}
}
func checkRunningOnAllNodes(f *framework.Framework, namespace string, selector map[string]string) (bool, error) {
nodeList, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{})
if err != nil {
return false, err
}
nodeNames := make([]string, 0)
for _, node := range nodeList.Items {
if len(node.Spec.Taints) == 0 {
nodeNames = append(nodeNames, node.Name)
} else {
framework.Logf("Node %v not expected to have DaemonSet pod, has taints %v", node.Name, node.Spec.Taints)
}
}
return checkDaemonPodOnNodes(f, namespace, selector, nodeNames)
}
func checkDaemonPodOnNodes(f *framework.Framework, namespace string, labelSet map[string]string, nodeNames []string) (bool, error) {
selector := labels.Set(labelSet).AsSelector()
options := metav1.ListOptions{LabelSelector: selector.String()}
podList, err := f.ClientSet.Core().Pods(namespace).List(options)
if err != nil {
return false, err
}
pods := podList.Items
nodesToPodCount := make(map[string]int)
for _, pod := range pods {
if controller.IsPodActive(&pod) {
framework.Logf("Pod name: %v\t Node Name: %v", pod.Name, pod.Spec.NodeName)
nodesToPodCount[pod.Spec.NodeName]++
}
}
framework.Logf("nodesToPodCount: %v", nodesToPodCount)
// Ensure that exactly 1 pod is running on all nodes in nodeNames.
for _, nodeName := range nodeNames {
if nodesToPodCount[nodeName] != 1 {
return false, nil
}
}
// Ensure that sizes of the lists are the same. We've verified that every element of nodeNames is in
// nodesToPodCount, so verifying the lengths are equal ensures that there aren't pods running on any
// other nodes.
return len(nodesToPodCount) == len(nodeNames), nil
}
func checkDaemonStatus(f *framework.Framework, namespace string, dsName string) (bool, error) {
ds, err := f.ClientSet.ExtensionsV1beta1().DaemonSets(namespace).Get(dsName, metav1.GetOptions{})
if err != nil {
return false, err
}
desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady
if desired != scheduled && desired != ready {
return false, nil
}
return true, nil
}