Skip to content

Commit

Permalink
e2e: node: unify sample device plugin utilities
Browse files Browse the repository at this point in the history
Start to consolidate the sample device plugin utility
and constants in a central place, because we need
to use it in different e2e tests.

Having a central dependency is better than a maze of
entangled e2e tests depending on each other helpers.

Signed-off-by: Francesco Romani <fromani@redhat.com>
  • Loading branch information
ffromani committed Feb 22, 2023
1 parent 871201b commit 92e0020
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 121 deletions.
92 changes: 16 additions & 76 deletions test/e2e_node/device_plugin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
admissionapi "k8s.io/pod-security-admission/api"

"k8s.io/apimachinery/pkg/api/resource"
Expand All @@ -44,17 +43,6 @@ import (
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
)

const (
// sampleResourceName is the name of the example resource which is used in the e2e test
sampleResourceName = "example.com/resource"
// sampleDevicePluginName is the name of the device plugin pod
sampleDevicePluginName = "sample-device-plugin"

// fake resource name
resourceName = "example.com/resource"
envVarNamePluginSockDir = "PLUGIN_SOCK_DIR"
)

var (
appsScheme = runtime.NewScheme()
appsCodecs = serializer.NewCodecFactory(appsScheme)
Expand All @@ -67,17 +55,6 @@ var _ = SIGDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:Devic
testDevicePlugin(f, kubeletdevicepluginv1beta1.DevicePluginPath)
})

// numberOfSampleResources returns the number of resources advertised by a node.
func numberOfSampleResources(node *v1.Node) int64 {
val, ok := node.Status.Capacity[sampleResourceName]

if !ok {
return 0
}

return val.Value()
}

// readDaemonSetV1OrDie reads daemonset object from bytes. Panics on error.
func readDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet {
appsv1.AddToScheme(appsScheme)
Expand Down Expand Up @@ -133,40 +110,23 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
}, f.Timeouts.PodDelete, f.Timeouts.Poll).Should(gomega.Succeed())

ginkgo.By("Scheduling a sample device plugin pod")
data, err := e2etestfiles.Read(SampleDevicePluginDSYAML)
if err != nil {
framework.Fail(err.Error())
}
ds := readDaemonSetV1OrDie(data)

dp := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: sampleDevicePluginName,
},
Spec: ds.Spec.Template.Spec,
}

for i := range dp.Spec.Containers[0].Env {
if dp.Spec.Containers[0].Env[i].Name == envVarNamePluginSockDir {
dp.Spec.Containers[0].Env[i].Value = pluginSockDir
}
}
dp := getSampleDevicePluginPod(pluginSockDir)
dptemplate = dp.DeepCopy()
devicePluginPod = e2epod.NewPodClient(f).CreateSync(ctx, dp)

ginkgo.By("Waiting for devices to become available on the local node")
gomega.Eventually(ctx, func(ctx context.Context) bool {
node, ready := getLocalTestNode(ctx, f)
return ready && numberOfSampleResources(node) > 0
return ready && CountSampleDeviceCapacity(node) > 0
}, 5*time.Minute, framework.Poll).Should(gomega.BeTrue())
framework.Logf("Successfully created device plugin pod")

ginkgo.By("Waiting for the resource exported by the sample device plugin to become available on the local node")
gomega.Eventually(ctx, func(ctx context.Context) bool {
node, ready := getLocalTestNode(ctx, f)
return ready &&
numberOfDevicesCapacity(node, resourceName) == devsLen &&
numberOfDevicesAllocatable(node, resourceName) == devsLen
CountSampleDeviceCapacity(node) == devsLen &&
CountSampleDeviceAllocatable(node) == devsLen
}, 30*time.Second, framework.Poll).Should(gomega.BeTrue())
})

Expand All @@ -191,15 +151,15 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
ginkgo.By("Waiting for devices to become unavailable on the local node")
gomega.Eventually(ctx, func(ctx context.Context) bool {
node, ready := getLocalTestNode(ctx, f)
return ready && numberOfSampleResources(node) <= 0
return ready && CountSampleDeviceCapacity(node) <= 0
}, 5*time.Minute, framework.Poll).Should(gomega.BeTrue())

ginkgo.By("devices now unavailable on the local node")
})

ginkgo.It("Can schedule a pod that requires a device", func(ctx context.Context) {
podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep 60"
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(resourceName, podRECMD))
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
deviceIDRE := "stub devices: (Dev-[0-9]+)"
devID1 := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
gomega.Expect(devID1).To(gomega.Not(gomega.Equal("")))
Expand Down Expand Up @@ -250,16 +210,16 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
framework.ExpectEqual(len(v1alphaResourcesForOurPod.Containers[0].Devices), 1)
framework.ExpectEqual(len(v1ResourcesForOurPod.Containers[0].Devices), 1)

framework.ExpectEqual(v1alphaResourcesForOurPod.Containers[0].Devices[0].ResourceName, resourceName)
framework.ExpectEqual(v1ResourcesForOurPod.Containers[0].Devices[0].ResourceName, resourceName)
framework.ExpectEqual(v1alphaResourcesForOurPod.Containers[0].Devices[0].ResourceName, SampleDeviceResourceName)
framework.ExpectEqual(v1ResourcesForOurPod.Containers[0].Devices[0].ResourceName, SampleDeviceResourceName)

framework.ExpectEqual(len(v1alphaResourcesForOurPod.Containers[0].Devices[0].DeviceIds), 1)
framework.ExpectEqual(len(v1ResourcesForOurPod.Containers[0].Devices[0].DeviceIds), 1)
})

ginkgo.It("Keeps device plugin assignments across pod and kubelet restarts", func(ctx context.Context) {
podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep 60"
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(resourceName, podRECMD))
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
deviceIDRE := "stub devices: (Dev-[0-9]+)"
devID1 := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
gomega.Expect(devID1).To(gomega.Not(gomega.Equal("")))
Expand Down Expand Up @@ -288,7 +248,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {

ginkgo.It("Keeps device plugin assignments after the device plugin has been re-registered", func(ctx context.Context) {
podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep 60"
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(resourceName, podRECMD))
pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))
deviceIDRE := "stub devices: (Dev-[0-9]+)"
devID1 := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE)
gomega.Expect(devID1).To(gomega.Not(gomega.Equal("")))
Expand Down Expand Up @@ -322,12 +282,12 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
gomega.Eventually(ctx, func() bool {
node, ready := getLocalTestNode(ctx, f)
return ready &&
numberOfDevicesCapacity(node, resourceName) == devsLen &&
numberOfDevicesAllocatable(node, resourceName) == devsLen
CountSampleDeviceCapacity(node) == devsLen &&
CountSampleDeviceAllocatable(node) == devsLen
}, 30*time.Second, framework.Poll).Should(gomega.BeTrue())

ginkgo.By("Creating another pod")
pod2 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(resourceName, podRECMD))
pod2 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD))

ginkgo.By("Checking that pod got a different fake device")
devID2 := parseLog(ctx, f, pod2.Name, pod2.Name, deviceIDRE)
Expand All @@ -338,10 +298,10 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
}

// makeBusyboxPod returns a simple Pod spec with a busybox container
// that requests resourceName and runs the specified command.
func makeBusyboxPod(resourceName, cmd string) *v1.Pod {
// that requests SampleDeviceResourceName and runs the specified command.
func makeBusyboxPod(SampleDeviceResourceName, cmd string) *v1.Pod {
podName := "device-plugin-test-" + string(uuid.NewUUID())
rl := v1.ResourceList{v1.ResourceName(resourceName): *resource.NewQuantity(1, resource.DecimalSI)}
rl := v1.ResourceList{v1.ResourceName(SampleDeviceResourceName): *resource.NewQuantity(1, resource.DecimalSI)}

return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: podName},
Expand Down Expand Up @@ -397,23 +357,3 @@ func parseLog(ctx context.Context, f *framework.Framework, podName string, contN

return matches[1]
}

// numberOfDevicesCapacity returns the number of devices of resourceName advertised by a node capacity
func numberOfDevicesCapacity(node *v1.Node, resourceName string) int64 {
val, ok := node.Status.Capacity[v1.ResourceName(resourceName)]
if !ok {
return 0
}

return val.Value()
}

// numberOfDevicesAllocatable returns the number of devices of resourceName advertised by a node allocatable
func numberOfDevicesAllocatable(node *v1.Node, resourceName string) int64 {
val, ok := node.Status.Allocatable[v1.ResourceName(resourceName)]
if !ok {
return 0
}

return val.Value()
}
29 changes: 26 additions & 3 deletions test/e2e_node/image_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,16 @@ import (
"sync"
"time"

v1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
internalapi "k8s.io/cri-api/pkg/apis"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
commontest "k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
e2egpu "k8s.io/kubernetes/test/e2e/framework/gpu"
e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
Expand All @@ -45,9 +48,6 @@ const (
imagePullRetryDelay = time.Second
// Number of parallel count to pull images.
maxParallelImagePullCount = 5

// SampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework.
SampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml"
)

// NodePrePullImageList is a list of images used in node e2e test. These images will be prepulled
Expand Down Expand Up @@ -239,6 +239,29 @@ func getSampleDevicePluginImage() (string, error) {
return ds.Spec.Template.Spec.Containers[0].Image, nil
}

// getSampleDevicePluginPod returns the Sample Device Plugin pod to be used e2e tests.
func getSampleDevicePluginPod(pluginSockDir string) *v1.Pod {
data, err := e2etestfiles.Read(SampleDevicePluginDSYAML)
if err != nil {
framework.Fail(err.Error())
}

ds := readDaemonSetV1OrDie(data)
dp := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: SampleDevicePluginName,
},
Spec: ds.Spec.Template.Spec,
}
for i := range dp.Spec.Containers[0].Env {
if dp.Spec.Containers[0].Env[i].Name == SampleDeviceEnvVarNamePluginSockDir {
dp.Spec.Containers[0].Env[i].Value = pluginSockDir
}
}

return dp
}

// getSRIOVDevicePluginImage returns the image of SRIOV device plugin.
func getSRIOVDevicePluginImage() (string, error) {
data, err := e2etestfiles.Read(SRIOVDevicePluginDSYAML)
Expand Down
48 changes: 6 additions & 42 deletions test/e2e_node/podresources_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1"
kubefeatures "k8s.io/kubernetes/pkg/features"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
Expand All @@ -45,7 +46,6 @@ import (
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
)

const (
Expand Down Expand Up @@ -892,12 +892,12 @@ func getOnlineCPUs() (cpuset.CPUSet, error) {
func setupSampleDevicePluginOrFail(ctx context.Context, f *framework.Framework) *v1.Pod {
e2enode.WaitForNodeToBeReady(ctx, f.ClientSet, framework.TestContext.NodeName, 5*time.Minute)

dp := getSampleDevicePluginPod()
dp := getSampleDevicePluginPod(kubeletdevicepluginv1beta1.DevicePluginPath)
dp.Spec.NodeName = framework.TestContext.NodeName

ginkgo.By("Create the sample device plugin pod")

dpPod = e2epod.NewPodClient(f).CreateSync(ctx, dp)
dpPod := e2epod.NewPodClient(f).CreateSync(ctx, dp)

err := e2epod.WaitForPodCondition(ctx, f.ClientSet, dpPod.Namespace, dpPod.Name, "Ready", 120*time.Second, testutils.PodRunningReady)
if err != nil {
Expand All @@ -920,52 +920,16 @@ func teardownSampleDevicePluginOrFail(ctx context.Context, f *framework.Framewor
waitForAllContainerRemoval(ctx, pod.Name, pod.Namespace)
}

func findTopologyUnawareResource(node *v1.Node) int64 {
framework.Logf("Node status allocatable: %v", node.Status.Allocatable)
for key, val := range node.Status.Allocatable {
if string(key) == defaultTopologyUnawareResourceName {
v := val.Value()
if v > 0 {
return v
}
}
}
return 0
}

func waitForTopologyUnawareResources(ctx context.Context, f *framework.Framework, pod *v1.Pod) {
func waitForTopologyUnawareResources(ctx context.Context, f *framework.Framework) {
ginkgo.By(fmt.Sprintf("Waiting for %q resources to become available on the local node", defaultTopologyUnawareResourceName))

gomega.Eventually(ctx, func(ctx context.Context) bool {
node := getLocalNode(ctx, f)
resourceAmount := findTopologyUnawareResource(node)
return resourceAmount != 0
resourceAmount := CountSampleDeviceAllocatable(node)
return resourceAmount > 0
}, 2*time.Minute, framework.Poll).Should(gomega.BeTrue())
}

// getSampleDevicePluginPod returns the Sample Device Plugin pod to be used e2e tests.
func getSampleDevicePluginPod() *v1.Pod {
data, err := e2etestfiles.Read(SampleDevicePluginDSYAML)
if err != nil {
framework.Fail(err.Error())
}

ds := readDaemonSetV1OrDie(data)
dp := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: sampleDevicePluginName,
},
Spec: ds.Spec.Template.Spec,
}
for i := range dp.Spec.Containers[0].Env {
if dp.Spec.Containers[0].Env[i].Name == envVarNamePluginSockDir {
dp.Spec.Containers[0].Env[i].Value = pluginSockDir
}
}

return dp
}

func getPodResourcesMetrics(ctx context.Context) (e2emetrics.KubeletMetrics, error) {
// we are running out of good names, so we need to be unnecessarily specific to avoid clashes
ginkgo.By("getting Pod Resources metrics from the metrics API")
Expand Down
52 changes: 52 additions & 0 deletions test/e2e_node/util_sampledevice.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2enode

import (
v1 "k8s.io/api/core/v1"
)

const (
// SampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework.
SampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml"

// SampleDevicePluginName is the name of the device plugin pod
SampleDevicePluginName = "sample-device-plugin"

// SampleDeviceResourceName is the name of the resource provided by the sample device plugin
SampleDeviceResourceName = "example.com/resource"

SampleDeviceEnvVarNamePluginSockDir = "PLUGIN_SOCK_DIR"
)

// CountSampleDeviceCapacity returns the number of devices of SampleDeviceResourceName advertised by a node capacity
func CountSampleDeviceCapacity(node *v1.Node) int64 {
val, ok := node.Status.Capacity[v1.ResourceName(SampleDeviceResourceName)]
if !ok {
return 0
}
return val.Value()
}

// CountSampleDeviceAllocatable returns the number of devices of SampleDeviceResourceName advertised by a node allocatable
func CountSampleDeviceAllocatable(node *v1.Node) int64 {
val, ok := node.Status.Allocatable[v1.ResourceName(SampleDeviceResourceName)]
if !ok {
return 0
}
return val.Value()
}

0 comments on commit 92e0020

Please sign in to comment.