Skip to content

Commit

Permalink
test/e2e_node: add e2e test for Kubeletconfig drop-in dir
Browse files Browse the repository at this point in the history
Signed-off-by: Sohan Kunkerkar <sohank2602@gmail.com>
Co-authored-by: Peter Hunt <pehunt@redhat.com>
  • Loading branch information
sohankunkerkar and haircommander committed Nov 3, 2023
1 parent ee5578b commit ad7b9b5
Show file tree
Hide file tree
Showing 5 changed files with 157 additions and 10 deletions.
15 changes: 8 additions & 7 deletions test/e2e/framework/test_context.go
Expand Up @@ -99,13 +99,14 @@ var (
// Test suite authors can use framework/viper to make all command line
// parameters also configurable via a configuration file.
type TestContextType struct {
KubeConfig string
KubeContext string
KubeAPIContentType string
KubeletRootDir string
CertDir string
Host string
BearerToken string `datapolicy:"token"`
KubeConfig string
KubeContext string
KubeAPIContentType string
KubeletRootDir string
KubeletConfigDropinDir string
CertDir string
Host string
BearerToken string `datapolicy:"token"`
// TODO: Deprecating this over time... instead just use gobindata_util.go , see #23987.
RepoRoot string
// ListImages will list off all images that are used then quit
Expand Down
1 change: 1 addition & 0 deletions test/e2e/nodefeature/nodefeature.go
Expand Up @@ -37,6 +37,7 @@ var (
GracefulNodeShutdownBasedOnPodPriority = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("GracefulNodeShutdownBasedOnPodPriority"))
HostAccess = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("HostAccess"))
ImageID = framework.WithNodeFeature(framework.ValidNodeFeatures.Add(" ImageID"))
KubeletConfigDropInDir = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("KubeletConfigDropInDir"))
LSCIQuotaMonitoring = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("LSCIQuotaMonitoring"))
NodeAllocatable = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeAllocatable"))
NodeProblemDetector = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("NodeProblemDetector"))
Expand Down
9 changes: 9 additions & 0 deletions test/e2e_node/e2e_node_suite_test.go
Expand Up @@ -90,6 +90,7 @@ func registerNodeFlags(flags *flag.FlagSet) {
framework.TestContext.NodeE2E = true
flags.StringVar(&framework.TestContext.BearerToken, "bearer-token", "", "The bearer token to authenticate with. If not specified, it would be a random token. Currently this token is only used in node e2e tests.")
flags.StringVar(&framework.TestContext.NodeName, "node-name", "", "Name of the node to run tests on.")
flags.StringVar(&framework.TestContext.KubeletConfigDropinDir, "config-dir", "", "Path to a directory containing drop-in configurations for the kubelet.")
// TODO(random-liu): Move kubelet start logic out of the test.
// TODO(random-liu): Move log fetch logic out of the test.
// There are different ways to start kubelet (systemd, initd, docker, manually started etc.)
Expand Down Expand Up @@ -200,6 +201,14 @@ func TestE2eNode(t *testing.T) {

// We're not running in a special mode so lets run tests.
gomega.RegisterFailHandler(ginkgo.Fail)
// Initialize the KubeletConfigDropinDir again if the test doesn't run in run-kubelet-mode.
if framework.TestContext.KubeletConfigDropinDir == "" {
var err error
framework.TestContext.KubeletConfigDropinDir, err = services.KubeletConfigDirCWDDir()
if err != nil {
klog.Errorf("failed to create kubelet config directory: %v", err)
}
}
reportDir := framework.TestContext.ReportDir
if reportDir != "" {
// Create the directory if it doesn't already exist
Expand Down
113 changes: 113 additions & 0 deletions test/e2e_node/kubelet_config_dir_test.go
@@ -0,0 +1,113 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2enode

import (
"context"
"os"
"path/filepath"
"time"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/nodefeature"
)

var _ = SIGDescribe("Kubelet Config", framework.WithSlow(), framework.WithSerial(), framework.WithDisruptive(), nodefeature.KubeletConfigDropInDir, func() {
f := framework.NewDefaultFramework("kubelet-config-drop-in-dir-test")
ginkgo.Context("when merging drop-in configs", func() {
var oldcfg *kubeletconfig.KubeletConfiguration
ginkgo.BeforeEach(func(ctx context.Context) {
var err error
oldcfg, err = getCurrentKubeletConfig(ctx)
framework.ExpectNoError(err)
})
ginkgo.AfterEach(func(ctx context.Context) {
files, err := filepath.Glob(filepath.Join(framework.TestContext.KubeletConfigDropinDir, "*"+".conf"))
framework.ExpectNoError(err)
for _, file := range files {
err := os.Remove(file)
framework.ExpectNoError(err)
}
updateKubeletConfig(ctx, f, oldcfg, true)
})
ginkgo.It("should merge kubelet configs correctly", func(ctx context.Context) {
// Get the initial kubelet configuration
initialConfig, err := getCurrentKubeletConfig(ctx)
framework.ExpectNoError(err)

ginkgo.By("Stopping the kubelet")
restartKubelet := stopKubelet()

// wait until the kubelet health check will fail
gomega.Eventually(ctx, func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse())

configDir := framework.TestContext.KubeletConfigDropinDir

contents := []byte(`apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
port: 10255
readOnlyPort: 10257
clusterDNS:
- 192.168.1.10
systemReserved:
memory: 1Gi`)
framework.ExpectNoError(os.WriteFile(filepath.Join(configDir, "10-kubelet.conf"), contents, 0755))
contents = []byte(`apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
clusterDNS:
- 192.168.1.1
- 192.168.1.5
- 192.168.1.8
port: 8080
cpuManagerReconcilePeriod: 0s
systemReserved:
memory: 2Gi`)
framework.ExpectNoError(os.WriteFile(filepath.Join(configDir, "20-kubelet.conf"), contents, 0755))
ginkgo.By("Restarting the kubelet")
restartKubelet()
// wait until the kubelet health check will succeed
gomega.Eventually(ctx, func() bool {
return kubeletHealthCheck(kubeletHealthCheckURL)
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue())

mergedConfig, err := getCurrentKubeletConfig(ctx)
framework.ExpectNoError(err)

// Replace specific fields in the initial configuration with expectedConfig values
initialConfig.Port = int32(8080) // not overridden by second file, should be retained.
initialConfig.ReadOnlyPort = int32(10257) // overridden by second file.
initialConfig.SystemReserved = map[string]string{ // overridden by map in second file.
"memory": "2Gi",
}
initialConfig.ClusterDNS = []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"} // overridden by slice in second file.
// This value was explicitly set in the drop-in, make sure it is retained
initialConfig.CPUManagerReconcilePeriod = metav1.Duration{Duration: time.Duration(0)}
// Meanwhile, this value was not explicitly set, but could have been overridden by a "default" of 0 for the type.
// Ensure the true default persists.
initialConfig.CPUCFSQuotaPeriod = metav1.Duration{Duration: time.Duration(100000000)}
// Compare the expected config with the merged config
gomega.Expect(initialConfig).To(gomega.BeComparableTo(mergedConfig), "Merged kubelet config does not match the expected configuration.")
})
})

})
29 changes: 26 additions & 3 deletions test/e2e_node/services/kubelet.go
Expand Up @@ -174,6 +174,12 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
return nil, err
}

// KubeletDropInConfiguration directory path
framework.TestContext.KubeletConfigDropinDir, err = KubeletConfigDirCWDDir()
if err != nil {
return nil, err
}

// Create pod directory
podPath, err := createPodDirectory()
if err != nil {
Expand Down Expand Up @@ -243,6 +249,8 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
unitName = fmt.Sprintf("kubelet-%s.service", unitTimestamp)
cmdArgs = append(cmdArgs,
systemdRun,
// Set the environment variable to enable kubelet config drop-in directory.
"-E", "KUBELET_CONFIG_DROPIN_DIR_ALPHA=yes",
"-p", "Delegate=true",
"-p", logLocation+framework.TestContext.ReportDir+"/kubelet.log",
"--unit="+unitName,
Expand Down Expand Up @@ -282,6 +290,9 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
kc.FeatureGates = featureGates
}

// Add the KubeletDropinConfigDirectory flag if set.
cmdArgs = append(cmdArgs, "--config-dir", framework.TestContext.KubeletConfigDropinDir)

// Keep hostname override for convenience.
if framework.TestContext.NodeName != "" { // If node name is specified, set hostname override.
cmdArgs = append(cmdArgs, "--hostname-override", framework.TestContext.NodeName)
Expand All @@ -295,7 +306,7 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
cmdArgs = append(cmdArgs, "--image-service-endpoint", framework.TestContext.ImageServiceEndpoint)
}

if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil {
if err := WriteKubeletConfigFile(kc, kubeletConfigPath); err != nil {
return nil, err
}
// add the flag to load config from a file
Expand Down Expand Up @@ -324,8 +335,8 @@ func (e *E2EServices) startKubelet(featureGates map[string]bool) (*server, error
return server, server.start()
}

// writeKubeletConfigFile writes the kubelet config file based on the args and returns the filename
func writeKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error {
// WriteKubeletConfigFile writes the kubelet config file based on the args and returns the filename
func WriteKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error {
data, err := kubeletconfigcodec.EncodeKubeletConfig(internal, kubeletconfigv1beta1.SchemeGroupVersion)
if err != nil {
return err
Expand Down Expand Up @@ -408,6 +419,18 @@ func kubeletConfigCWDPath() (string, error) {
return filepath.Join(cwd, "kubelet-config"), nil
}

func KubeletConfigDirCWDDir() (string, error) {
cwd, err := os.Getwd()
if err != nil {
return "", fmt.Errorf("failed to get current working directory: %w", err)
}
dir := filepath.Join(cwd, "kubelet.conf.d")
if err := os.MkdirAll(dir, 0755); err != nil {
return "", err
}
return dir, nil
}

// like createKubeconfig, but creates kubeconfig at current-working-directory/kubeconfig
// returns a fully-qualified path to the kubeconfig file
func createKubeconfigCWD() (string, error) {
Expand Down

0 comments on commit ad7b9b5

Please sign in to comment.