cmd/kubelet: implement drop-in configuration directory for kubelet

This implements a drop-in configuration directory for the kubelet
by introducing a "--config-dir" flag. Users can provide individual
kubelet config snippets in separate files, formatted similarly to
kubelet.conf. The kubelet will process the files in alphanumeric order,
appending configurations if subfield(s) doesn't exist, overwriting them if
they do, and handling lists by overwriting instead of merging.

Co-authored-by: Yu Qi Zhang <jerzhang@redhat.com>

cmd/kubelet/app/options/options.go

@@ -86,6 +86,10 @@ type KubeletFlags struct {
 	// Omit this flag to use the combination of built-in default configuration values and flags.
 	KubeletConfigFile string
 
+	// kubeletDropinConfigDirectory is a path to a directory to specify dropins allows the user to optionally specify
+	// additional configs to overwrite what is provided by default and in the KubeletConfigFile flag
+	KubeletDropinConfigDirectory string
+
 	// WindowsService should be set to true if kubelet is running as a service on Windows.
 	// Its corresponding flag only gets registered in Windows builds.
 	WindowsService bool
@@ -281,6 +285,7 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
 	f.addOSFlags(fs)
 
 	fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.")
+	fs.StringVar(&f.KubeletDropinConfigDirectory, "config-dir", "", "Path to a directory to specify drop-ins, allows the user to optionally specify additional configs to overwrite what is provided by default and in the KubeletConfigFile flag. Note: Set the 'KUBELET_CONFIG_DROPIN_DIR_ALPHA' environment variable to specify the directory. [default='']")
 	fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.")
 
 	fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+

cmd/kubelet/app/server.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"math"
 	"net"
 	"net/http"
@@ -33,6 +34,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-systemd/v22/daemon"
+	"github.com/imdario/mergo"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"google.golang.org/grpc/codes"
@@ -202,11 +204,24 @@ is checked every 20 seconds (also configurable with a flag).`,
 			}
 
 			// load kubelet config file, if provided
-			if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 {
-				kubeletConfig, err = loadConfigFile(configFile)
+			if len(kubeletFlags.KubeletConfigFile) > 0 {
+				kubeletConfig, err = loadConfigFile(kubeletFlags.KubeletConfigFile)
 				if err != nil {
-					return fmt.Errorf("failed to load kubelet config file, error: %w, path: %s", err, configFile)
+					return fmt.Errorf("failed to load kubelet config file, path: %s, error: %w", kubeletFlags.KubeletConfigFile, err)
 				}
+			}
+			// Merge the kubelet configurations if --config-dir is set
+			if len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
+				_, ok := os.LookupEnv("KUBELET_CONFIG_DROPIN_DIR_ALPHA")
+				if !ok {
+					return fmt.Errorf("flag %s specified but environment variable KUBELET_CONFIG_DROPIN_DIR_ALPHA not set, cannot start kubelet", kubeletFlags.KubeletDropinConfigDirectory)
+				}
+				if err := mergeKubeletConfigurations(kubeletConfig, kubeletFlags.KubeletDropinConfigDirectory); err != nil {
+					return fmt.Errorf("failed to merge kubelet configs: %w", err)
+				}
+			}
+
+			if len(kubeletFlags.KubeletConfigFile) > 0 || len(kubeletFlags.KubeletDropinConfigDirectory) > 0 {
 				// We must enforce flag precedence by re-parsing the command line into the new object.
 				// This is necessary to preserve backwards-compatibility across binary upgrades.
 				// See issue #56171 for more details.
@@ -288,6 +303,41 @@ is checked every 20 seconds (also configurable with a flag).`,
 	return cmd
 }
 
+// mergeKubeletConfigurations merges the provided drop-in configurations with the base kubelet configuration.
+// The drop-in configurations are processed in lexical order based on the file names. This means that the
+// configurations in files with lower numeric prefixes are applied first, followed by higher numeric prefixes.
+// For example, if the drop-in directory contains files named "10-config.conf" and "20-config.conf",
+// the configurations in "10-config.conf" will be applied first, and then the configurations in "20-config.conf" will be applied,
+// potentially overriding the previous values.
+func mergeKubeletConfigurations(kubeletConfig *kubeletconfiginternal.KubeletConfiguration, kubeletDropInConfigDir string) error {
+	const dropinFileExtension = ".conf"
+
+	// Walk through the drop-in directory and update the configuration for each file
+	err := filepath.WalkDir(kubeletDropInConfigDir, func(path string, info fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && filepath.Ext(info.Name()) == dropinFileExtension {
+			dropinConfig, err := loadConfigFile(path)
+			if err != nil {
+				return fmt.Errorf("failed to load kubelet dropin file, path: %s, error: %w", path, err)
+			}
+
+			// Merge dropinConfig with kubeletConfig
+			if err := mergo.Merge(kubeletConfig, dropinConfig, mergo.WithOverride); err != nil {
+				return fmt.Errorf("failed to merge kubelet drop-in config, path: %s, error: %w", path, err)
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed to walk through kubelet dropin directory %q: %w", kubeletDropInConfigDir, err)
+	}
+
+	return nil
+}
+
 // newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered
 // on it.
 func newFlagSetWithGlobals() *pflag.FlagSet {

cmd/kubelet/app/server_test.go

@@ -17,7 +17,14 @@ limitations under the License.
 package app
 
 import (
+	"os"
+	"path/filepath"
+	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/require"
+	"k8s.io/kubernetes/cmd/kubelet/app/options"
+	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
 )
 
 func TestValueOfAllocatableResources(t *testing.T) {
@@ -61,3 +68,194 @@ func TestValueOfAllocatableResources(t *testing.T) {
 		}
 	}
 }
+
+func TestMergeKubeletConfigurations(t *testing.T) {
+	testCases := []struct {
+		kubeletConfig           string
+		dropin1                 string
+		dropin2                 string
+		overwrittenConfigFields map[string]interface{}
+		cliArgs                 []string
+		name                    string
+	}{
+		{
+			kubeletConfig: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9080
+readOnlyPort: 10257
+`,
+			dropin1: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+`,
+			dropin2: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 8080
+readOnlyPort: 10255
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"Port":         int32(8080),
+				"ReadOnlyPort": int32(10255),
+			},
+			name: "kubelet.conf.d overrides kubelet.conf",
+		},
+		{
+			kubeletConfig: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+readOnlyPort: 10256
+kubeReserved:
+	memory: 70Mi
+`,
+			dropin1: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+readOnlyPort: 10255
+kubeReserved:
+  memory: 150Mi
+  cpu: 200m
+`,
+			dropin2: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+readOnlyPort: 10257
+kubeReserved:
+  memory: 100Mi
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"ReadOnlyPort": int32(10257),
+				"KubeReserved": map[string]string{
+					"cpu":    "200m",
+					"memory": "100Mi",
+				},
+			},
+			name: "kubelet.conf.d overrides kubelet.conf with subfield override",
+		},
+		{
+			kubeletConfig: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+clusterDNS:
+	- 192.168.1.3
+	- 192.168.1.4
+`,
+			dropin1: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+systemReserved:
+  memory: 1Gi
+`,
+			dropin2: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 8080
+readOnlyPort: 10255
+systemReserved:
+  memory: 2Gi
+clusterDNS:
+  - 192.168.1.1
+  - 192.168.1.5
+  - 192.168.1.8
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"Port":         int32(8080),
+				"ReadOnlyPort": int32(10255),
+				"SystemReserved": map[string]string{
+					"memory": "2Gi",
+				},
+				"ClusterDNS": []string{"192.168.1.1", "192.168.1.5", "192.168.1.8"},
+			},
+			name: "kubelet.conf.d overrides kubelet.conf with slices/lists",
+		},
+		{
+			dropin1: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+`,
+			dropin2: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 8080
+readOnlyPort: 10255
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"Port":         int32(8081),
+				"ReadOnlyPort": int32(10256),
+			},
+			cliArgs: []string{
+				"--port=8081",
+				"--read-only-port=10256",
+			},
+			name: "cli args override kubelet.conf.d",
+		},
+		{
+			kubeletConfig: `
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+port: 9090
+clusterDNS:
+	- 192.168.1.3
+`,
+			overwrittenConfigFields: map[string]interface{}{
+				"Port":       int32(9090),
+				"ClusterDNS": []string{"192.168.1.2"},
+			},
+			cliArgs: []string{
+				"--port=9090",
+				"--cluster-dns=192.168.1.2",
+			},
+			name: "cli args override kubelet.conf",
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			// Prepare a temporary directory for testing
+			tempDir := t.TempDir()
+
+			kubeletConfig := &kubeletconfiginternal.KubeletConfiguration{}
+			kubeletFlags := &options.KubeletFlags{}
+
+			if len(test.kubeletConfig) > 0 {
+				// Create the Kubeletconfig
+				kubeletConfFile := filepath.Join(tempDir, "kubelet.conf")
+				err := os.WriteFile(kubeletConfFile, []byte(test.kubeletConfig), 0644)
+				require.NoError(t, err, "failed to create config from a yaml file")
+				kubeletFlags.KubeletConfigFile = kubeletConfFile
+			}
+			if len(test.dropin1) > 0 || len(test.dropin2) > 0 {
+				// Create kubelet.conf.d directory and drop-in configuration files
+				kubeletConfDir := filepath.Join(tempDir, "kubelet.conf.d")
+				err := os.Mkdir(kubeletConfDir, 0755)
+				require.NoError(t, err, "Failed to create kubelet.conf.d directory")
+
+				err = os.WriteFile(filepath.Join(kubeletConfDir, "10-kubelet.conf"), []byte(test.dropin1), 0644)
+				require.NoError(t, err, "failed to create config from a yaml file")
+
+				err = os.WriteFile(filepath.Join(kubeletConfDir, "20-kubelet.conf"), []byte(test.dropin2), 0644)
+				require.NoError(t, err, "failed to create config from a yaml file")
+
+				// Merge the kubelet configurations
+				err = mergeKubeletConfigurations(kubeletConfig, kubeletConfDir)
+				require.NoError(t, err, "failed to merge kubelet drop-in configs")
+			}
+
+			// Use kubelet config flag precedence
+			err := kubeletConfigFlagPrecedence(kubeletConfig, test.cliArgs)
+			require.NoError(t, err, "failed to set the kubelet config flag precedence")
+
+			// Verify the merged configuration fields
+			for fieldName, expectedValue := range test.overwrittenConfigFields {
+				value := reflect.ValueOf(kubeletConfig).Elem()
+				field := value.FieldByName(fieldName)
+				require.Equal(t, expectedValue, field.Interface(), "Field mismatch: "+fieldName)
+			}
+		})
+	}
+}

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/gofuzz v1.2.0
 	github.com/google/uuid v1.3.0
+	github.com/imdario/mergo v0.3.6
 	github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5
 	github.com/libopenstorage/openstorage v1.0.0
 	github.com/lithammer/dedent v1.1.0
@@ -183,7 +184,6 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect