component-base: move v/vmodule/log-flush-frequency into LoggingConfig…

…uration

These three options are the ones from logs.AddFlags which are not deprecated.
Therefore it makes sense to make them available also via the configuration file
support in the one command which currently supports that (kubelet).

Long-term, all commands should use LoggingConfiguration, either with a
configuration file (as in kubelet) or via flags (kube-scheduler,
kube-apiserver, kube-controller-manager).

Short-term, both approaches have to be supported. As the majority of the
commands only use logs.AddFlags, that function by default continues to register
the flags and only leaves that to Options.AddFlags when explicitly requested.

A drive-by bug fix is done for log flushing: the periodic flushing called
klog.Flush and therefore missed explicit flushing of the newer logr
backend. This bug was never present in any release Kubernetes and therefore the
fix is not submitted in a separate PR.

cmd/kube-apiserver/app/server.go

@@ -56,6 +56,7 @@ import (
 	"k8s.io/client-go/util/keyutil"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
+	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration
 	"k8s.io/component-base/term"
 	"k8s.io/component-base/version"
@@ -146,7 +147,7 @@ cluster's shared state through which all other components interact.`,
 	fs := cmd.Flags()
 	namedFlagSets := s.Flags()
 	verflag.AddFlags(namedFlagSets.FlagSet("global"))
-	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
+	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
 	options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
 	for _, f := range namedFlagSets.FlagSets {
 		fs.AddFlagSet(f)

cmd/kube-controller-manager/app/controllermanager.go

@@ -55,6 +55,7 @@ import (
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/configz"
+	"k8s.io/component-base/logs"
 	"k8s.io/component-base/term"
 	"k8s.io/component-base/version"
 	"k8s.io/component-base/version/verflag"
@@ -160,7 +161,7 @@ controller, and serviceaccounts controller.`,
 	fs := cmd.Flags()
 	namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
 	verflag.AddFlags(namedFlagSets.FlagSet("global"))
-	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
+	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
 	registerLegacyGlobalFlags(namedFlagSets)
 	for _, f := range namedFlagSets.FlagSets {
 		fs.AddFlagSet(f)

cmd/kube-scheduler/app/server.go

@@ -91,7 +91,7 @@ for more information about scheduling and the kube-scheduler component.`,
 
 	nfs := opts.Flags
 	verflag.AddFlags(nfs.FlagSet("global"))
-	globalflag.AddGlobalFlags(nfs.FlagSet("global"), cmd.Name())
+	globalflag.AddGlobalFlags(nfs.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
 	fs := cmd.Flags()
 	for _, f := range nfs.FlagSets {
 		fs.AddFlagSet(f)

cmd/kubelet/app/options/globalflags.go

@@ -40,7 +40,7 @@ func AddGlobalFlags(fs *pflag.FlagSet) {
 	addCadvisorFlags(fs)
 	addCredentialProviderFlags(fs)
 	verflag.AddFlags(fs)
-	logs.AddFlags(fs)
+	logs.AddFlags(fs, logs.SkipLoggingConfigurationFlags())
 }
 
 // normalize replaces underscores with hyphens

pkg/kubelet/apis/config/helpers_test.go

@@ -187,6 +187,7 @@ var (
 		"HairpinMode",
 		"HealthzBindAddress",
 		"HealthzPort",
+		"Logging.FlushFrequency",
 		"Logging.Format",
 		"Logging.Options.JSON.InfoBufferSize.Quantity.Format",
 		"Logging.Options.JSON.InfoBufferSize.Quantity.d.Dec.scale",
@@ -197,6 +198,9 @@ var (
 		"Logging.Options.JSON.InfoBufferSize.Quantity.s",
 		"Logging.Options.JSON.SplitStream",
 		"Logging.Sanitization",
+		"Logging.VModule[*].FilePattern",
+		"Logging.VModule[*].Verbosity",
+		"Logging.Verbosity",
 		"TLSCipherSuites[*]",
 		"TLSMinVersion",
 		"IPTablesDropBit",

pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml

@@ -53,10 +53,12 @@ kind: KubeletConfiguration
 kubeAPIBurst: 10
 kubeAPIQPS: 5
 logging:
+  flushFrequency: 5000000000
   format: text
   options:
     json:
       infoBufferSize: "0"
+  verbosity: 0
 makeIPTablesUtilChains: true
 maxOpenFiles: 1000000
 maxPods: 110

pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml

@@ -53,10 +53,12 @@ kind: KubeletConfiguration
 kubeAPIBurst: 10
 kubeAPIQPS: 5
 logging:
+  flushFrequency: 5000000000
   format: text
   options:
     json:
       infoBufferSize: "0"
+  verbosity: 0
 makeIPTablesUtilChains: true
 maxOpenFiles: 1000000
 maxPods: 110

pkg/kubelet/apis/config/v1beta1/defaults_test.go

@@ -112,7 +112,8 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				EnforceNodeAllocatable:                    DefaultNodeAllocatableEnforcement,
 				VolumePluginDir:                           DefaultVolumePluginDir,
 				Logging: componentbaseconfigv1alpha1.LoggingConfiguration{
-					Format: "text",
+					Format:         "text",
+					FlushFrequency: 5 * time.Second,
 				},
 				EnableSystemLogHandler:  utilpointer.BoolPtr(true),
 				EnableProfilingHandler:  utilpointer.BoolPtr(true),
@@ -231,8 +232,9 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				ProviderID:                  "",
 				KernelMemcgNotification:     false,
 				Logging: componentbaseconfigv1alpha1.LoggingConfiguration{
-					Format:       "",
-					Sanitization: false,
+					Format:         "",
+					FlushFrequency: 5 * time.Second,
+					Sanitization:   false,
 				},
 				EnableSystemLogHandler:          utilpointer.Bool(false),
 				ShutdownGracePeriod:             zeroDuration,
@@ -327,8 +329,9 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				AllowedUnsafeSysctls:   []string{},
 				VolumePluginDir:        DefaultVolumePluginDir,
 				Logging: componentbaseconfigv1alpha1.LoggingConfiguration{
-					Format:       "text",
-					Sanitization: false,
+					Format:         "text",
+					FlushFrequency: 5 * time.Second,
+					Sanitization:   false,
 				},
 				EnableSystemLogHandler:  utilpointer.Bool(false),
 				ReservedMemory:          []v1beta1.MemoryReservation{},
@@ -468,8 +471,9 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				ProviderID:                  "provider-id",
 				KernelMemcgNotification:     true,
 				Logging: componentbaseconfigv1alpha1.LoggingConfiguration{
-					Format:       "json",
-					Sanitization: true,
+					Format:         "json",
+					FlushFrequency: 5 * time.Second,
+					Sanitization:   true,
 				},
 				EnableSystemLogHandler:          utilpointer.Bool(true),
 				ShutdownGracePeriod:             metav1.Duration{Duration: 60 * time.Second},
@@ -613,8 +617,9 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				ProviderID:                  "provider-id",
 				KernelMemcgNotification:     true,
 				Logging: componentbaseconfigv1alpha1.LoggingConfiguration{
-					Format:       "json",
-					Sanitization: true,
+					Format:         "json",
+					FlushFrequency: 5 * time.Second,
+					Sanitization:   true,
 				},
 				EnableSystemLogHandler:          utilpointer.Bool(true),
 				ShutdownGracePeriod:             metav1.Duration{Duration: 60 * time.Second},
@@ -706,7 +711,8 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				EnforceNodeAllocatable:                    DefaultNodeAllocatableEnforcement,
 				VolumePluginDir:                           DefaultVolumePluginDir,
 				Logging: componentbaseconfigv1alpha1.LoggingConfiguration{
-					Format: "text",
+					Format:         "text",
+					FlushFrequency: 5 * time.Second,
 				},
 				EnableSystemLogHandler:  utilpointer.BoolPtr(true),
 				EnableProfilingHandler:  utilpointer.BoolPtr(true),

staging/src/k8s.io/component-base/cli/globalflag/globalflags.go

@@ -27,8 +27,12 @@ import (
 // AddGlobalFlags explicitly registers flags that libraries (klog, verflag, etc.) register
 // against the global flagsets from "flag" and "k8s.io/klog/v2".
 // We do this in order to prevent unwanted flags from leaking into the component's flagset.
-func AddGlobalFlags(fs *pflag.FlagSet, name string) {
-	logs.AddFlags(fs)
+//
+// k8s.io/component-base/logs.SkipLoggingConfigurationFlags must be used as
+// option when the program also uses a LoggingConfiguration struct for
+// configuring logging. Then only flags not covered by that get added.
+func AddGlobalFlags(fs *pflag.FlagSet, name string, opts ...logs.Option) {
+	logs.AddFlags(fs, opts...)
 
 	fs.BoolP("help", "h", false, fmt.Sprintf("help for %s", name))
 }

staging/src/k8s.io/component-base/config/types.go

@@ -17,6 +17,13 @@ limitations under the License.
 package config
 
 import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -86,6 +93,17 @@ type LoggingConfiguration struct {
 	// Format Flag specifies the structure of log messages.
 	// default value of format is `text`
 	Format string
+	// Maximum number of seconds between log flushes. Ignored if the
+	// selected logging backend writes log messages without buffering.
+	FlushFrequency time.Duration
+	// Verbosity is the threshold that determines which log messages are
+	// logged. Default is zero which logs only the most important
+	// messages. Higher values enable additional messages. Error messages
+	// are always logged.
+	Verbosity VerbosityLevel
+	// VModule overrides the verbosity threshold for individual files.
+	// Only supported for "text" log format.
+	VModule VModuleConfiguration
 	// [Experimental] When enabled prevents logging of fields tagged as sensitive (passwords, keys, tokens).
 	// Runtime log sanitization may introduce significant computation overhead and therefore should not be enabled in production.`)
 	Sanitization bool
@@ -111,3 +129,86 @@ type JSONOptions struct {
 	// using split streams. The default is zero, which disables buffering.
 	InfoBufferSize resource.QuantityValue
 }
+
+// VModuleConfiguration is a collection of individual file names or patterns
+// and the corresponding verbosity threshold.
+type VModuleConfiguration []VModuleItem
+
+var _ pflag.Value = &VModuleConfiguration{}
+
+// VModuleItem defines verbosity for one or more files which match a certain
+// glob pattern.
+type VModuleItem struct {
+	// FilePattern is a base file name (i.e. minus the ".go" suffix and
+	// directory) or a "glob" pattern for such a name. It must not contain
+	// comma and equal signs because those are separators for the
+	// corresponding klog command line argument.
+	FilePattern string
+	// Verbosity is the threshold for log messages emitted inside files
+	// that match the pattern.
+	Verbosity VerbosityLevel
+}
+
+// String returns the -vmodule parameter (comma-separated list of pattern=N).
+func (vmodule *VModuleConfiguration) String() string {
+	var patterns []string
+	for _, item := range *vmodule {
+		patterns = append(patterns, fmt.Sprintf("%s=%d", item.FilePattern, item.Verbosity))
+	}
+	return strings.Join(patterns, ",")
+}
+
+// Set parses the -vmodule parameter (comma-separated list of pattern=N).
+func (vmodule *VModuleConfiguration) Set(value string) error {
+	// This code mirrors https://github.com/kubernetes/klog/blob/9ad246211af1ed84621ee94a26fcce0038b69cd1/klog.go#L287-L313
+
+	for _, pat := range strings.Split(value, ",") {
+		if len(pat) == 0 {
+			// Empty strings such as from a trailing comma can be ignored.
+			continue
+		}
+		patLev := strings.Split(pat, "=")
+		if len(patLev) != 2 || len(patLev[0]) == 0 || len(patLev[1]) == 0 {
+			return fmt.Errorf("%q does not have the pattern=N format", pat)
+		}
+		pattern := patLev[0]
+		// 31 instead of 32 to ensure that it also fits into int32.
+		v, err := strconv.ParseUint(patLev[1], 10, 31)
+		if err != nil {
+			return fmt.Errorf("parsing verbosity in %q: %v", pat, err)
+		}
+		*vmodule = append(*vmodule, VModuleItem{FilePattern: pattern, Verbosity: VerbosityLevel(v)})
+	}
+	return nil
+}
+
+func (vmodule *VModuleConfiguration) Type() string {
+	return "pattern=N,..."
+}
+
+// VerbosityLevel represents a klog or logr verbosity threshold.
+type VerbosityLevel uint32
+
+var _ pflag.Value = new(VerbosityLevel)
+
+func (l *VerbosityLevel) String() string {
+	return strconv.FormatInt(int64(*l), 10)
+}
+
+func (l *VerbosityLevel) Get() interface{} {
+	return *l
+}
+
+func (l *VerbosityLevel) Set(value string) error {
+	// Limited to int32 for compatibility with klog.
+	v, err := strconv.ParseUint(value, 10, 31)
+	if err != nil {
+		return err
+	}
+	*l = VerbosityLevel(v)
+	return nil
+}
+
+func (l *VerbosityLevel) Type() string {
+	return "Level"
+}