Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

kubelet: enable configurable rotation duration and parallel rotate #114301

Merged
merged 1 commit into from Feb 14, 2024
+391 −128
Merged

kubelet: enable configurable rotation duration and parallel rotate #114301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 13 additions & 0 deletions pkg/generated/openapi/zz_generated.openapi.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions pkg/kubelet/apis/config/fuzzer/fuzzer.go
View file
Open in desktop
Expand Up @@ -106,6 +106,8 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
obj.StaticPodURLHeader = make(map[string][]string)
obj.ContainerLogMaxFiles = 5
obj.ContainerLogMaxSize = "10Mi"
obj.ContainerLogMaxWorkers = 1
obj.ContainerLogMonitorInterval = metav1.Duration{Duration: 10 * time.Second}
obj.ConfigMapAndSecretChangeDetectionStrategy = "Watch"
obj.AllowedUnsafeSysctls = []string{}
obj.VolumePluginDir = kubeletconfigv1beta1.DefaultVolumePluginDir
Expand Down
2 changes: 2 additions & 0 deletions pkg/kubelet/apis/config/helpers_test.go
View file
Open in desktop
Expand Up @@ -185,6 +185,8 @@ var (
"ConfigMapAndSecretChangeDetectionStrategy",
"ContainerLogMaxFiles",
"ContainerLogMaxSize",
"ContainerLogMaxWorkers",
"ContainerLogMonitorInterval",
"ContentType",
"EnableContentionProfiling",
"EnableControllerAttachDetach",
Expand Down
2 changes: 2 additions & 0 deletions pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml
View file
Open in desktop
Expand Up @@ -17,6 +17,8 @@ cgroupsPerQOS: true
configMapAndSecretChangeDetectionStrategy: Watch
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
containerLogMaxWorkers: 1
containerLogMonitorInterval: 10s
containerRuntimeEndpoint: unix:///run/containerd/containerd.sock
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
Expand Down
2 changes: 2 additions & 0 deletions pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml
View file
Open in desktop
Expand Up @@ -17,6 +17,8 @@ cgroupsPerQOS: true
configMapAndSecretChangeDetectionStrategy: Watch
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
containerLogMaxWorkers: 1
containerLogMonitorInterval: 10s
containerRuntimeEndpoint: unix:///run/containerd/containerd.sock
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
Expand Down
5 changes: 5 additions & 0 deletions pkg/kubelet/apis/config/types.go
View file
Open in desktop
Expand Up @@ -347,6 +347,11 @@ type KubeletConfiguration struct {
ContainerLogMaxSize string
// Maximum number of container log files that can be present for a container.
ContainerLogMaxFiles int32
// Maximum number of concurrent log rotation workers to spawn for processing the log rotation
// requests
ContainerLogMaxWorkers int32
harshanarayana marked this conversation as resolved.
Show resolved Hide resolved
// Interval at which the container logs are monitored for rotation
ContainerLogMonitorInterval metav1.Duration
// ConfigMapAndSecretChangeDetectionStrategy is a mode in which config map and secret managers are running.
ConfigMapAndSecretChangeDetectionStrategy ResourceChangeDetectionStrategy
// A comma separated allowlist of unsafe sysctls or sysctl patterns (ending in `*`).
Expand Down
6 changes: 6 additions & 0 deletions pkg/kubelet/apis/config/v1beta1/defaults.go
View file
Open in desktop
Expand Up @@ -239,6 +239,12 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura
if obj.ContainerLogMaxFiles == nil {
obj.ContainerLogMaxFiles = utilpointer.Int32(5)
}
if obj.ContainerLogMaxWorkers == nil {
obj.ContainerLogMaxWorkers = utilpointer.Int32(1)
}
if obj.ContainerLogMonitorInterval == nil {
obj.ContainerLogMonitorInterval = &metav1.Duration{Duration: 10 * time.Second}
}
if obj.ConfigMapAndSecretChangeDetectionStrategy == "" {
obj.ConfigMapAndSecretChangeDetectionStrategy = kubeletconfigv1beta1.WatchChangeDetectionStrategy
}
Expand Down
140 changes: 78 additions & 62 deletions pkg/kubelet/apis/config/v1beta1/defaults_test.go
View file
Open in desktop
Expand Up @@ -112,6 +112,8 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
FailSwapOn: utilpointer.Bool(true),
ContainerLogMaxSize: "10Mi",
ContainerLogMaxFiles: utilpointer.Int32(5),
ContainerLogMaxWorkers: utilpointer.Int32(1),
ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second},
ConfigMapAndSecretChangeDetectionStrategy: v1beta1.WatchChangeDetectionStrategy,
EnforceNodeAllocatable: DefaultNodeAllocatableEnforcement,
VolumePluginDir: DefaultVolumePluginDir,
Expand Down Expand Up @@ -227,6 +229,8 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: ""},
ContainerLogMaxSize: "",
ContainerLogMaxFiles: utilpointer.Int32(0),
ContainerLogMaxWorkers: utilpointer.Int32(1),
ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second},
ConfigMapAndSecretChangeDetectionStrategy: v1beta1.WatchChangeDetectionStrategy,
SystemReserved: map[string]string{},
KubeReserved: map[string]string{},
Expand Down Expand Up @@ -278,67 +282,69 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
CacheUnauthorizedTTL: metav1.Duration{Duration: 30 * time.Second},
},
},
RegistryPullQPS: utilpointer.Int32(0),
RegistryBurst: 10,
EventRecordQPS: utilpointer.Int32(0),
EventBurst: 100,
EnableDebuggingHandlers: utilpointer.Bool(false),
HealthzPort: utilpointer.Int32(0),
HealthzBindAddress: "127.0.0.1",
OOMScoreAdj: utilpointer.Int32(0),
ClusterDNS: []string{},
StreamingConnectionIdleTimeout: metav1.Duration{Duration: 4 * time.Hour},
NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Second},
NodeStatusReportFrequency: metav1.Duration{Duration: 5 * time.Minute},
NodeLeaseDurationSeconds: 40,
ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock",
ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute},
ImageGCHighThresholdPercent: utilpointer.Int32(0),
ImageGCLowThresholdPercent: utilpointer.Int32(0),
VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute},
CgroupsPerQOS: utilpointer.Bool(false),
CgroupDriver: "cgroupfs",
CPUManagerPolicy: "none",
CPUManagerPolicyOptions: map[string]string{},
CPUManagerReconcilePeriod: metav1.Duration{Duration: 10 * time.Second},
MemoryManagerPolicy: v1beta1.NoneMemoryManagerPolicy,
TopologyManagerPolicy: v1beta1.NoneTopologyManagerPolicy,
TopologyManagerScope: v1beta1.ContainerTopologyManagerScope,
QOSReserved: map[string]string{},
RuntimeRequestTimeout: metav1.Duration{Duration: 2 * time.Minute},
HairpinMode: v1beta1.PromiscuousBridge,
MaxPods: 110,
PodPidsLimit: utilpointer.Int64(0),
ResolverConfig: utilpointer.String(""),
CPUCFSQuota: utilpointer.Bool(false),
CPUCFSQuotaPeriod: &zeroDuration,
NodeStatusMaxImages: utilpointer.Int32(0),
MaxOpenFiles: 1000000,
ContentType: "application/vnd.kubernetes.protobuf",
KubeAPIQPS: utilpointer.Int32(0),
KubeAPIBurst: 100,
SerializeImagePulls: utilpointer.Bool(false),
MaxParallelImagePulls: nil,
EvictionHard: map[string]string{},
EvictionSoft: map[string]string{},
EvictionSoftGracePeriod: map[string]string{},
EvictionPressureTransitionPeriod: metav1.Duration{Duration: 5 * time.Minute},
EvictionMinimumReclaim: map[string]string{},
EnableControllerAttachDetach: utilpointer.Bool(false),
MakeIPTablesUtilChains: utilpointer.Bool(false),
IPTablesMasqueradeBit: utilpointer.Int32(0),
IPTablesDropBit: utilpointer.Int32(0),
FeatureGates: map[string]bool{},
FailSwapOn: utilpointer.Bool(false),
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: ""},
ContainerLogMaxSize: "10Mi",
ContainerLogMaxFiles: utilpointer.Int32(0),
RegistryPullQPS: utilpointer.Int32(0),
RegistryBurst: 10,
EventRecordQPS: utilpointer.Int32(0),
EventBurst: 100,
EnableDebuggingHandlers: utilpointer.Bool(false),
HealthzPort: utilpointer.Int32(0),
HealthzBindAddress: "127.0.0.1",
OOMScoreAdj: utilpointer.Int32(0),
ClusterDNS: []string{},
StreamingConnectionIdleTimeout: metav1.Duration{Duration: 4 * time.Hour},
NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Second},
NodeStatusReportFrequency: metav1.Duration{Duration: 5 * time.Minute},
NodeLeaseDurationSeconds: 40,
ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock",
ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute},
ImageGCHighThresholdPercent: utilpointer.Int32(0),
ImageGCLowThresholdPercent: utilpointer.Int32(0),
VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute},
CgroupsPerQOS: utilpointer.Bool(false),
CgroupDriver: "cgroupfs",
CPUManagerPolicy: "none",
CPUManagerPolicyOptions: map[string]string{},
CPUManagerReconcilePeriod: metav1.Duration{Duration: 10 * time.Second},
MemoryManagerPolicy: v1beta1.NoneMemoryManagerPolicy,
TopologyManagerPolicy: v1beta1.NoneTopologyManagerPolicy,
TopologyManagerScope: v1beta1.ContainerTopologyManagerScope,
QOSReserved: map[string]string{},
RuntimeRequestTimeout: metav1.Duration{Duration: 2 * time.Minute},
HairpinMode: v1beta1.PromiscuousBridge,
MaxPods: 110,
PodPidsLimit: utilpointer.Int64(0),
ResolverConfig: utilpointer.String(""),
CPUCFSQuota: utilpointer.Bool(false),
CPUCFSQuotaPeriod: &zeroDuration,
NodeStatusMaxImages: utilpointer.Int32(0),
MaxOpenFiles: 1000000,
ContentType: "application/vnd.kubernetes.protobuf",
KubeAPIQPS: utilpointer.Int32(0),
KubeAPIBurst: 100,
SerializeImagePulls: utilpointer.Bool(false),
MaxParallelImagePulls: nil,
EvictionHard: map[string]string{},
EvictionSoft: map[string]string{},
EvictionSoftGracePeriod: map[string]string{},
EvictionPressureTransitionPeriod: metav1.Duration{Duration: 5 * time.Minute},
EvictionMinimumReclaim: map[string]string{},
EnableControllerAttachDetach: utilpointer.Bool(false),
MakeIPTablesUtilChains: utilpointer.Bool(false),
IPTablesMasqueradeBit: utilpointer.Int32(0),
IPTablesDropBit: utilpointer.Int32(0),
FeatureGates: map[string]bool{},
FailSwapOn: utilpointer.Bool(false),
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: ""},
ContainerLogMaxSize: "10Mi",
ContainerLogMaxFiles: utilpointer.Int32(0),
ContainerLogMaxWorkers: utilpointer.Int32(1),
ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second},
ConfigMapAndSecretChangeDetectionStrategy: v1beta1.WatchChangeDetectionStrategy,
SystemReserved: map[string]string{},
KubeReserved: map[string]string{},
EnforceNodeAllocatable: []string{},
AllowedUnsafeSysctls: []string{},
VolumePluginDir: DefaultVolumePluginDir,
SystemReserved: map[string]string{},
KubeReserved: map[string]string{},
EnforceNodeAllocatable: []string{},
AllowedUnsafeSysctls: []string{},
VolumePluginDir: DefaultVolumePluginDir,
Logging: logsapi.LoggingConfiguration{
Format: "text",
FlushFrequency: logsapi.TimeOrMetaDuration{Duration: metav1.Duration{Duration: 5 * time.Second}, SerializeAsString: true},
Expand Down Expand Up @@ -465,6 +471,8 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "UnlimitedSwap"},
ContainerLogMaxSize: "1Mi",
ContainerLogMaxFiles: utilpointer.Int32(1),
ContainerLogMaxWorkers: utilpointer.Int32(1),
ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second},
ConfigMapAndSecretChangeDetectionStrategy: v1beta1.TTLCacheChangeDetectionStrategy,
SystemReserved: map[string]string{
"memory": "1Gi",
Expand Down Expand Up @@ -611,6 +619,8 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "UnlimitedSwap"},
ContainerLogMaxSize: "1Mi",
ContainerLogMaxFiles: utilpointer.Int32(1),
ContainerLogMaxWorkers: utilpointer.Int32(1),
ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second},
ConfigMapAndSecretChangeDetectionStrategy: v1beta1.TTLCacheChangeDetectionStrategy,
SystemReserved: map[string]string{
"memory": "1Gi",
Expand Down Expand Up @@ -720,7 +730,9 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
IPTablesDropBit: utilpointer.Int32Ptr(DefaultIPTablesDropBit),
FailSwapOn: utilpointer.Bool(true),
ContainerLogMaxSize: "10Mi",
ContainerLogMaxFiles: utilpointer.Int32Ptr(5),
ContainerLogMaxFiles: utilpointer.Int32(5),
ContainerLogMaxWorkers: utilpointer.Int32(1),
ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second},
ConfigMapAndSecretChangeDetectionStrategy: v1beta1.WatchChangeDetectionStrategy,
EnforceNodeAllocatable: DefaultNodeAllocatableEnforcement,
VolumePluginDir: DefaultVolumePluginDir,
Expand Down Expand Up @@ -809,7 +821,9 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
IPTablesDropBit: utilpointer.Int32Ptr(DefaultIPTablesDropBit),
FailSwapOn: utilpointer.Bool(true),
ContainerLogMaxSize: "10Mi",
ContainerLogMaxFiles: utilpointer.Int32Ptr(5),
ContainerLogMaxFiles: utilpointer.Int32(5),
ContainerLogMaxWorkers: utilpointer.Int32(1),
ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second},
ConfigMapAndSecretChangeDetectionStrategy: v1beta1.WatchChangeDetectionStrategy,
EnforceNodeAllocatable: DefaultNodeAllocatableEnforcement,
VolumePluginDir: DefaultVolumePluginDir,
Expand Down Expand Up @@ -899,6 +913,8 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
FailSwapOn: utilpointer.Bool(true),
ContainerLogMaxSize: "10Mi",
ContainerLogMaxFiles: utilpointer.Int32(5),
ContainerLogMaxWorkers: utilpointer.Int32(1),
ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second},
ConfigMapAndSecretChangeDetectionStrategy: v1beta1.WatchChangeDetectionStrategy,
EnforceNodeAllocatable: DefaultNodeAllocatableEnforcement,
VolumePluginDir: DefaultVolumePluginDir,
Expand Down
12 changes: 12 additions & 0 deletions pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions pkg/kubelet/apis/config/validation/validation.go
View file
Open in desktop
Expand Up @@ -279,5 +279,12 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur
fmt.Errorf("invalid configuration: enableSystemLogHandler is required for enableSystemLogQuery"))
}

if kc.ContainerLogMaxWorkers < 1 {
allErrors = append(allErrors, fmt.Errorf("invalid configuration: containerLogMaxWorkers must be greater than or equal to 1"))
}

if kc.ContainerLogMonitorInterval.Duration.Seconds() < 3 {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How did you decide on 3 seconds?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cartermckinnon It was based on this comment #114301 (comment)

allErrors = append(allErrors, fmt.Errorf("invalid configuration: containerLogMonitorInterval must be a positive time duration greater than or equal to 3s"))
}
return utilerrors.NewAggregate(allErrors)
}