Skip to content

Commit

Permalink
kubelet: use cache configMap and secrets change strategy
Browse files Browse the repository at this point in the history 
Watched based strategy has a couple bugs, 1) golang http2 max
streams blocking when the stream limit is reached and 2) the kubelet not
cleaning up watches for terminated pods.

This patch configures the cache based strategy. Once golang 1.12 is in
use, and the kubelet patch is merged we can use the watch based
strategy.

ref: kubernetes/kubernetes#74412
ref: kubernetes/kubernetes#74412 (comment)
  • Loading branch information
@rphillips
rphillips committed Mar 4, 2019
1 parent fc1a99e commit 33e69a7
Show file tree
Hide file tree
Showing 11 changed files with 15 additions and 8 deletions.
3 changes: 3 additions & 0 deletions pkg/controller/kubelet-config/kubelet_config_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ var blacklistKubeletConfigurationFields = []string{
"CgroupDriver",
"ClusterDNS",
"ClusterDomain",
// Bugfix to force cache based configmap and secret watches
// https://github.com/kubernetes/kubernetes/issues/74412
"ConfigMapAndSecretChangeDetectionStrategy",
"RuntimeRequestTimeout",
"StaticPodPath",
}
Expand Down
2 changes: 1 addition & 1 deletion ...plate/test_data/templates/master/01-master-kubelet/aws/files/-etc-kubernetes-kubelet.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
contents:
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AmaxPods%3A%20250%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0A%23%201.12%2C%201.13%20ConfigMap%20watch%20fix%20https%3A%2F%2Fgithub.com%2Fkubernetes%2Fkubernetes%2Fissues%2F74412%23issuecomment-468437599%0AconfigMapAndSecretChangeDetectionStrategy%3A%20%22Cache%22%0AmaxPods%3A%20250%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
verification: {}
filesystem: root
mode: 420
Expand Down
2 changes: 1 addition & 1 deletion ...e/test_data/templates/master/01-master-kubelet/libvirt/files/-etc-kubernetes-kubelet.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
contents:
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AmaxPods%3A%20250%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0A%23%201.12%2C%201.13%20ConfigMap%20watch%20fix%20https%3A%2F%2Fgithub.com%2Fkubernetes%2Fkubernetes%2Fissues%2F74412%23issuecomment-468437599%0AconfigMapAndSecretChangeDetectionStrategy%3A%20%22Cache%22%0AmaxPods%3A%20250%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
verification: {}
filesystem: root
mode: 420
Expand Down
2 changes: 1 addition & 1 deletion ...late/test_data/templates/master/01-master-kubelet/none/files/-etc-kubernetes-kubelet.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
contents:
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AmaxPods%3A%20250%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0A%23%201.12%2C%201.13%20ConfigMap%20watch%20fix%20https%3A%2F%2Fgithub.com%2Fkubernetes%2Fkubernetes%2Fissues%2F74412%23issuecomment-468437599%0AconfigMapAndSecretChangeDetectionStrategy%3A%20%22Cache%22%0AmaxPods%3A%20250%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
verification: {}
filesystem: root
mode: 420
Expand Down
2 changes: 1 addition & 1 deletion ...test_data/templates/master/01-master-kubelet/openstack/files/-etc-kubernetes-kubelet.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
contents:
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AmaxPods%3A%20250%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0A%23%201.12%2C%201.13%20ConfigMap%20watch%20fix%20https%3A%2F%2Fgithub.com%2Fkubernetes%2Fkubernetes%2Fissues%2F74412%23issuecomment-468437599%0AconfigMapAndSecretChangeDetectionStrategy%3A%20%22Cache%22%0AmaxPods%3A%20250%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
verification: {}
filesystem: root
mode: 420
Expand Down
2 changes: 1 addition & 1 deletion ...plate/test_data/templates/worker/01-worker-kubelet/aws/files/-etc-kubernetes-kubelet.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
contents:
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AmaxPods%3A%20250%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0A%23%201.12%2C%201.13%20ConfigMap%20watch%20fix%20https%3A%2F%2Fgithub.com%2Fkubernetes%2Fkubernetes%2Fissues%2F74412%23issuecomment-468437599%0AconfigMapAndSecretChangeDetectionStrategy%3A%20%22Cache%22%0AmaxPods%3A%20250%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
verification: {}
filesystem: root
mode: 420
Expand Down
2 changes: 1 addition & 1 deletion ...e/test_data/templates/worker/01-worker-kubelet/libvirt/files/-etc-kubernetes-kubelet.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
contents:
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AmaxPods%3A%20250%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0A%23%201.12%2C%201.13%20ConfigMap%20watch%20fix%20https%3A%2F%2Fgithub.com%2Fkubernetes%2Fkubernetes%2Fissues%2F74412%23issuecomment-468437599%0AconfigMapAndSecretChangeDetectionStrategy%3A%20%22Cache%22%0AmaxPods%3A%20250%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
verification: {}
filesystem: root
mode: 420
Expand Down
2 changes: 1 addition & 1 deletion ...late/test_data/templates/worker/01-worker-kubelet/none/files/-etc-kubernetes-kubelet.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
contents:
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AmaxPods%3A%20250%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0A%23%201.12%2C%201.13%20ConfigMap%20watch%20fix%20https%3A%2F%2Fgithub.com%2Fkubernetes%2Fkubernetes%2Fissues%2F74412%23issuecomment-468437599%0AconfigMapAndSecretChangeDetectionStrategy%3A%20%22Cache%22%0AmaxPods%3A%20250%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
verification: {}
filesystem: root
mode: 420
Expand Down
2 changes: 1 addition & 1 deletion ...test_data/templates/worker/01-worker-kubelet/openstack/files/-etc-kubernetes-kubelet.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
contents:
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0AmaxPods%3A%20250%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
source: data:,kind%3A%20KubeletConfiguration%0AapiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A%20%20-%2010.3.0.10%0AclusterDomain%3A%20cluster.local%0A%23%201.12%2C%201.13%20ConfigMap%20watch%20fix%20https%3A%2F%2Fgithub.com%2Fkubernetes%2Fkubernetes%2Fissues%2F74412%23issuecomment-468437599%0AconfigMapAndSecretChangeDetectionStrategy%3A%20%22Cache%22%0AmaxPods%3A%20250%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%2010m%0AserializeImagePulls%3A%20false%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AsystemReserved%3A%0A%20%20cpu%3A%20500m%0A%20%20memory%3A%20500Mi%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AserverTLSBootstrap%3A%20true%0A
verification: {}
filesystem: root
mode: 420
Expand Down
2 changes: 2 additions & 0 deletions templates/master/01-master-kubelet/_base/files/kubelet.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ contents:
clusterDNS:
- {{.ClusterDNSIP}}
clusterDomain: cluster.local
# 1.12, 1.13 ConfigMap watch fix https://github.com/kubernetes/kubernetes/issues/74412#issuecomment-468437599
configMapAndSecretChangeDetectionStrategy: "Cache"
maxPods: 250
runtimeRequestTimeout: 10m
serializeImagePulls: false
Expand Down
2 changes: 2 additions & 0 deletions templates/worker/01-worker-kubelet/_base/files/kubelet.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ contents:
clusterDNS:
- {{.ClusterDNSIP}}
clusterDomain: cluster.local
# 1.12, 1.13 ConfigMap watch fix https://github.com/kubernetes/kubernetes/issues/74412#issuecomment-468437599
configMapAndSecretChangeDetectionStrategy: "Cache"
maxPods: 250
rotateCertificates: true
runtimeRequestTimeout: 10m
Expand Down

1 comment on commit 33e69a7

@RobertKrawitz
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to see an explicit comment that this should be removed when we cut over to go1.12. If for whatever reason we decide to do that earlier than we rebase to 1.14, that's the point at which we no longer need to block it.

Please sign in to comment.