forked from kubernetes/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 1
/
podspec_mutation.go
197 lines (169 loc) · 8.36 KB
/
podspec_mutation.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"path/filepath"
"strings"
"k8s.io/api/core/v1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
const (
// selfHostedKubeConfigDir sets the directory where kubeconfig files for the scheduler and controller-manager should be mounted
// Due to how the projected volume mount works (can only be a full directory, not mount individual files), we must change this from
// the default as mounts cannot be nested (/etc/kubernetes would override /etc/kubernetes/pki)
selfHostedKubeConfigDir = "/etc/kubernetes/kubeconfig"
)
// PodSpecMutatorFunc is a function capable of mutating a PodSpec
type PodSpecMutatorFunc func(*v1.PodSpec)
// GetDefaultMutators gets the mutator functions that always should be used
func GetDefaultMutators() map[string][]PodSpecMutatorFunc {
return map[string][]PodSpecMutatorFunc{
kubeadmconstants.KubeAPIServer: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
setHostIPOnPodSpec,
},
kubeadmconstants.KubeControllerManager: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
kubeadmconstants.KubeScheduler: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
}
}
// GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed
func GetMutatorsFromFeatureGates(certsInSecrets bool) map[string][]PodSpecMutatorFunc {
// Here the map of different mutators to use for the control plane's podspec is stored
mutators := GetDefaultMutators()
if certsInSecrets {
// Some extra work to be done if we should store the control plane certificates in Secrets
// Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them
mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer)
mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager)
mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler)
}
return mutators
}
// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) {
// Get the mutator functions for the component in question, then loop through and execute them
mutatorsForComponent := mutators[name]
for _, mutateFunc := range mutatorsForComponent {
mutateFunc(podSpec)
}
}
// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the master label
func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) {
if podSpec.NodeSelector == nil {
podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}
return
}
podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = ""
}
// setMasterTolerationOnPodSpec makes the Pod tolerate the master taint
func setMasterTolerationOnPodSpec(podSpec *v1.PodSpec) {
if podSpec.Tolerations == nil {
podSpec.Tolerations = []v1.Toleration{kubeadmconstants.MasterToleration}
return
}
podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.MasterToleration)
}
// setHostIPOnPodSpec sets the environment variable HOST_IP using downward API
func setHostIPOnPodSpec(podSpec *v1.PodSpec) {
envVar := v1.EnvVar{
Name: "HOST_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
},
}
podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, envVar)
for i := range podSpec.Containers[0].Command {
if strings.Contains(podSpec.Containers[0].Command[i], "advertise-address") {
podSpec.Containers[0].Command[i] = "--advertise-address=$(HOST_IP)"
}
}
}
// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary
func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) {
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}
// setSelfHostedVolumesForAPIServer makes sure the self-hosted api server has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForAPIServer(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeCertificatesVolumeName {
podSpec.Volumes[i].VolumeSource = apiServerCertificatesVolumeSource()
}
}
}
// setSelfHostedVolumesForControllerManager makes sure the self-hosted controller manager has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForControllerManager(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeCertificatesVolumeName {
podSpec.Volumes[i].VolumeSource = controllerManagerCertificatesVolumeSource()
} else if v.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName)
}
}
// Change directory for the kubeconfig directory to selfHostedKubeConfigDir
for i, vm := range podSpec.Containers[0].VolumeMounts {
if vm.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir
}
}
// Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki)
// This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes
// don't support that.
podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string {
controllerManagerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
argMap["kubeconfig"] = controllerManagerKubeConfigPath
if _, ok := argMap["authentication-kubeconfig"]; ok {
argMap["authentication-kubeconfig"] = controllerManagerKubeConfigPath
}
if _, ok := argMap["authorization-kubeconfig"]; ok {
argMap["authorization-kubeconfig"] = controllerManagerKubeConfigPath
}
return argMap
})
}
// setSelfHostedVolumesForScheduler makes sure the self-hosted scheduler has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForScheduler(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName)
}
}
// Change directory for the kubeconfig directory to selfHostedKubeConfigDir
for i, vm := range podSpec.Containers[0].VolumeMounts {
if vm.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir
}
}
// Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki)
// This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes
// don't support that.
podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string {
argMap["kubeconfig"] = filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.SchedulerKubeConfigFileName)
return argMap
})
}