forked from k3s-io/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 0
/
volumes.go
226 lines (198 loc) · 11.3 KB
/
volumes.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"os"
"path/filepath"
"strings"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
)
const (
caCertsVolumeName = "ca-certs"
caCertsVolumePath = "/etc/ssl/certs"
caCertsPkiVolumeName = "ca-certs-etc-pki"
flexvolumeDirVolumeName = "flexvolume-dir"
cloudConfigVolumeName = "cloud-config"
flexvolumeDirVolumePath = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"
)
// caCertsPkiVolumePath specifies the path that can be conditionally mounted into the apiserver and controller-manager containers
// as /etc/ssl/certs might be a symlink to it. It's a variable since it may be changed in unit testing. This var MUST NOT be changed
// in normal codepaths during runtime.
var caCertsPkiVolumePath = "/etc/pki"
// getHostPathVolumesForTheControlPlane gets the required hostPath volumes and mounts for the control plane
func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) controlPlaneHostPathMounts {
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
hostPathFileOrCreate := v1.HostPathFileOrCreate
mounts := newControlPlaneHostPathMounts()
// HostPath volumes for the API Server
// Read-only mount for the certificates directory
// TODO: Always mount the K8s Certificates directory to a static path inside of the container
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate)
// Read-only mount for the ca certs (/etc/ssl/certs) directory
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate)
// If external etcd is specified, mount the directories needed for accessing the CA/serving certs and the private key
if len(cfg.Etcd.Endpoints) != 0 {
etcdVols, etcdVolMounts := getEtcdCertVolumes(cfg.Etcd, cfg.CertificatesDir)
mounts.AddHostPathMounts(kubeadmconstants.KubeAPIServer, etcdVols, etcdVolMounts)
}
// HostPath volumes for the controller manager
// Read-only mount for the certificates directory
// TODO: Always mount the K8s Certificates directory to a static path inside of the container
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate)
// Read-only mount for the ca certs (/etc/ssl/certs) directory
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate)
// Read-only mount for the controller manager kubeconfig file
controllerManagerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeConfigVolumeName, controllerManagerKubeConfigFile, controllerManagerKubeConfigFile, true, &hostPathFileOrCreate)
// Read-only mount of the cloud config file if present
if cfg.CloudProvider != "" {
if _, err := os.Stat(DefaultCloudConfigPath); err == nil {
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, cloudConfigVolumeName, DefaultCloudConfigPath, DefaultCloudConfigPath, true, &hostPathFileOrCreate)
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, cloudConfigVolumeName, DefaultCloudConfigPath, DefaultCloudConfigPath, true, &hostPathFileOrCreate)
}
}
// Mount for the flexvolume directory (/usr/libexec/kubernetes/kubelet-plugins/volume/exec) directory
// Flexvolume dir must NOT be readonly as it is used for third-party plugins to integrate with their storage backends via unix domain socket.
if stat, err := os.Stat(flexvolumeDirVolumePath); err == nil && stat.IsDir() {
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, flexvolumeDirVolumeName, flexvolumeDirVolumePath, flexvolumeDirVolumePath, false, &hostPathDirectoryOrCreate)
}
// HostPath volumes for the scheduler
// Read-only mount for the scheduler kubeconfig file
schedulerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName)
mounts.NewHostPathMount(kubeadmconstants.KubeScheduler, kubeadmconstants.KubeConfigVolumeName, schedulerKubeConfigFile, schedulerKubeConfigFile, true, &hostPathFileOrCreate)
// On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed
// due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/
if isPkiVolumeMountNeeded() {
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true, &hostPathDirectoryOrCreate)
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true, &hostPathDirectoryOrCreate)
}
// Merge user defined mounts and ensure unique volume and volume mount
// names
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeAPIServer, cfg.APIServerExtraVolumes, true, &hostPathDirectoryOrCreate)
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeControllerManager, cfg.ControllerManagerExtraVolumes, true, &hostPathDirectoryOrCreate)
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeScheduler, cfg.SchedulerExtraVolumes, true, &hostPathDirectoryOrCreate)
return mounts
}
// controlPlaneHostPathMounts is a helper struct for handling all the control plane's hostPath mounts in an easy way
type controlPlaneHostPathMounts struct {
// volumes is a nested map that forces a unique volumes. The outer map's
// keys are a string that should specify the target component to add the
// volume to. The values (inner map) of the outer map are maps with string
// keys and v1.Volume values. The inner map's key should specify the volume
// name.
volumes map[string]map[string]v1.Volume
// volumeMounts is a nested map that forces a unique volume mounts. The
// outer map's keys are a string that should specify the target component
// to add the volume mount to. The values (inner map) of the outer map are
// maps with string keys and v1.VolumeMount values. The inner map's key
// should specify the volume mount name.
volumeMounts map[string]map[string]v1.VolumeMount
}
func newControlPlaneHostPathMounts() controlPlaneHostPathMounts {
return controlPlaneHostPathMounts{
volumes: map[string]map[string]v1.Volume{},
volumeMounts: map[string]map[string]v1.VolumeMount{},
}
}
func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool, hostPathType *v1.HostPathType) {
vol := staticpodutil.NewVolume(mountName, hostPath, hostPathType)
c.addComponentVolume(component, vol)
volMount := staticpodutil.NewVolumeMount(mountName, containerPath, readOnly)
c.addComponentVolumeMount(component, volMount)
}
func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols []v1.Volume, volMounts []v1.VolumeMount) {
for _, v := range vols {
c.addComponentVolume(component, v)
}
for _, v := range volMounts {
c.addComponentVolumeMount(component, v)
}
}
// AddExtraHostPathMounts adds host path mounts and overwrites the default
// paths in the case that a user specifies the same volume/volume mount name.
func (c *controlPlaneHostPathMounts) AddExtraHostPathMounts(component string, extraVols []kubeadmapi.HostPathMount, readOnly bool, hostPathType *v1.HostPathType) {
for _, extraVol := range extraVols {
fmt.Printf("[controlplane] Adding extra host path mount %q to %q\n", extraVol.Name, component)
c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, readOnly, hostPathType)
}
}
func (c *controlPlaneHostPathMounts) GetVolumes(component string) map[string]v1.Volume {
return c.volumes[component]
}
func (c *controlPlaneHostPathMounts) GetVolumeMounts(component string) map[string]v1.VolumeMount {
return c.volumeMounts[component]
}
func (c *controlPlaneHostPathMounts) addComponentVolume(component string, vol v1.Volume) {
if _, ok := c.volumes[component]; !ok {
c.volumes[component] = map[string]v1.Volume{}
}
c.volumes[component][vol.Name] = vol
}
func (c *controlPlaneHostPathMounts) addComponentVolumeMount(component string, volMount v1.VolumeMount) {
if _, ok := c.volumeMounts[component]; !ok {
c.volumeMounts[component] = map[string]v1.VolumeMount{}
}
c.volumeMounts[component][volMount.Name] = volMount
}
// getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster
func getEtcdCertVolumes(etcdCfg kubeadmapi.Etcd, k8sCertificatesDir string) ([]v1.Volume, []v1.VolumeMount) {
certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile}
certDirs := sets.NewString()
for _, certPath := range certPaths {
certDir := filepath.Dir(certPath)
// Ignore ".", which is the result of passing an empty path.
// Also ignore the cert directories that already may be mounted; /etc/ssl/certs, /etc/pki or Kubernetes CertificatesDir
// If the etcd certs are in there, it's okay, we don't have to do anything
if certDir == "." || strings.HasPrefix(certDir, caCertsVolumePath) || strings.HasPrefix(certDir, caCertsPkiVolumePath) || strings.HasPrefix(certDir, k8sCertificatesDir) {
continue
}
// Filter out any existing hostpath mounts in the list that contains a subset of the path
alreadyExists := false
for _, existingCertDir := range certDirs.List() {
// If the current directory is a parent of an existing one, remove the already existing one
if strings.HasPrefix(existingCertDir, certDir) {
certDirs.Delete(existingCertDir)
} else if strings.HasPrefix(certDir, existingCertDir) {
// If an existing directory is a parent of the current one, don't add the current one
alreadyExists = true
}
}
if alreadyExists {
continue
}
certDirs.Insert(certDir)
}
volumes := []v1.Volume{}
volumeMounts := []v1.VolumeMount{}
pathType := v1.HostPathDirectoryOrCreate
for i, certDir := range certDirs.List() {
name := fmt.Sprintf("etcd-certs-%d", i)
volumes = append(volumes, staticpodutil.NewVolume(name, certDir, &pathType))
volumeMounts = append(volumeMounts, staticpodutil.NewVolumeMount(name, certDir, true))
}
return volumes, volumeMounts
}
// isPkiVolumeMountNeeded specifies whether /etc/pki should be host-mounted into the containers
// On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed
// due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/
func isPkiVolumeMountNeeded() bool {
if _, err := os.Stat(caCertsPkiVolumePath); err == nil {
return true
}
return false
}