forked from kubernetes/kops
-
Notifications
You must be signed in to change notification settings - Fork 0
/
cluster.go
458 lines (385 loc) · 16.1 KB
/
cluster.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
package api
import (
"encoding/binary"
"fmt"
"github.com/golang/glog"
"k8s.io/kops/util/pkg/vfs"
k8sapi "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/unversioned"
"net"
"strconv"
"strings"
)
type Cluster struct {
unversioned.TypeMeta `json:",inline"`
k8sapi.ObjectMeta `json:"metadata,omitempty"`
Spec ClusterSpec `json:"spec,omitempty"`
}
type ClusterList struct {
Items []Cluster `json:"items"`
}
type ClusterSpec struct {
// The Channel we are following
Channel string `json:"channel,omitempty"`
// ConfigBase is the path where we store configuration for the cluster
// This might be different that the location when the cluster spec itself is stored,
// both because this must be accessible to the cluster,
// and because it might be on a different cloud or storage system (etcd vs S3)
ConfigBase string `json:"configBase,omitempty"`
// The CloudProvider to use (aws or gce)
CloudProvider string `json:"cloudProvider,omitempty"`
// The version of kubernetes to install (optional, and can be a "spec" like stable)
KubernetesVersion string `json:"kubernetesVersion,omitempty"`
//
//// The Node initializer technique to use: cloudinit or nodeup
//NodeInit string `json:",omitempty"`
// Configuration of zones we are targeting
Zones []*ClusterZoneSpec `json:"zones,omitempty"`
//Region string `json:",omitempty"`
// Project is the cloud project we should use, required on GCE
Project string `json:"project,omitempty"`
// MasterPublicName is the external DNS name for the master nodes
MasterPublicName string `json:"masterPublicName,omitempty"`
// MasterInternalName is the internal DNS name for the master nodes
MasterInternalName string `json:"masterInternalName,omitempty"`
// The CIDR used for the AWS VPC / GCE Network, or otherwise allocated to k8s
// This is a real CIDR, not the internal k8s network
NetworkCIDR string `json:"networkCIDR,omitempty"`
// NetworkID is an identifier of a network, if we want to reuse/share an existing network (e.g. an AWS VPC)
NetworkID string `json:"networkID,omitempty"`
// SecretStore is the VFS path to where secrets are stored
SecretStore string `json:"secretStore,omitempty"`
// KeyStore is the VFS path to where SSL keys and certificates are stored
KeyStore string `json:"keyStore,omitempty"`
// ConfigStore is the VFS path to where the configuration (CloudConfig, NodeSetConfig etc) is stored
ConfigStore string `json:"configStore,omitempty"`
// DNSZone is the DNS zone we should use when configuring DNS
// This is because some clouds let us define a managed zone foo.bar, and then have
// kubernetes.dev.foo.bar, without needing to define dev.foo.bar as a hosted zone.
// DNSZone will probably be a suffix of the MasterPublicName and MasterInternalName
// Note that DNSZone can either by the host name of the zone (containing dots),
// or can be an identifier for the zone.
DNSZone string `json:"dnsZone,omitempty"`
// ClusterDNSDomain is the suffix we use for internal DNS names (normally cluster.local)
ClusterDNSDomain string `json:"clusterDNSDomain,omitempty"`
//InstancePrefix string `json:",omitempty"`
// ClusterName is a unique identifier for the cluster, and currently must be a DNS name
//ClusterName string `json:",omitempty"`
Multizone *bool `json:"multizone,omitempty"`
//ClusterIPRange string `json:",omitempty"`
// ServiceClusterIPRange is the CIDR, from the internal network, where we allocate IPs for services
ServiceClusterIPRange string `json:"serviceClusterIPRange,omitempty"`
//MasterIPRange string `json:",omitempty"`
// NonMasqueradeCIDR is the CIDR for the internal k8s network (on which pods & services live)
// It cannot overlap ServiceClusterIPRange
NonMasqueradeCIDR string `json:"nonMasqueradeCIDR,omitempty"`
// AdminAccess determines the permitted access to the admin endpoints (SSH & master HTTPS)
// Currently only a single CIDR is supported (though a richer grammar could be added in future)
AdminAccess []string `json:"adminAccess,omitempty"`
// IsolatesMasters determines whether we should lock down masters so that they are not on the pod network.
// true is the kube-up behaviour, but it is very surprising: it means that daemonsets only work on the master
// if they have hostNetwork=true.
// false is now the default, and it will:
// * give the master a normal PodCIDR
// * run kube-proxy on the master
// * enable debugging handlers on the master, so kubectl logs works
IsolateMasters *bool `json:"isolateMasters,omitempty"`
// UpdatePolicy determines the policy for applying upgrades automatically.
// Valid values:
// 'external' do not apply updates automatically - they are applied manually or by an external system
// missing: default policy (currently OS security upgrades that do not require a reboot)
UpdatePolicy *string `json:"updatePolicy,omitempty"`
//HairpinMode string `json:",omitempty"`
//
//OpencontrailTag string `json:",omitempty"`
//OpencontrailKubernetesTag string `json:",omitempty"`
//OpencontrailPublicSubnet string `json:",omitempty"`
//
//EnableClusterMonitoring string `json:",omitempty"`
//EnableL7LoadBalancing string `json:",omitempty"`
//EnableClusterUI *bool `json:",omitempty"`
//
//EnableClusterDNS *bool `json:",omitempty"`
//DNSReplicas int `json:",omitempty"`
//DNSServerIP string `json:",omitempty"`
//EnableClusterLogging *bool `json:",omitempty"`
//EnableNodeLogging *bool `json:",omitempty"`
//LoggingDestination string `json:",omitempty"`
//ElasticsearchLoggingReplicas int `json:",omitempty"`
//
//EnableClusterRegistry *bool `json:",omitempty"`
//ClusterRegistryDisk string `json:",omitempty"`
//ClusterRegistryDiskSize int `json:",omitempty"`
//
//EnableCustomMetrics *bool `json:",omitempty"`
//
//RegisterMasterKubelet *bool `json:",omitempty"`
//// Image is the default image spec to use for the cluster
//Image string `json:",omitempty"`
//KubeUser string `json:",omitempty"`
//
//// These are moved to CAStore / SecretStore
////KubePassword string
////KubeletToken string
////KubeProxyToken string
////BearerToken string
////CACert []byte
////CAKey []byte
////KubeletCert []byte
////KubeletKey []byte
////MasterCert []byte
////MasterKey []byte
////KubecfgCert []byte
////KubecfgKey []byte
//
//AdmissionControl string `json:",omitempty"`
//
//KubeImageTag string `json:",omitempty"`
//KubeDockerRegistry string `json:",omitempty"`
//KubeAddonRegistry string `json:",omitempty"`
//
//KubeletPort int `json:",omitempty"`
//
//KubeApiserverRequestTimeout int `json:",omitempty"`
//
//TerminatedPodGcThreshold string `json:",omitempty"`
//
//EnableManifestURL *bool `json:",omitempty"`
//ManifestURL string `json:",omitempty"`
//ManifestURLHeader string `json:",omitempty"`
//
//TestCluster string `json:",omitempty"`
//
//E2EStorageTestEnvironment string `json:",omitempty"`
//KubeletTestArgs string `json:",omitempty"`
//KubeletTestLogLevel string `json:",omitempty"`
//DockerTestArgs string `json:",omitempty"`
//DockerTestLogLevel string `json:",omitempty"`
//ApiserverTestArgs string `json:",omitempty"`
//ApiserverTestLogLevel string `json:",omitempty"`
//ControllerManagerTestArgs string `json:",omitempty"`
//ControllerManagerTestLogLevel string `json:",omitempty"`
//SchedulerTestArgs string `json:",omitempty"`
//SchedulerTestLogLevel string `json:",omitempty"`
//KubeProxyTestArgs string `json:",omitempty"`
//KubeProxyTestLogLevel string `json:",omitempty"`
//NodeUp *NodeUpConfig `json:",omitempty"`
// nodeSets is a list of all the NodeSets in the cluster.
// It is not exported: we populate it from other files
//nodeSets []*NodeSetConfig `json:",omitempty"`
//// Masters is the configuration for each master in the cluster
//Masters []*MasterConfig `json:",omitempty"`
// EtcdClusters stores the configuration for each cluster
EtcdClusters []*EtcdClusterSpec `json:"etcdClusters,omitempty"`
// Component configurations
Docker *DockerConfig `json:"docker,omitempty"`
KubeDNS *KubeDNSConfig `json:"kubeDNS,omitempty"`
KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"`
KubeControllerManager *KubeControllerManagerConfig `json:"kubeControllerManager,omitempty"`
KubeScheduler *KubeSchedulerConfig `json:"kubeScheduler,omitempty"`
KubeProxy *KubeProxyConfig `json:"kubeProxy,omitempty"`
Kubelet *KubeletConfigSpec `json:"kubelet,omitempty"`
MasterKubelet *KubeletConfigSpec `json:"masterKubelet,omitempty"`
// Networking configuration
Networking *NetworkingSpec `json:"networking,omitempty"`
}
type KubeDNSConfig struct {
// Image is the name of the docker image to run
Image string `json:"image,omitempty"`
Replicas int `json:"replicas,omitempty"`
Domain string `json:"domain,omitempty"`
ServerIP string `json:"serverIP,omitempty"`
}
//
//type MasterConfig struct {
// Name string `json:",omitempty"`
//
// Image string `json:",omitempty"`
// Zone string `json:",omitempty"`
// MachineType string `json:",omitempty"`
//}
//
type EtcdClusterSpec struct {
// Name is the name of the etcd cluster (main, events etc)
Name string `json:"name,omitempty"`
// EtcdMember stores the configurations for each member of the cluster (including the data volume)
Members []*EtcdMemberSpec `json:"etcdMembers,omitempty"`
}
type EtcdMemberSpec struct {
// Name is the name of the member within the etcd cluster
Name string `json:"name,omitempty"`
Zone *string `json:"zone,omitempty"`
VolumeType *string `json:"volumeType,omitempty"`
VolumeSize *int `json:"volumeSize,omitempty"`
KmsKeyId *string `json:"kmsKeyId,omitempty"`
EncryptedVolume *bool `json:"encryptedVolume,omitempty"`
}
type ClusterZoneSpec struct {
Name string `json:"name,omitempty"`
CIDR string `json:"cidr,omitempty"`
// ProviderID is the cloud provider id for the objects associated with the zone (the subnet on AWS)
ProviderID string `json:"id,omitempty"`
}
//type NodeUpConfig struct {
// Source string `json:",omitempty"`
// SourceHash string `json:",omitempty"`
//
// Tags []string `json:",omitempty"`
//
// // Assets that NodeUp should use. This is a "search-path" for resolving dependencies.
// Assets []string `json:",omitempty"`
//}
// PerformAssignments populates values that are required and immutable
// For example, it assigns stable Keys to NodeSets & Masters, and
// it assigns CIDRs to subnets
// We also assign KubernetesVersion, because we want it to be explicit
func (c *Cluster) PerformAssignments() error {
if c.Spec.NetworkCIDR == "" && !c.SharedVPC() {
// TODO: Choose non-overlapping networking CIDRs for VPCs?
c.Spec.NetworkCIDR = "172.20.0.0/16"
}
if c.Spec.NonMasqueradeCIDR == "" {
c.Spec.NonMasqueradeCIDR = "100.64.0.0/10"
}
if c.Spec.MasterPublicName == "" && c.Name != "" {
c.Spec.MasterPublicName = "api." + c.Name
}
for _, zone := range c.Spec.Zones {
err := zone.performAssignments(c)
if err != nil {
return err
}
}
err := c.ensureKubernetesVersion()
if err != nil {
return err
}
return nil
}
// FillDefaults populates default values.
// This is different from PerformAssignments, because these values are changeable, and thus we don't need to
// store them (i.e. we don't need to 'lock them')
func (c *Cluster) FillDefaults() error {
if len(c.Spec.AdminAccess) == 0 {
c.Spec.AdminAccess = append(c.Spec.AdminAccess, "0.0.0.0/0")
}
if c.Spec.Networking == nil {
c.Spec.Networking = &NetworkingSpec{}
}
if c.Spec.Networking.Classic != nil {
// OK
} else if c.Spec.Networking.Kubenet != nil {
// OK
} else if c.Spec.Networking.External != nil {
// OK
} else {
// No networking model selected; choose Kubenet
c.Spec.Networking.Kubenet = &KubenetNetworkingSpec{}
}
if c.Spec.Channel == "" {
c.Spec.Channel = DefaultChannel
}
err := c.ensureKubernetesVersion()
if err != nil {
return err
}
return nil
}
// ensureKubernetesVersion populates KubernetesVersion, if it is not already set
// It will be populated with the latest stable kubernetes version, or the version from the channel
func (c *Cluster) ensureKubernetesVersion() error {
if c.Spec.KubernetesVersion == "" {
if c.Spec.Channel != "" {
channel, err := LoadChannel(c.Spec.Channel)
if err != nil {
return err
}
if channel.Spec.Cluster.KubernetesVersion != "" {
c.Spec.KubernetesVersion = channel.Spec.Cluster.KubernetesVersion
}
}
}
if c.Spec.KubernetesVersion == "" {
latestVersion, err := FindLatestKubernetesVersion()
if err != nil {
return err
}
glog.Infof("Using kubernetes latest stable version: %s", latestVersion)
c.Spec.KubernetesVersion = latestVersion
}
return nil
}
// FindLatestKubernetesVersion returns the latest kubernetes version,
// as stored at https://storage.googleapis.com/kubernetes-release/release/stable.txt
func FindLatestKubernetesVersion() (string, error) {
stableURL := "https://storage.googleapis.com/kubernetes-release/release/stable.txt"
b, err := vfs.Context.ReadFile(stableURL)
if err != nil {
return "", fmt.Errorf("KubernetesVersion not specified, and unable to download latest version from %q: %v", stableURL, err)
}
latestVersion := strings.TrimSpace(string(b))
return latestVersion, nil
}
func (z *ClusterZoneSpec) performAssignments(c *Cluster) error {
if z.CIDR == "" {
cidr, err := z.assignCIDR(c)
if err != nil {
return err
}
glog.Infof("Assigned CIDR %s to zone %s", cidr, z.Name)
z.CIDR = cidr
}
return nil
}
func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) {
// TODO: We probably could query for the existing subnets & allocate appropriately
// for now we'll require users to set CIDRs themselves
lastCharMap := make(map[byte]bool)
for _, nodeZone := range c.Spec.Zones {
lastChar := nodeZone.Name[len(nodeZone.Name)-1]
lastCharMap[lastChar] = true
}
index := -1
if len(lastCharMap) == len(c.Spec.Zones) {
// Last char of zones are unique (GCE, AWS)
// At least on AWS, we also want 'a' to be 1, so that we don't collide with the lowest range,
// because kube-up uses that range
index = int(z.Name[len(z.Name)-1])
} else {
glog.Warningf("Last char of zone names not unique")
for i, nodeZone := range c.Spec.Zones {
if nodeZone.Name == z.Name {
index = i
break
}
}
if index == -1 {
return "", fmt.Errorf("zone not configured: %q", z.Name)
}
}
_, cidr, err := net.ParseCIDR(c.Spec.NetworkCIDR)
if err != nil {
return "", fmt.Errorf("Invalid NetworkCIDR: %q", c.Spec.NetworkCIDR)
}
networkLength, _ := cidr.Mask.Size()
// We assume a maximum of 8 subnets per network
// TODO: Does this make sense on GCE?
// TODO: Should we limit this to say 1000 IPs per subnet? (any reason to?)
index = index % 8
networkLength += 3
ip4 := cidr.IP.To4()
if ip4 != nil {
n := binary.BigEndian.Uint32(ip4)
n += uint32(index) << uint(32-networkLength)
subnetIP := make(net.IP, len(ip4))
binary.BigEndian.PutUint32(subnetIP, n)
subnetCIDR := subnetIP.String() + "/" + strconv.Itoa(networkLength)
glog.V(2).Infof("Computed CIDR for subnet in zone %q as %q", z.Name, subnetCIDR)
return subnetCIDR, nil
}
return "", fmt.Errorf("Unexpected IP address type for NetworkCIDR: %s", c.Spec.NetworkCIDR)
}
// SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
func (c *Cluster) SharedVPC() bool {
return c.Spec.NetworkID != ""
}