forked from k3s-io/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 0
/
master.go
420 lines (365 loc) · 17.5 KB
/
master.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package master
import (
"fmt"
"net"
"net/http"
"reflect"
"strconv"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/endpoints/discovery"
"k8s.io/apiserver/pkg/registry/generic"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/healthz"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/pkg/api"
apiv1 "k8s.io/kubernetes/pkg/api/v1"
appsv1beta1 "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
authenticationv1 "k8s.io/kubernetes/pkg/apis/authentication/v1"
authenticationv1beta1 "k8s.io/kubernetes/pkg/apis/authentication/v1beta1"
authorizationapiv1 "k8s.io/kubernetes/pkg/apis/authorization/v1"
authorizationapiv1beta1 "k8s.io/kubernetes/pkg/apis/authorization/v1beta1"
autoscalingapiv1 "k8s.io/kubernetes/pkg/apis/autoscaling/v1"
batchapiv1 "k8s.io/kubernetes/pkg/apis/batch/v1"
certificatesapiv1beta1 "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
extensionsapiv1beta1 "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
networkingapiv1 "k8s.io/kubernetes/pkg/apis/networking/v1"
policyapiv1beta1 "k8s.io/kubernetes/pkg/apis/policy/v1beta1"
rbacv1alpha1 "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1"
rbacv1beta1 "k8s.io/kubernetes/pkg/apis/rbac/v1beta1"
settingv1alpha1 "k8s.io/kubernetes/pkg/apis/settings/v1alpha1"
storageapiv1 "k8s.io/kubernetes/pkg/apis/storage/v1"
storageapiv1beta1 "k8s.io/kubernetes/pkg/apis/storage/v1beta1"
corev1client "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
"k8s.io/kubernetes/pkg/master/thirdparty"
"k8s.io/kubernetes/pkg/master/tunneler"
"k8s.io/kubernetes/pkg/routes"
nodeutil "k8s.io/kubernetes/pkg/util/node"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
// RESTStorage installers
admissionregistrationrest "k8s.io/kubernetes/pkg/registry/admissionregistration/rest"
appsrest "k8s.io/kubernetes/pkg/registry/apps/rest"
authenticationrest "k8s.io/kubernetes/pkg/registry/authentication/rest"
authorizationrest "k8s.io/kubernetes/pkg/registry/authorization/rest"
autoscalingrest "k8s.io/kubernetes/pkg/registry/autoscaling/rest"
batchrest "k8s.io/kubernetes/pkg/registry/batch/rest"
certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest"
corerest "k8s.io/kubernetes/pkg/registry/core/rest"
extensionsrest "k8s.io/kubernetes/pkg/registry/extensions/rest"
networkingrest "k8s.io/kubernetes/pkg/registry/networking/rest"
policyrest "k8s.io/kubernetes/pkg/registry/policy/rest"
rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
settingsrest "k8s.io/kubernetes/pkg/registry/settings/rest"
storagerest "k8s.io/kubernetes/pkg/registry/storage/rest"
)
const (
// DefaultEndpointReconcilerInterval is the default amount of time for how often the endpoints for
// the kubernetes Service are reconciled.
DefaultEndpointReconcilerInterval = 10 * time.Second
)
type Config struct {
GenericConfig *genericapiserver.Config
ClientCARegistrationHook ClientCARegistrationHook
APIResourceConfigSource serverstorage.APIResourceConfigSource
StorageFactory serverstorage.StorageFactory
EnableCoreControllers bool
EndpointReconcilerConfig EndpointReconcilerConfig
EventTTL time.Duration
KubeletClientConfig kubeletclient.KubeletClientConfig
// Used to start and monitor tunneling
Tunneler tunneler.Tunneler
EnableUISupport bool
EnableLogsSupport bool
ProxyTransport http.RoundTripper
// Values to build the IP addresses used by discovery
// The range of IPs to be assigned to services with type=ClusterIP or greater
ServiceIPRange net.IPNet
// The IP address for the GenericAPIServer service (must be inside ServiceIPRange)
APIServerServiceIP net.IP
// Port for the apiserver service.
APIServerServicePort int
// TODO, we can probably group service related items into a substruct to make it easier to configure
// the API server items and `Extra*` fields likely fit nicely together.
// The range of ports to be assigned to services with type=NodePort or greater
ServiceNodePortRange utilnet.PortRange
// Additional ports to be exposed on the GenericAPIServer service
// extraServicePorts is injectable in the event that more ports
// (other than the default 443/tcp) are exposed on the GenericAPIServer
// and those ports need to be load balanced by the GenericAPIServer
// service because this pkg is linked by out-of-tree projects
// like openshift which want to use the GenericAPIServer but also do
// more stuff.
ExtraServicePorts []api.ServicePort
// Additional ports to be exposed on the GenericAPIServer endpoints
// Port names should align with ports defined in ExtraServicePorts
ExtraEndpointPorts []api.EndpointPort
// If non-zero, the "kubernetes" services uses this port as NodePort.
KubernetesServiceNodePort int
// Number of masters running; all masters must be started with the
// same value for this field. (Numbers > 1 currently untested.)
MasterCount int
}
// EndpointReconcilerConfig holds the endpoint reconciler and endpoint reconciliation interval to be
// used by the master.
type EndpointReconcilerConfig struct {
Reconciler EndpointReconciler
Interval time.Duration
}
// Master contains state for a Kubernetes cluster master/api server.
type Master struct {
GenericAPIServer *genericapiserver.GenericAPIServer
ClientCARegistrationHook ClientCARegistrationHook
}
type completedConfig struct {
*Config
}
// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *Config) Complete() completedConfig {
c.GenericConfig.Complete()
serviceIPRange, apiServerServiceIP, err := DefaultServiceIPRange(c.ServiceIPRange)
if err != nil {
glog.Fatalf("Error determining service IP ranges: %v", err)
}
if c.ServiceIPRange.IP == nil {
c.ServiceIPRange = serviceIPRange
}
if c.APIServerServiceIP == nil {
c.APIServerServiceIP = apiServerServiceIP
}
discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: c.GenericConfig.ExternalAddress}
discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules,
discovery.CIDRRule{IPRange: c.ServiceIPRange, Address: net.JoinHostPort(c.APIServerServiceIP.String(), strconv.Itoa(c.APIServerServicePort))})
c.GenericConfig.DiscoveryAddresses = discoveryAddresses
if c.ServiceNodePortRange.Size == 0 {
// TODO: Currently no way to specify an empty range (do we need to allow this?)
// We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE)
// but then that breaks the strict nestedness of ServiceType.
// Review post-v1
c.ServiceNodePortRange = options.DefaultServiceNodePortRange
glog.Infof("Node port range unspecified. Defaulting to %v.", c.ServiceNodePortRange)
}
// enable swagger UI only if general UI support is on
c.GenericConfig.EnableSwaggerUI = c.GenericConfig.EnableSwaggerUI && c.EnableUISupport
if c.EndpointReconcilerConfig.Interval == 0 {
c.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval
}
if c.EndpointReconcilerConfig.Reconciler == nil {
// use a default endpoint reconciler if nothing is set
endpointClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
c.EndpointReconcilerConfig.Reconciler = NewMasterCountEndpointReconciler(c.MasterCount, endpointClient)
}
// this has always been hardcoded true in the past
c.GenericConfig.EnableMetrics = true
return completedConfig{c}
}
// SkipComplete provides a way to construct a server instance without config completion.
func (c *Config) SkipComplete() completedConfig {
return completedConfig{c}
}
// New returns a new instance of Master from the given config.
// Certain config fields will be set to a default value if unset.
// Certain config fields must be specified, including:
// KubeletClientConfig
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget, crdRESTOptionsGetter genericregistry.RESTOptionsGetter) (*Master, error) {
if reflect.DeepEqual(c.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) {
return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig")
}
s, err := c.Config.GenericConfig.SkipComplete().New("kube-apiserver", delegationTarget) // completion is done in Complete, no need for a second time
if err != nil {
return nil, err
}
if c.EnableUISupport {
routes.UIRedirect{}.Install(s.Handler.NonGoRestfulMux)
}
if c.EnableLogsSupport {
routes.Logs{}.Install(s.Handler.GoRestfulContainer)
}
m := &Master{
GenericAPIServer: s,
}
// install legacy rest storage
if c.APIResourceConfigSource.AnyResourcesForVersionEnabled(apiv1.SchemeGroupVersion) {
legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{
StorageFactory: c.StorageFactory,
ProxyTransport: c.ProxyTransport,
KubeletClientConfig: c.KubeletClientConfig,
EventTTL: c.EventTTL,
ServiceIPRange: c.ServiceIPRange,
ServiceNodePortRange: c.ServiceNodePortRange,
LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig,
}
m.InstallLegacyAPI(c.Config, c.Config.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider)
}
// The order here is preserved in discovery.
// If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"),
// the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer.
// This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go
// with specific priorities.
// TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery
// handlers that we have.
restStorageProviders := []RESTStorageProvider{
authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authenticator},
authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorizer},
autoscalingrest.RESTStorageProvider{},
batchrest.RESTStorageProvider{},
certificatesrest.RESTStorageProvider{},
// TODO(enisoc): Remove crdRESTOptionsGetter input argument when TPR code is removed.
extensionsrest.RESTStorageProvider{ResourceInterface: thirdparty.NewThirdPartyResourceServer(s, s.DiscoveryGroupManager, c.StorageFactory, crdRESTOptionsGetter)},
networkingrest.RESTStorageProvider{},
policyrest.RESTStorageProvider{},
rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorizer},
settingsrest.RESTStorageProvider{},
storagerest.RESTStorageProvider{},
// keep apps after extensions so legacy clients resolve the extensions versions of shared resource names.
// See https://github.com/kubernetes/kubernetes/issues/42392
appsrest.RESTStorageProvider{},
admissionregistrationrest.RESTStorageProvider{},
}
m.InstallAPIs(c.Config.APIResourceConfigSource, c.Config.GenericConfig.RESTOptionsGetter, restStorageProviders...)
if c.Tunneler != nil {
m.installTunneler(c.Tunneler, corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig).Nodes())
}
if err := m.GenericAPIServer.AddPostStartHook("ca-registration", c.ClientCARegistrationHook.PostStartHook); err != nil {
glog.Fatalf("Error registering PostStartHook %q: %v", "ca-registration", err)
}
return m, nil
}
func (m *Master) InstallLegacyAPI(c *Config, restOptionsGetter generic.RESTOptionsGetter, legacyRESTStorageProvider corerest.LegacyRESTStorageProvider) {
legacyRESTStorage, apiGroupInfo, err := legacyRESTStorageProvider.NewLegacyRESTStorage(restOptionsGetter)
if err != nil {
glog.Fatalf("Error building core storage: %v", err)
}
if c.EnableCoreControllers {
coreClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
bootstrapController := c.NewBootstrapController(legacyRESTStorage, coreClient, coreClient)
if err := m.GenericAPIServer.AddPostStartHook("bootstrap-controller", bootstrapController.PostStartHook); err != nil {
glog.Fatalf("Error registering PostStartHook %q: %v", "bootstrap-controller", err)
}
}
if err := m.GenericAPIServer.InstallLegacyAPIGroup(genericapiserver.DefaultLegacyAPIPrefix, &apiGroupInfo); err != nil {
glog.Fatalf("Error in registering group versions: %v", err)
}
}
func (m *Master) installTunneler(nodeTunneler tunneler.Tunneler, nodeClient corev1client.NodeInterface) {
nodeTunneler.Run(nodeAddressProvider{nodeClient}.externalAddresses)
m.GenericAPIServer.AddHealthzChecks(healthz.NamedCheck("SSH Tunnel Check", tunneler.TunnelSyncHealthChecker(nodeTunneler)))
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "apiserver_proxy_tunnel_sync_latency_secs",
Help: "The time since the last successful synchronization of the SSH tunnels for proxy requests.",
}, func() float64 { return float64(nodeTunneler.SecondsSinceSync()) })
}
// RESTStorageProvider is a factory type for REST storage.
type RESTStorageProvider interface {
GroupName() string
NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool)
}
// InstallAPIs will install the APIs for the restStorageProviders if they are enabled.
func (m *Master) InstallAPIs(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter, restStorageProviders ...RESTStorageProvider) {
apiGroupsInfo := []genericapiserver.APIGroupInfo{}
for _, restStorageBuilder := range restStorageProviders {
groupName := restStorageBuilder.GroupName()
if !apiResourceConfigSource.AnyResourcesForGroupEnabled(groupName) {
glog.V(1).Infof("Skipping disabled API group %q.", groupName)
continue
}
apiGroupInfo, enabled := restStorageBuilder.NewRESTStorage(apiResourceConfigSource, restOptionsGetter)
if !enabled {
glog.Warningf("Problem initializing API group %q, skipping.", groupName)
continue
}
glog.V(1).Infof("Enabling API group %q.", groupName)
if postHookProvider, ok := restStorageBuilder.(genericapiserver.PostStartHookProvider); ok {
name, hook, err := postHookProvider.PostStartHook()
if err != nil {
glog.Fatalf("Error building PostStartHook: %v", err)
}
if err := m.GenericAPIServer.AddPostStartHook(name, hook); err != nil {
glog.Fatalf("Error registering PostStartHook %q: %v", name, err)
}
}
apiGroupsInfo = append(apiGroupsInfo, apiGroupInfo)
}
for i := range apiGroupsInfo {
if err := m.GenericAPIServer.InstallAPIGroup(&apiGroupsInfo[i]); err != nil {
glog.Fatalf("Error in registering group versions: %v", err)
}
}
}
type nodeAddressProvider struct {
nodeClient corev1client.NodeInterface
}
func (n nodeAddressProvider) externalAddresses() ([]string, error) {
preferredAddressTypes := []apiv1.NodeAddressType{
apiv1.NodeExternalIP,
}
nodes, err := n.nodeClient.List(metav1.ListOptions{})
if err != nil {
return nil, err
}
addrs := []string{}
for ix := range nodes.Items {
node := &nodes.Items[ix]
addr, err := nodeutil.GetPreferredNodeAddress(node, preferredAddressTypes)
if err != nil {
return nil, err
}
addrs = append(addrs, addr)
}
return addrs, nil
}
func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
ret := serverstorage.NewResourceConfig()
// NOTE: GroupVersions listed here will be enabled by default. Don't put alpha versions in the list.
ret.EnableVersions(
apiv1.SchemeGroupVersion,
extensionsapiv1beta1.SchemeGroupVersion,
batchapiv1.SchemeGroupVersion,
authenticationv1.SchemeGroupVersion,
authenticationv1beta1.SchemeGroupVersion,
autoscalingapiv1.SchemeGroupVersion,
appsv1beta1.SchemeGroupVersion,
policyapiv1beta1.SchemeGroupVersion,
rbacv1beta1.SchemeGroupVersion,
// Don't copy this pattern. We enable rbac/v1alpha1 and settings/v1laph1
// by default only because they were enabled in previous releases.
// See https://github.com/kubernetes/kubernetes/pull/47690.
// TODO: disable rbac/v1alpha1 and settings/v1alpha1 by default in 1.8
rbacv1alpha1.SchemeGroupVersion,
settingv1alpha1.SchemeGroupVersion,
storageapiv1.SchemeGroupVersion,
storageapiv1beta1.SchemeGroupVersion,
certificatesapiv1beta1.SchemeGroupVersion,
authorizationapiv1.SchemeGroupVersion,
authorizationapiv1beta1.SchemeGroupVersion,
networkingapiv1.SchemeGroupVersion,
)
// all extensions resources except these are disabled by default
ret.EnableResources(
extensionsapiv1beta1.SchemeGroupVersion.WithResource("daemonsets"),
extensionsapiv1beta1.SchemeGroupVersion.WithResource("deployments"),
extensionsapiv1beta1.SchemeGroupVersion.WithResource("ingresses"),
extensionsapiv1beta1.SchemeGroupVersion.WithResource("networkpolicies"),
extensionsapiv1beta1.SchemeGroupVersion.WithResource("replicasets"),
extensionsapiv1beta1.SchemeGroupVersion.WithResource("thirdpartyresources"),
extensionsapiv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"),
)
return ret
}