forked from kubernetes/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 1
/
genericapiserver.go
498 lines (417 loc) · 18.5 KB
/
genericapiserver.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package server
import (
"fmt"
"mime"
"net/http"
"strings"
"sync"
"time"
systemd "github.com/coreos/go-systemd/daemon"
"github.com/emicklei/go-restful"
"github.com/emicklei/go-restful/swagger"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/apimachinery"
"k8s.io/apimachinery/pkg/apimachinery/registered"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
openapicommon "k8s.io/apimachinery/pkg/openapi"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
genericapi "k8s.io/apiserver/pkg/endpoints"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/mux"
genericmux "k8s.io/apiserver/pkg/server/mux"
"k8s.io/apiserver/pkg/server/routes"
restclient "k8s.io/client-go/rest"
)
// Info about an API group.
type APIGroupInfo struct {
GroupMeta apimachinery.GroupMeta
// Info about the resources in this group. Its a map from version to resource to the storage.
VersionedResourcesStorageMap map[string]map[string]rest.Storage
// OptionsExternalVersion controls the APIVersion used for common objects in the
// schema like api.Status, api.DeleteOptions, and metav1.ListOptions. Other implementors may
// define a version "v1beta1" but want to use the Kubernetes "v1" internal objects.
// If nil, defaults to groupMeta.GroupVersion.
// TODO: Remove this when https://github.com/kubernetes/kubernetes/issues/19018 is fixed.
OptionsExternalVersion *schema.GroupVersion
// MetaGroupVersion defaults to "meta.k8s.io/v1" and is the scheme group version used to decode
// common API implementations like ListOptions. Future changes will allow this to vary by group
// version (for when the inevitable meta/v2 group emerges).
MetaGroupVersion *schema.GroupVersion
// Scheme includes all of the types used by this group and how to convert between them (or
// to convert objects from outside of this group that are accepted in this API).
// TODO: replace with interfaces
Scheme *runtime.Scheme
// NegotiatedSerializer controls how this group encodes and decodes data
NegotiatedSerializer runtime.NegotiatedSerializer
// ParameterCodec performs conversions for query parameters passed to API calls
ParameterCodec runtime.ParameterCodec
// SubresourceGroupVersionKind contains the GroupVersionKind overrides for each subresource that is
// accessible from this API group version. The GroupVersionKind is that of the external version of
// the subresource. The key of this map should be the path of the subresource. The keys here should
// match the keys in the Storage map above for subresources.
SubresourceGroupVersionKind map[string]schema.GroupVersionKind
}
// GenericAPIServer contains state for a Kubernetes cluster api server.
type GenericAPIServer struct {
// discoveryAddresses is used to build cluster IPs for discovery.
discoveryAddresses DiscoveryAddresses
// LoopbackClientConfig is a config for a privileged loopback connection to the API server
LoopbackClientConfig *restclient.Config
// minRequestTimeout is how short the request timeout can be. This is used to build the RESTHandler
minRequestTimeout time.Duration
// legacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
// to InstallLegacyAPIGroup
legacyAPIGroupPrefixes sets.String
// admissionControl is used to build the RESTStorage that backs an API Group.
admissionControl admission.Interface
// requestContextMapper provides a way to get the context for a request. It may be nil.
requestContextMapper apirequest.RequestContextMapper
// The registered APIs
HandlerContainer *genericmux.APIContainer
SecureServingInfo *SecureServingInfo
// numerical ports, set after listening
effectiveSecurePort int
// ExternalAddress is the address (hostname or IP and port) that should be used in
// external (public internet) URLs for this GenericAPIServer.
ExternalAddress string
// storage contains the RESTful endpoints exposed by this GenericAPIServer
storage map[string]rest.Storage
// Serializer controls how common API objects not in a group/version prefix are serialized for this server.
// Individual APIGroups may define their own serializers.
Serializer runtime.NegotiatedSerializer
// "Outputs"
Handler http.Handler
// FallThroughHandler is the final HTTP handler in the chain.
// It comes after all filters and the API handling
FallThroughHandler *mux.PathRecorderMux
// listedPathProvider is a lister which provides the set of paths to show at /
listedPathProvider routes.ListedPathProvider
// Map storing information about all groups to be exposed in discovery response.
// The map is from name to the group.
// The slice preserves group name insertion order.
apiGroupsForDiscoveryLock sync.RWMutex
apiGroupsForDiscovery map[string]metav1.APIGroup
apiGroupNamesForDiscovery []string
// Enable swagger and/or OpenAPI if these configs are non-nil.
swaggerConfig *swagger.Config
openAPIConfig *openapicommon.Config
// PostStartHooks are each called after the server has started listening, in a separate go func for each
// with no guarantee of ordering between them. The map key is a name used for error reporting.
// It may kill the process with a panic if it wishes to by returning an error.
postStartHookLock sync.Mutex
postStartHooks map[string]postStartHookEntry
postStartHooksCalled bool
disabledPostStartHooks sets.String
// healthz checks
healthzLock sync.Mutex
healthzChecks []healthz.HealthzChecker
healthzCreated bool
}
// DelegationTarget is an interface which allows for composition of API servers with top level handling that works
// as expected.
type DelegationTarget interface {
// UnprotectedHandler returns a handler that is NOT protected by a normal chain
UnprotectedHandler() http.Handler
// RequestContextMapper returns the existing RequestContextMapper. Because we cannot rewire all existing
// uses of this function, this will be used in any delegating API server
RequestContextMapper() apirequest.RequestContextMapper
// PostStartHooks returns the post-start hooks that need to be combined
PostStartHooks() map[string]postStartHookEntry
// HealthzChecks returns the healthz checks that need to be combined
HealthzChecks() []healthz.HealthzChecker
// ListedPaths returns the paths for supporting an index
ListedPaths() []string
}
func (s *GenericAPIServer) UnprotectedHandler() http.Handler {
return s.HandlerContainer.ServeMux
}
func (s *GenericAPIServer) PostStartHooks() map[string]postStartHookEntry {
return s.postStartHooks
}
func (s *GenericAPIServer) HealthzChecks() []healthz.HealthzChecker {
return s.healthzChecks
}
func (s *GenericAPIServer) ListedPaths() []string {
return s.listedPathProvider.ListedPaths()
}
var EmptyDelegate = emptyDelegate{
requestContextMapper: apirequest.NewRequestContextMapper(),
}
type emptyDelegate struct {
requestContextMapper apirequest.RequestContextMapper
}
func (s emptyDelegate) UnprotectedHandler() http.Handler {
return http.NotFoundHandler()
}
func (s emptyDelegate) PostStartHooks() map[string]postStartHookEntry {
return map[string]postStartHookEntry{}
}
func (s emptyDelegate) HealthzChecks() []healthz.HealthzChecker {
return []healthz.HealthzChecker{}
}
func (s emptyDelegate) ListedPaths() []string {
return []string{}
}
func (s emptyDelegate) RequestContextMapper() apirequest.RequestContextMapper {
return s.requestContextMapper
}
func init() {
// Send correct mime type for .svg files.
// TODO: remove when https://github.com/golang/go/commit/21e47d831bafb59f22b1ea8098f709677ec8ce33
// makes it into all of our supported go versions (only in v1.7.1 now).
mime.AddExtensionType(".svg", "image/svg+xml")
}
// RequestContextMapper is exposed so that third party resource storage can be build in a different location.
// TODO refactor third party resource storage
func (s *GenericAPIServer) RequestContextMapper() apirequest.RequestContextMapper {
return s.requestContextMapper
}
// MinRequestTimeout is exposed so that third party resource storage can be build in a different location.
// TODO refactor third party resource storage
func (s *GenericAPIServer) MinRequestTimeout() time.Duration {
return s.minRequestTimeout
}
type preparedGenericAPIServer struct {
*GenericAPIServer
}
// PrepareRun does post API installation setup steps.
func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
if s.swaggerConfig != nil {
routes.Swagger{Config: s.swaggerConfig}.Install(s.HandlerContainer)
}
if s.openAPIConfig != nil {
routes.OpenAPI{
Config: s.openAPIConfig,
}.Install(s.HandlerContainer, s.FallThroughHandler)
}
s.installHealthz()
return preparedGenericAPIServer{s}
}
// Run spawns the http servers (secure and insecure). It only returns if stopCh is closed
// or one of the ports cannot be listened on initially.
func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
err := s.NonBlockingRun(stopCh)
if err != nil {
return err
}
<-stopCh
return nil
}
// NonBlockingRun spawns the http servers (secure and insecure). An error is
// returned if either of the ports cannot be listened on.
func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
// Use an internal stop channel to allow cleanup of the listeners on error.
internalStopCh := make(chan struct{})
if s.SecureServingInfo != nil && s.Handler != nil {
if err := s.serveSecurely(internalStopCh); err != nil {
close(internalStopCh)
return err
}
}
// Now that both listeners have bound successfully, it is the
// responsibility of the caller to close the provided channel to
// ensure cleanup.
go func() {
<-stopCh
close(internalStopCh)
}()
s.RunPostStartHooks()
if _, err := systemd.SdNotify(true, "READY=1\n"); err != nil {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
return nil
}
// EffectiveSecurePort returns the secure port we bound to.
func (s *GenericAPIServer) EffectiveSecurePort() int {
return s.effectiveSecurePort
}
// installAPIResources is a private method for installing the REST storage backing each api groupversionresource
func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions {
if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
glog.Warningf("Skipping API %v because it has no resources.", groupVersion)
continue
}
apiGroupVersion := s.getAPIGroupVersion(apiGroupInfo, groupVersion, apiPrefix)
if apiGroupInfo.OptionsExternalVersion != nil {
apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
}
if err := apiGroupVersion.InstallREST(s.HandlerContainer.Container); err != nil {
return fmt.Errorf("Unable to setup API %v: %v", apiGroupInfo, err)
}
}
return nil
}
func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
if !s.legacyAPIGroupPrefixes.Has(apiPrefix) {
return fmt.Errorf("%q is not in the allowed legacy API prefixes: %v", apiPrefix, s.legacyAPIGroupPrefixes.List())
}
if err := s.installAPIResources(apiPrefix, apiGroupInfo); err != nil {
return err
}
// setup discovery
apiVersions := []string{}
for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions {
apiVersions = append(apiVersions, groupVersion.Version)
}
// Install the version handler.
// Add a handler at /<apiPrefix> to enumerate the supported api versions.
genericapi.AddApiWebService(s.Serializer, s.HandlerContainer.Container, apiPrefix, func(req *restful.Request) *metav1.APIVersions {
clientIP := utilnet.GetClientIP(req.Request)
apiVersionsForDiscovery := metav1.APIVersions{
ServerAddressByClientCIDRs: s.discoveryAddresses.ServerAddressByClientCIDRs(clientIP),
Versions: apiVersions,
}
return &apiVersionsForDiscovery
})
return nil
}
// Exposes the given api group in the API.
func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error {
// Do not register empty group or empty version. Doing so claims /apis/ for the wrong entity to be returned.
// Catching these here places the error much closer to its origin
if len(apiGroupInfo.GroupMeta.GroupVersion.Group) == 0 {
return fmt.Errorf("cannot register handler with an empty group for %#v", *apiGroupInfo)
}
if len(apiGroupInfo.GroupMeta.GroupVersion.Version) == 0 {
return fmt.Errorf("cannot register handler with an empty version for %#v", *apiGroupInfo)
}
if err := s.installAPIResources(APIGroupPrefix, apiGroupInfo); err != nil {
return err
}
// setup discovery
// Install the version handler.
// Add a handler at /apis/<groupName> to enumerate all versions supported by this group.
apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{}
for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions {
// Check the config to make sure that we elide versions that don't have any resources
if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
continue
}
apiVersionsForDiscovery = append(apiVersionsForDiscovery, metav1.GroupVersionForDiscovery{
GroupVersion: groupVersion.String(),
Version: groupVersion.Version,
})
}
preferedVersionForDiscovery := metav1.GroupVersionForDiscovery{
GroupVersion: apiGroupInfo.GroupMeta.GroupVersion.String(),
Version: apiGroupInfo.GroupMeta.GroupVersion.Version,
}
apiGroup := metav1.APIGroup{
Name: apiGroupInfo.GroupMeta.GroupVersion.Group,
Versions: apiVersionsForDiscovery,
PreferredVersion: preferedVersionForDiscovery,
}
s.AddAPIGroupForDiscovery(apiGroup)
s.HandlerContainer.Add(genericapi.NewGroupWebService(s.Serializer, APIGroupPrefix+"/"+apiGroup.Name, apiGroup))
return nil
}
// AddAPIGroupForDiscovery adds the specified group to the list served to discovery queries.
// Groups are listed in the order they are added.
func (s *GenericAPIServer) AddAPIGroupForDiscovery(apiGroup metav1.APIGroup) {
s.apiGroupsForDiscoveryLock.Lock()
defer s.apiGroupsForDiscoveryLock.Unlock()
// Insert the group into the ordered list if it is not already present
if _, exists := s.apiGroupsForDiscovery[apiGroup.Name]; !exists {
s.apiGroupNamesForDiscovery = append(s.apiGroupNamesForDiscovery, apiGroup.Name)
}
s.apiGroupsForDiscovery[apiGroup.Name] = apiGroup
}
func (s *GenericAPIServer) RemoveAPIGroupForDiscovery(groupName string) {
s.apiGroupsForDiscoveryLock.Lock()
defer s.apiGroupsForDiscoveryLock.Unlock()
// Remove the group from the ordered list
// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
newOrder := s.apiGroupNamesForDiscovery[:0]
for _, orderedGroupName := range s.apiGroupNamesForDiscovery {
if orderedGroupName != groupName {
newOrder = append(newOrder, orderedGroupName)
}
}
s.apiGroupNamesForDiscovery = newOrder
delete(s.apiGroupsForDiscovery, groupName)
}
func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion, apiPrefix string) *genericapi.APIGroupVersion {
storage := make(map[string]rest.Storage)
for k, v := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
storage[strings.ToLower(k)] = v
}
version := s.newAPIGroupVersion(apiGroupInfo, groupVersion)
version.Root = apiPrefix
version.Storage = storage
return version
}
func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion) *genericapi.APIGroupVersion {
return &genericapi.APIGroupVersion{
GroupVersion: groupVersion,
MetaGroupVersion: apiGroupInfo.MetaGroupVersion,
ParameterCodec: apiGroupInfo.ParameterCodec,
Serializer: apiGroupInfo.NegotiatedSerializer,
Creater: apiGroupInfo.Scheme,
Convertor: apiGroupInfo.Scheme,
UnsafeConvertor: runtime.UnsafeObjectConvertor(apiGroupInfo.Scheme),
Copier: apiGroupInfo.Scheme,
Defaulter: apiGroupInfo.Scheme,
Typer: apiGroupInfo.Scheme,
SubresourceGroupVersionKind: apiGroupInfo.SubresourceGroupVersionKind,
Linker: apiGroupInfo.GroupMeta.SelfLinker,
Mapper: apiGroupInfo.GroupMeta.RESTMapper,
Admit: s.admissionControl,
Context: s.RequestContextMapper(),
MinRequestTimeout: s.minRequestTimeout,
}
}
// DynamicApisDiscovery returns a webservice serving api group discovery.
// Note: during the server runtime apiGroupsForDiscovery might change.
func (s *GenericAPIServer) DynamicApisDiscovery() *restful.WebService {
return genericapi.NewApisWebService(s.Serializer, APIGroupPrefix, func(req *restful.Request) []metav1.APIGroup {
s.apiGroupsForDiscoveryLock.RLock()
defer s.apiGroupsForDiscoveryLock.RUnlock()
sortedGroups := []metav1.APIGroup{}
// ranging over apiGroupNamesForDiscovery preserves the registration order
for _, groupName := range s.apiGroupNamesForDiscovery {
sortedGroups = append(sortedGroups, s.apiGroupsForDiscovery[groupName])
}
clientIP := utilnet.GetClientIP(req.Request)
serverCIDR := s.discoveryAddresses.ServerAddressByClientCIDRs(clientIP)
groups := make([]metav1.APIGroup, len(sortedGroups))
for i := range sortedGroups {
groups[i] = sortedGroups[i]
groups[i].ServerAddressByClientCIDRs = serverCIDR
}
return groups
})
}
// NewDefaultAPIGroupInfo returns an APIGroupInfo stubbed with "normal" values
// exposed for easier composition from other packages
func NewDefaultAPIGroupInfo(group string, registry *registered.APIRegistrationManager, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo {
groupMeta := registry.GroupOrDie(group)
return APIGroupInfo{
GroupMeta: *groupMeta,
VersionedResourcesStorageMap: map[string]map[string]rest.Storage{},
// TODO unhardcode this. It was hardcoded before, but we need to re-evaluate
OptionsExternalVersion: &schema.GroupVersion{Version: "v1"},
Scheme: scheme,
ParameterCodec: parameterCodec,
NegotiatedSerializer: codecs,
}
}